Most of basic functionality implemented

This commit is contained in:
Andrew Phillips
2023-08-31 19:38:59 +00:00
parent b9bf5a831e
commit 49a77f9090
5 changed files with 1382 additions and 142 deletions

225
src/compression.rs Normal file → Executable file
View File

@@ -1,86 +1,166 @@
use std::fmt;
use std::os::unix::fs::PermissionsExt;
use anyhow::{Context, Result, anyhow};
use strum::IntoEnumIterator;
use std::fs::File;
use std::io;
use std::io::Write;
use std::process::{Command,Stdio};
use std::path::PathBuf;
use std::env;
use std::fs;
use std::os::unix::fs::PermissionsExt;
use log::*;
use lazy_static::lazy_static;
extern crate enum_map;
use enum_map::enum_map;
use enum_map::{EnumMap,Enum};
#[derive(Debug, Clone)]
pub struct CompressionType {
name: String,
binary: String,
compress: String,
decompress: String,
#[derive(Debug, Eq, PartialEq, Clone, strum::EnumIter, strum::Display, strum::EnumString, Enum)]
#[strum(ascii_case_insensitive)]
pub enum CompressionType {
LZ4,
GZip,
BZip2,
XZ,
ZStd,
None
}
impl fmt::Display for CompressionType {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "[name='{}', binary='{}', compress='{}', decompress='{}']", self.name, self.binary, self.compress, self.decompress)
#[derive(Debug, Eq, PartialEq, Clone)]
pub struct CompressionProgram {
pub program: String,
pub compress: Vec<String>,
pub decompress: Vec<String>,
pub supported: bool
}
lazy_static! {
static ref COMPRESSION_PROGRAMS: EnumMap<CompressionType, Option<CompressionProgram>> = enum_map! {
CompressionType::LZ4 => Some(CompressionProgram::new("lz4", vec!["-qcf"], vec!["-dcf"])),
CompressionType::GZip => Some(CompressionProgram::new("gzip", vec!["-qcf"], vec!["-dcf"])),
CompressionType::BZip2 => Some(CompressionProgram::new("bzip2", vec!["-qcf"], vec!["-dcf"])),
CompressionType::XZ => Some(CompressionProgram::new("xz", vec!["-qcf"], vec!["-dcf"])),
CompressionType::ZStd => Some(CompressionProgram::new("zstd", vec!["-qcf"], vec!["-dcf"])),
CompressionType::None => None
};
}
impl CompressionProgram {
pub fn new(program: &str, compress: Vec<&str>, decompress: Vec<&str>) -> CompressionProgram {
let program_path = get_program_path(program);
let supported = program_path.is_ok();
CompressionProgram {
program: program_path.unwrap_or(program.to_string()),
compress: compress.iter().map(|s| {s.to_string()}).collect(),
decompress: decompress.iter().map(|s| {s.to_string()}).collect(),
supported
}
}
}
pub fn add_compression_type(compression_types: &mut Vec<CompressionType>, name: String, binary: String, compress: String, decompress: String) {
let path = is_program_in_path(binary);
pub trait CompressionEngine {
fn is_supported(&self) -> bool;
fn cat(&self, file_path: PathBuf) -> Result<()>;
fn create(&self, file_path: PathBuf) -> Result<Box<dyn Write>>;
}
if let Ok(path) = path {
compression_types.push(
CompressionType {
name,
binary: path,
compress,
decompress
});
impl CompressionEngine for CompressionProgram {
fn is_supported(&self) -> bool {
self.supported
}
fn cat(&self, file_path: PathBuf) -> Result<()> {
debug!("COMPRESSION: Outputting {:?} to STDOUT using {:?}", file_path, *self);
let program = self.program.clone();
let args = self.decompress.clone();
debug!("COMPRESSION: Executing command: {:?} {:?} writing to {:?}", program, args, file_path);
let file = File::open(file_path).context("Unable to open file for reading")?;
let mut process = Command::new(program.clone())
.args(args.clone())
.stdin(file)
.spawn()
.context(anyhow!("Unable to spawn child process: {:?} {:?}", program, args))?;
let result = process.wait()
.context(anyhow!("Unable to wait for child process: {:?} {:?}", program, args))?;
if result.success() {
Ok(())
} else {
Err(anyhow!("Decompression program returned {}", result))
}
}
fn create(&self, file_path: PathBuf) -> Result<Box<dyn Write>> {
debug!("COMPRESSION: Writting to {:?} using {:?}", file_path, *self);
let program = self.program.clone();
let args = self.compress.clone();
debug!("COMPRESSION: Executing command: {:?} {:?} writing to {:?}", program, args, file_path);
let file = File::create(file_path).context("Unable to open file for writing")?;
let process = Command::new(program.clone())
.args(args.clone())
.stdin(Stdio::piped())
.stdout(file)
.spawn()
.context(anyhow!("Problem spawning child process: {:?} {:?}", program, args))?;
Ok(Box::new(process.stdin.unwrap()))
}
}
pub fn supported_compression_types() -> Vec<CompressionType> {
let mut compression_types = Vec::new();
add_compression_type(&mut compression_types, "lz4".to_string(), "lz4".to_string(), "-qc".to_string() ,"-dc".to_string());
add_compression_type(&mut compression_types, "gzip".to_string(), "gzip".to_string(), "-qc".to_string() ,"-dc".to_string());
add_compression_type(&mut compression_types, "bzip2".to_string(), "bzip2".to_string(), "-qc".to_string() ,"-dc".to_string());
add_compression_type(&mut compression_types, "xz".to_string(), "xz".to_string(), "-qc".to_string() ,"-dc".to_string());
compression_types.push(
CompressionType {
name: "none".to_string(),
binary: "".to_string(),
compress: "".to_string(),
decompress: "".to_string(),
});
return compression_types;
#[derive(Debug, Eq, PartialEq, Clone)]
pub struct CompressionEngineNone {
}
pub fn get_compression_default(compression_types: Vec<CompressionType>) -> Result<CompressionType, String> {
debug!("Compression type: default");
match compression_types.first() {
None => Err(String::from("Unable to find default compression type")),
Some(compression_type) => Ok(compression_type.clone())
impl CompressionEngineNone {
pub fn new() -> CompressionEngineNone {
CompressionEngineNone {}
}
}
pub fn get_compression_named(compression_types: Vec<CompressionType>, compression_name: String) -> Result<CompressionType, String> {
debug!("Compression type: {}", compression_name);
match compression_types.iter().find(|&c| c.name == *compression_name) {
None => Err(format!("Unable to find compression type: {}", compression_name)),
Some(compression_type) => Ok(compression_type.clone())
impl Default for CompressionEngineNone {
fn default() -> Self {
Self::new()
}
}
pub fn get_compression(compression_name: Option<String>) -> Result<CompressionType, String> {
let compression_types = supported_compression_types();
impl CompressionEngine for CompressionEngineNone {
fn is_supported(&self) -> bool {
true
}
match compression_name {
None => get_compression_default(compression_types),
Some(compression_name) => get_compression_named(compression_types, compression_name),
fn cat(&self, file_path: PathBuf) -> Result<()> {
debug!("COMPRESSION: Outputting {:?} to STDOUT using {:?}", file_path, *self);
let mut stdout = io::stdout().lock();
let mut file = File::open(file_path)?;
io::copy(&mut file, &mut stdout)?;
stdout.flush()?;
Ok(())
}
fn create(&self, file_path: PathBuf) -> Result<Box<dyn Write>> {
debug!("COMPRESSION: Writting to {:?} using {:?}", file_path, *self);
Ok(Box::new(File::create(file_path)?))
}
}
fn is_program_in_path(program: String) -> Result<String, ()> {
debug!("Looking for executable: {}", program);
fn get_program_path(program: &str) -> Result<String> {
debug!("COMPRESSION: Looking for executable: {}", program);
if let Ok(path) = env::var("PATH") {
for p in path.split(':') {
let p_str = format!("{}/{}", p, program);
@@ -94,5 +174,34 @@ fn is_program_in_path(program: String) -> Result<String, ()> {
}
}
}
Err(())
Err(anyhow!("Unable to find binary {} in PATH", program))
}
pub fn get_program(compression_type: CompressionType) -> Result<CompressionProgram> {
match &COMPRESSION_PROGRAMS[compression_type.clone()] {
Some(compression_program) => Ok(compression_program.clone()),
None => Err(anyhow!("Compression type {} has no program", compression_type))
}
}
pub fn get_engine(compression_type: CompressionType) -> Result<Box<dyn CompressionEngine>> {
match compression_type {
CompressionType::None => Ok(Box::new(CompressionEngineNone::new())),
compression_type => Ok(Box::new(COMPRESSION_PROGRAMS[compression_type.clone()].clone().unwrap()))
}
}
pub fn default_type() -> CompressionType {
let mut default = CompressionType::None;
for compression_type in CompressionType::iter() {
let compression_engine = get_engine(compression_type.clone()).expect("Missing engine");
if compression_engine.is_supported() {
default = compression_type;
break;
}
}
default
}

374
src/db.rs
View File

@@ -1,36 +1,372 @@
use rusqlite::{params, Connection, Error};
use std::path::PathBuf;
use std::collections::HashMap;
use std::rc::Rc;
use anyhow::{Context, Result, Error};
use rusqlite::{Connection, OpenFlags};
use rusqlite_migration::{Migrations, M};
use log::*;
use chrono::prelude::*;
use lazy_static::lazy_static;
use log::*;
lazy_static! {
static ref MIGRATIONS: Migrations<'static> = Migrations::new(vec![
M::up("CREATE TABLE keep(
id INTEGER AUTOINCREMENT NOT NULL,
M::up("CREATE TABLE items(
id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
ts TEXT NOT NULL,
compress TEXT NOT NULL,
hostname TEXT NOT NULL,
comment TEXT NOT NULL)
PRIMARY KEY(id);"),
size INTEGER NULL,
compression TEXT NOT NULL)"),
M::up("CREATE TABLE tags (
id INTEGER NOT NULL,
name TEXT NOT NULL,
FOREIGN KEY(id) REFERENCES keep(id) ON DELETE CASCADE,
FOREIGN KEY(id) REFERENCES items(id) ON DELETE CASCADE,
PRIMARY KEY(id, name));"),
M::up("CREATE TABLE metas (
id INTEGER NOT NULL,
name TEXT NOT NULL,
value TEXT NOT NULL,
FOREIGN KEY(id) REFERENCES items(id) ON DELETE CASCADE,
PRIMARY KEY(id, name));")
]);
}
fn open(path: String) -> Result<Connection, String> {
debug!("Opening DB {}", path);
#[derive(Debug, Clone)]
pub struct Item {
pub id: Option<i64>,
pub ts: DateTime<Utc>,
pub size: Option<i64>,
pub compression: String
}
match Connection::open(path) {
Ok(mut conn) => match conn.pragma_update(None, "foreign_keys", "ON") {
Ok(()) => match MIGRATIONS.to_latest(&mut conn) {
Ok(()) => Ok(conn),
Err(e) => Err(format!("Error migrating sqlite schema: {}", e))
},
Err(e) => Err(format!("Error setting sqlite pragma: {}", e))
#[derive(Debug, Clone)]
pub struct Tag {
pub id: i64,
pub name: String
}
#[derive(Debug, Clone)]
pub struct Meta {
pub id: i64,
pub name: String,
pub value: String
}
pub fn open(path: PathBuf) -> Result<Connection, Error> {
debug!("DB: Opening file: {:?}", path);
let mut conn = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_CREATE)
.context("Problem opening file")?;
conn.pragma_update(None, "foreign_keys", "ON")
.context("Problem enabling SQLite foreign_keys pragma")?;
MIGRATIONS.to_latest(&mut conn)
.context("Problem performing database migrations")?;
rusqlite::vtab::array::load_module(&conn)
.context("Problem enabling array module")?;
Ok(conn)
}
pub fn insert_item(conn: &Connection, item: Item) -> Result<i64> {
debug!("DB: Inserting item: {:?}", item);
conn.execute(
"INSERT INTO items (ts, size, compression) VALUES (?1, ?2, ?3)",
(item.ts, item.size, item.compression))?;
Ok(conn.last_insert_rowid())
}
pub fn update_item(conn: &Connection, item: Item) -> Result<()> {
debug!("DB: Updating item: {:?}", item);
conn.execute(
"UPDATE items SET size=?2, compression=?3 WHERE id=?1",
(item.id, item.size, item.compression))?;
Ok(())
}
pub fn delete_item(conn: &Connection, item: Item) -> Result<()> {
debug!("DB: Deleting item: {:?}", item);
conn.execute("DELETE FROM items WHERE id=?1", [item.id])?;
Ok(())
}
pub fn query_delete_meta(conn: &Connection, meta: Meta) -> Result<()> {
debug!("DB: Deleting meta: {:?}", meta);
conn.execute(
"DELETE FROM metas WHERE id=?1 AND name=?2",
(meta.id, meta.name))?;
Ok(())
}
pub fn query_upsert_meta(conn: &Connection, meta: Meta) -> Result<()> {
debug!("DB: Inserting meta: {:?}", meta);
conn.execute(
"INSERT INTO metas (id, name, value) VALUES (?1, ?2, ?3)
ON CONFLICT(id, name) DO UPDATE SET value=?3",
(meta.id, meta.name, meta.value))?;
Ok(())
}
pub fn store_meta(conn: &Connection, meta: Meta) -> Result<()> {
debug!("DB: Storing meta: {:?}", meta);
if meta.value.eq("") {
query_delete_meta(conn, meta)?;
} else {
query_upsert_meta(conn, meta)?;
}
Ok(())
}
pub fn insert_tag(conn: &Connection, tag: Tag) -> Result<()> {
debug!("DB: Inserting tag: {:?}", tag);
conn.execute(
"INSERT INTO tags (id, name) VALUES (?1, ?2)",
(tag.id, tag.name))?;
Ok(())
}
pub fn delete_item_tags(conn: &Connection, item: Item) -> Result<()> {
debug!("DB: Deleting all item tags: {:?}", item);
conn.execute(
"DELETE FROM tags WHERE id=?1",
[item.id])?;
Ok(())
}
pub fn set_item_tags(conn: &Connection, item: Item, tags: &Vec<String>) -> Result<()> {
debug!("DB: Setting tags for item: {:?} ?{:?}", item, tags);
delete_item_tags(conn, item.clone())?;
let item_id = item.id.unwrap();
for tag_name in tags {
insert_tag(conn,
Tag {
id: item_id,
name: tag_name.to_string()
})?;
}
Ok(())
}
pub fn query_all_items(conn: &Connection) -> Result<Vec<Item>> {
debug!("DB: Querying all items");
let mut statement = conn
.prepare("SELECT id, ts, size, compression FROM items ORDER BY id ASC")
.context("Problem preparing SQL statement")?;
let mut rows = statement.query([])?;
let mut items = Vec::new();
while let Some(row) = rows.next()? {
let item = Item {
id: row.get(0)?,
ts: row.get(1)?,
size: row.get(2)?,
compression: row.get(3)?
};
items.push(item);
}
Ok(items)
}
pub fn query_tagged_items<'a>(conn: &'a Connection, tags: &'a Vec<String>) -> Result<Vec<Item>> {
debug!("DB: Querying tagged items: {:?}", tags);
let mut statement = conn
.prepare_cached("
SELECT items.id,
items.ts,
items.size,
items.compression,
count(tags_match.id) as tags_score
FROM items,
(SELECT tags.id FROM tags WHERE tags.name IN rarray(?1)) as tags_match
WHERE items.id = tags_match.id
GROUP BY items.id
HAVING tags_score = ?2
ORDER BY items.id ASC")
.context("Problem preparing SQL statement")?;
let tags_values: Vec<rusqlite::types::Value> = tags
.iter()
.map(|s| {rusqlite::types::Value::from(s.clone())})
.collect();
let tags_ptr = Rc::new(tags_values);
let mut rows = statement.query((&tags_ptr, &tags.len()))?;
let mut items = Vec::new();
while let Some(row) = rows.next()? {
let item = Item {
id: row.get(0)?,
ts: row.get(1)?,
size: row.get(2)?,
compression: row.get(3)?
};
items.push(item);
}
Ok(items)
}
pub fn get_items(conn: &Connection) -> Result<Vec<Item>> {
debug!("DB: Getting all items");
query_all_items(conn)
}
pub fn get_items_matching(conn: &Connection, tags: &Vec<String>, meta: &HashMap<String,String>) -> Result<Vec<Item>> {
debug!("DB: Getting items matching: tags={:?} meta={:?}", tags, meta);
let items = match tags.is_empty() {
true => query_all_items(conn)?,
false => query_tagged_items(conn, tags)?
};
if meta.is_empty() {
debug!("DB: Not filtering on meta");
Ok(items)
} else {
debug!("DB: Filtering on meta");
let mut filtered_items: Vec<Item> = Vec::new();
for item in items.iter() {
let mut item_ok = true;
let mut item_meta: HashMap<String, String> = HashMap::new();
for meta in get_item_meta(conn, item)? {
item_meta.insert(meta.name, meta.value);
}
debug!("DB: Matching: {:?}: {:?}", item, item_meta);
for (k, v) in meta.iter() {
match item_meta.get(k) {
Some(value) => item_ok = v.eq(value),
None => item_ok = false
}
if item_ok {
break;
}
}
if item_ok {
filtered_items.push(item.clone());
}
}
Err(e) => Err(format!("Error connecting to database: {}", e))
Ok(filtered_items)
}
}
pub fn get_item_matching(conn: &Connection, tags: &Vec<String>, _meta: &HashMap<String, String>) -> Result<Option<Item>> {
debug!("DB: Get item matching tags: {:?}", tags);
let mut statement = conn
.prepare_cached("
SELECT items.id,
items.ts,
items.size,
items.compression,
count(sel.id) as score
FROM items,
(SELECT tags.id FROM tags WHERE tags.name IN rarray(?1)) as sel
WHERE items.id = sel.id
GROUP BY items.id
HAVING score = ?2
ORDER BY items.id DESC
LIMIT 1")
.context("Problem preparing SQL statement")?;
let tags_values: Vec<rusqlite::types::Value> = tags
.iter()
.map(|s| {rusqlite::types::Value::from(s.clone()) })
.collect();
let tags_ptr = Rc::new(tags_values);
let mut rows = statement.query((&tags_ptr, &tags.len()))?;
match rows.next()? {
Some(row) => Ok(Some(Item {
id: row.get(0)?,
ts: row.get(1)?,
size: row.get(2)?,
compression: row.get(3)?
})),
None => Ok(None)
}
}
pub fn get_item(conn: &Connection, item_id: i64) -> Result<Option<Item>> {
debug!("DB: Getting item {:?}", item_id);
let mut statement = conn
.prepare_cached("
SELECT id, ts, size, compression
FROM items
WHERE items.id = ?1")
.context("Problem preparing SQL statement")?;
let mut rows = statement.query([item_id])?;
match rows.next()? {
Some(row) => Ok(Some(Item {
id: row.get(0)?,
ts: row.get(1)?,
size: row.get(2)?,
compression: row.get(3)?
})),
None => Ok(None)
}
}
pub fn get_item_tags(conn: &Connection, item: &Item) -> Result<Vec<Tag>> {
debug!("DB: Getting tags for item: {:?}", item);
let mut statement = conn
.prepare_cached("SELECT id, name FROM tags WHERE id=?1 ORDER BY name ASC")
.context("Problem preparing SQL statement")?;
let mut rows = statement.query([item.id])?;
let mut tags = Vec::new();
while let Some(row) = rows.next()? {
tags.push(Tag {
id: row.get(0)?,
name: row.get(1)?,
});
}
Ok(tags)
}
pub fn get_item_meta(conn: &Connection, item: &Item) -> Result<Vec<Meta>> {
debug!("DB: Getting item meta: {:?}", item);
let mut statement = conn
.prepare_cached("SELECT id, name, value FROM metas WHERE id=?1 ORDER BY name ASC")
.context("Problem preparing SQL statement")?;
let mut rows = statement.query([item.id])?;
let mut metas = Vec::new();
while let Some(row) = rows.next()? {
metas.push(Meta {
id: row.get(0)?,
name: row.get(1)?,
value: row.get(2)?
});
}
Ok(metas)
}

View File

@@ -1,12 +1,69 @@
use std::io;
use std::io::{Read, Write};
use std::fs;
use std::str::FromStr;
use std::path::PathBuf;
use std::collections::{HashMap, HashSet};
use anyhow::{Context, Result, Error, anyhow};
use rusqlite::Connection;
use gethostname::gethostname;
use strum::IntoEnumIterator;
use clap::error::ErrorKind;
use clap::*;
use log::*;
extern crate directories;
use directories::ProjectDirs;
extern crate prettytable;
use prettytable::{Table, Row, Cell, Attr};
use prettytable::format;
use prettytable::format::{TableFormat, Alignment};
use prettytable::format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR;
use prettytable::row;
use prettytable::color;
use chrono::prelude::*;
#[macro_use]
extern crate lazy_static;
use crate::compression::CompressionType;
pub mod compression;
pub mod db;
use humansize::{format_size, BINARY};
use is_terminal::IsTerminal;
extern crate term;
const BUFSIZ: usize = 8192;
lazy_static! {
static ref FORMAT_BOX_CHARS_NO_BORDER_LINE_SEPARATOR: TableFormat = format::FormatBuilder::new()
.column_separator('│')
.borders('│')
.separators(&[format::LinePosition::Top],
format::LineSeparator::new('─',
'┬',
'┌',
'┐'))
.separators(&[format::LinePosition::Title],
format::LineSeparator::new('─',
'┼',
'├',
'┤'))
.separators(&[format::LinePosition::Bottom],
format::LineSeparator::new('─',
'┴',
'└',
'┘'))
.padding(1, 1)
.build();
}
#[derive(Parser, Debug)]
#[command(author, version, about, long_about = None)]
struct Args {
@@ -17,60 +74,66 @@ struct Args {
#[command(flatten)]
options: OptionsArgs,
#[arg()]
#[arg(help("A list of either item IDs or tags"))]
ids_or_tags: Vec<NumberOrString>
}
#[derive(Parser, Debug)]
struct ModeArgs {
#[arg(group("mode"), help_heading("Mode"), short, long, conflicts_with_all(["get", "list", "update", "delete", "status"]))]
#[arg(group("mode"), help_heading("Mode Options"), short, long, conflicts_with_all(["get", "list", "update", "delete", "status"]))]
#[arg(help("Save an item using any tags or metadata provided"))]
save: bool,
#[arg(group("mode"), help_heading("Mode"), short, long, conflicts_with_all(["save", "list", "update", "delete", "status"]))]
#[arg(group("mode"), help_heading("Mode Options"), short, long, conflicts_with_all(["save", "list", "update", "delete", "status"]))]
#[arg(help("Get an item either by it's ID or by a combination of matching tags and metatdata"))]
get: bool,
#[arg(group("mode"), help_heading("Mode"), short, long, conflicts_with_all(["save", "get", "update", "delete", "status"]))]
#[arg(group("mode"), help_heading("Mode Options"), short, long, conflicts_with_all(["save", "get", "update", "delete", "status"]))]
#[arg(help("List items, filtering on tags or metadata if given"))]
list: bool,
#[arg(group("mode"), help_heading("Mode"), short, long, conflicts_with_all(["save", "get", "list", "delete", "status"]), requires("ids_or_tags"))]
#[arg(group("mode"), help_heading("Mode Options"), short, long, conflicts_with_all(["save", "get", "list", "delete", "status"]), requires("ids_or_tags"))]
#[arg(help("Update a specified item ID's tags and/or metadata"))]
update: bool,
#[arg(group("mode"), help_heading("Mode"), short, long, conflicts_with_all(["save", "get", "list", "update", "status"]), requires("ids_or_tags"))]
#[arg(group("mode"), help_heading("Mode Options"), short, long, conflicts_with_all(["save", "get", "list", "update", "status"]), requires("ids_or_tags"))]
#[arg(help("Delete items either by ID or by matching tags"))]
delete: bool,
#[arg(group("mode"), help_heading("Mode"), short('S'), long, conflicts_with_all(["save", "get", "list", "update", "delete"]))]
#[arg(group("mode"), help_heading("Mode Options"), short('S'), long, conflicts_with_all(["save", "get", "list", "update", "delete"]))]
#[arg(help("Show status of directories and supported compression algorithms"))]
status: bool
}
#[derive(Parser, Debug)]
struct ItemArgs {
#[arg(help_heading("Item"), short, long, conflicts_with("get"), conflicts_with("list"))]
comment: Option<String>,
#[arg(help_heading("Item Options"), short, long, conflicts_with_all(["get", "delete", "status"]))]
#[arg(help("Set metadata for the item using the format KEY=[VALUE], the metadata will be removed if VALUE is not provided"))]
meta: Option<Vec<KeyValue>>,
#[arg(help_heading("Item"), short('C'), long, conflicts_with("get"), conflicts_with("list"), env("KEEP_COMPRESS"))]
compress: Option<String>,
#[arg(help_heading("Item Options"), short('C'), long, conflicts_with("get"), conflicts_with("list"), env("KEEP_COMPRESSION"), )]
#[arg(help("Compression algorithm to use when saving items"))]
compression: Option<String>,
}
#[derive(Parser, Debug)]
struct OptionsArgs {
#[arg(help_heading("Options"), long, env("KEEP_DIR"))]
#[arg(long, env("KEEP_DIR"))]
#[arg(help("Specify the directory to use for storage"))]
dir: Option<PathBuf>,
#[arg(help_heading("Options"), short, long)]
force: bool,
#[arg(help_heading("Options"), short, long, action = clap::ArgAction::Count, conflicts_with("quiet"))]
#[arg(short, long, action = clap::ArgAction::Count, conflicts_with("quiet"))]
#[arg(help("Increase message verbosity, can be given more than once"))]
verbose: u8,
#[arg(help_heading("Options"), short, long)]
#[arg(short, long)]
#[arg(help("Do show any messages"))]
quiet: bool,
}
#[derive(Debug,Clone)]
enum NumberOrString {
Number(u32),
Str(String),
}
#[derive(Debug,PartialEq)]
enum KeepModes {
@@ -83,35 +146,60 @@ enum KeepModes {
Status
}
#[derive(Debug,Clone)]
struct KeyValue {
key: String,
value: String
}
impl FromStr for KeyValue {
type Err = Error;
fn from_str(s: &str) -> Result<Self, Error> {
match s.split_once('=') {
Some(kv) => Ok(KeyValue {
key: kv.0.to_string(),
value: kv.1.to_string()
}),
None => Err(anyhow!("Unable to parse key=value pair"))
}
}
}
#[derive(Debug,Clone)]
enum NumberOrString {
Number(i64),
Str(String),
}
impl FromStr for NumberOrString {
type Err = &'static str; // The actual type doesn't matter since we never error, but it must implement `Display`
fn from_str(s: &str) -> Result<Self, Self::Err>
{
Ok (s.parse::<u32>()
type Err = Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
Ok (s.parse::<i64>()
.map(NumberOrString::Number)
.unwrap_or_else(|_| NumberOrString::Str (s.to_string())))
}
}
fn main() -> Result<(), Error> {
let proj_dirs = ProjectDirs::from("gt0.ca", "Andrew Phillips", "Keep");
fn main() {
let mut cmd = Args::command();
let args = Args::parse();
let mut args = Args::parse();
stderrlog::new()
.module(module_path!())
.quiet(args.options.quiet)
.verbosity(usize::from(args.options.verbose + 2))
.timestamp(stderrlog::Timestamp::Second)
//.timestamp(stderrlog::Timestamp::Second)
.init()
.unwrap();
debug!("Start");
debug!("MAIN: Start");
let ids = &mut Vec::new();
let tags = &mut Vec::new();
for v in args.ids_or_tags.iter() {
match v.clone() {
NumberOrString::Number(num) => ids.push(num),
@@ -119,7 +207,11 @@ fn main() {
}
}
tags.sort();
tags.dedup();
let mut mode: KeepModes = KeepModes::Unknown;
if args.mode.save {
mode = KeepModes::Save;
} else if args.mode.get {
@@ -133,7 +225,7 @@ fn main() {
} else if args.mode.status {
mode = KeepModes::Status;
}
if mode == KeepModes::Unknown {
if ! ids.is_empty() {
mode = KeepModes::Get;
@@ -142,24 +234,53 @@ fn main() {
}
}
debug!("args: {:?}", args);
debug!("ids: {:?}", ids);
debug!("tags: {:?}", tags);
debug!("mode: {:?}", mode);
debug!("MAIN: args: {:?}", args);
debug!("MAIN: ids: {:?}", ids);
debug!("MAIN: tags: {:?}", tags);
debug!("MAIN: mode: {:?}", mode);
if args.options.dir.is_none() {
match proj_dirs {
Some(proj_dirs) => args.options.dir = Some(proj_dirs.data_dir().to_path_buf()),
None => return Err(anyhow!("Unable to determine data directory"))
}
}
unsafe {
libc::umask(0o077);
}
let data_path = args.options.dir.clone().unwrap();
let mut db_path = data_path.clone();
db_path.push("keep-1.db");
debug!("MAIN: Data directory: {:?}", data_path);
debug!("MAIN: DB file: {:?}", db_path);
fs::create_dir_all(data_path.clone())
.context("Problem creating data directory")?;
debug!("MAIN: Data directory created or already exists");
let mut conn = db::open(db_path.clone())
.context("Problem opening database")?;
debug!("MAIN: DB opened successfully");
match mode {
KeepModes::Save => mode_save(&mut cmd, args, ids, tags),
KeepModes::Get => mode_get(&mut cmd, args, ids, tags),
KeepModes::List => mode_list(&mut cmd, args, ids, tags),
KeepModes::Update => mode_update(&mut cmd, args, ids, tags),
KeepModes::Delete => mode_delete(&mut cmd, args, ids, tags),
KeepModes::Status => mode_status(&mut cmd, args),
KeepModes::Save => mode_save(&mut cmd, args, ids, tags, conn, data_path)?,
KeepModes::Get => mode_get(&mut cmd, args, ids, tags, &mut conn, data_path)?,
KeepModes::List => mode_list(&mut cmd, args, ids, tags, &mut conn, data_path)?,
KeepModes::Update => mode_update(&mut cmd, args, ids, tags, &mut conn)?,
KeepModes::Delete => mode_delete(&mut cmd, args, ids, tags, &mut conn, data_path)?,
KeepModes::Status => mode_status(&mut cmd, args, data_path, db_path)?,
_ => todo!()
}
Ok(())
}
fn mode_save(cmd: &mut Command, args: Args, ids: &mut Vec<u32>, tags: &mut Vec<String>) {
fn mode_save(cmd: &mut Command, args: Args, ids: &mut Vec<i64>, tags: &mut Vec<String>, conn: Connection, data_path: PathBuf) -> Result<()> {
if ! ids.is_empty() {
cmd.error(ErrorKind::InvalidValue, "ID given, you cannot supply IDs when using --save").exit();
}
@@ -168,37 +289,294 @@ fn mode_save(cmd: &mut Command, args: Args, ids: &mut Vec<u32>, tags: &mut Vec<S
tags.push("none".to_string());
}
let compression_type = compression::get_compression(args.item.compress);
let compression_name = args.item.compression.unwrap_or(compression::default_type().to_string());
let compression_type_opt = CompressionType::from_str(&compression_name);
if compression_type_opt.is_err() {
cmd.error(ErrorKind::InvalidValue, format!("Unknown compression type: {}", compression_name)).exit();
}
let compression_type = compression_type_opt.unwrap();
debug!("MAIN: Compression type: {}", compression_type);
let mut item = db::Item {
id: None,
ts: Utc::now(),
size: None,
compression: compression_type.to_string()
};
let id = db::insert_item(&conn, item.clone())?;
item.id = Some(id);
debug!("MAIN: Added item {:?}", item.clone());
if ! args.options.quiet {
if std::io::stderr().is_terminal() {
let mut t = term::stderr().unwrap();
t.reset().unwrap_or(());
t.attr(term::Attr::Bold).unwrap_or(());
write!(t, "KEEP:").unwrap_or(());
t.reset().unwrap_or(());
write!(t, " New item ").unwrap_or(());
t.attr(term::Attr::Bold).unwrap_or(());
write!(t, "{id}")?;
t.reset().unwrap_or(());
write!(t, " tags: ")?;
t.attr(term::Attr::Bold).unwrap_or(());
write!(t, "{}", tags.join(" "))?;
t.reset().unwrap_or(());
writeln!(t, "")?;
std::io::stderr().flush()?;
} else {
let mut t = std::io::stderr();
writeln!(t, "KEEP: New item: {} tags: {:?}", id, tags)?;
}
}
db::set_item_tags(&conn, item.clone(), tags)?;
let mut item_meta: HashMap<String, String> = HashMap::new();
if let Ok(hostname) = gethostname().into_string() {
item_meta.insert("hostname".to_string(), hostname);
}
if args.item.meta.is_some() {
for item in args.item.meta.unwrap().iter() {
let item = item.clone();
item_meta.insert(item.key, item.value);
}
}
for kv in item_meta.iter() {
let meta = db::Meta {
id: item.id.unwrap(),
name: kv.0.to_string(),
value: kv.1.to_string()
};
db::store_meta(&conn, meta)?;
}
let mut item_path = data_path.clone();
item_path.push(id.to_string());
let mut stdin = io::stdin().lock();
let mut stdout = io::stdout().lock();
let mut buffer = [0; BUFSIZ];
let compression_engine = compression::get_engine(compression_type.clone()).expect("Unable to get compression engine");
let mut item_out: Box<dyn Write> = compression_engine.create(item_path.clone())
.context(anyhow!("Unable to write file {:?} using compression {:?}", item_path, compression_type))?;
debug!("MAIN: Starting IO loop");
loop {
let n = stdin.read(&mut buffer[..BUFSIZ])?;
if n == 0 {
debug!("MAIN: EOF on STDIN");
break;
}
stdout.write_all(&buffer[..n])?;
item_out.write_all(&buffer[..n])?;
item.size = match item.size {
None => Some(n as i64),
Some(prev_n) => Some(prev_n + n as i64)
};
}
debug!("MAIN: Ending IO loop");
stdout.flush()?;
item_out.flush()?;
db::update_item(&conn, item.clone())?;
Ok(())
}
fn mode_get(cmd: &mut Command, args: Args, ids: &mut Vec<u32>, tags: &mut Vec<String>) {
fn mode_get(cmd: &mut Command, args: Args, ids: &mut Vec<i64>, tags: &mut Vec<String>, conn: &mut Connection, data_path: PathBuf) -> Result<()> {
if ids.is_empty() && tags.is_empty() {
cmd.error(ErrorKind::InvalidValue, "No ID or tags given, ou must supply one ID or atleast one tag when using --get").exit();
cmd.error(ErrorKind::InvalidValue, "No ID or tags given, you must supply one ID or atleast one tag when using --get").exit();
} else if ! ids.is_empty() && ! tags.is_empty() {
cmd.error(ErrorKind::InvalidValue, "Both ID and tags given, you must supply one ID or atleast one tag when using --get").exit();
} else if ids.len() > 1 {
cmd.error(ErrorKind::InvalidValue, "More than one ID given, you must supply one ID or atleast one tag when using --get").exit();
}
let mut meta: HashMap<String, String> = HashMap::new();
if args.item.meta.is_some() {
for item in args.item.meta.unwrap().iter() {
let item = item.clone();
meta.insert(item.key, item.value);
}
}
let item_maybe = match tags.is_empty() && meta.is_empty() {
true => match ids.iter().next() {
Some(item_id) => db::get_item(conn, *item_id)?,
None => None
},
false => db::get_item_matching(conn, tags, &meta)?
};
if let Some(item) = item_maybe {
debug!("MAIN: Found item {:?}", item);
let mut item_path = data_path.clone();
item_path.push(item.id.unwrap().to_string());
let compression_type = CompressionType::from_str(&item.compression)?;
debug!("MAIN: Item has compression type {:?}", compression_type.clone());
let compression_engine = compression::get_engine(compression_type).expect("Unable to get compression engine");
compression_engine.cat(item_path.clone())
} else {
Err(anyhow!("Unable to find matching item in database"))
}
}
fn mode_list(cmd: &mut Command, args: Args, ids: &mut Vec<u32>, tags: &mut Vec<String>) {
fn mode_list(cmd: &mut Command, args: Args, ids: &mut Vec<i64>, tags: &Vec<String>, conn: &mut Connection, data_path: PathBuf) -> Result<()> {
if ! ids.is_empty() {
cmd.error(ErrorKind::InvalidValue, "ID given, you can only supply tags when using --list").exit();
}
let mut meta: HashMap<String, String> = HashMap::new();
if args.item.meta.is_some() {
for item in args.item.meta.unwrap().iter() {
let item = item.clone();
meta.insert(item.key, item.value);
}
}
let items = match tags.is_empty() && meta.is_empty() {
true => db::get_items(conn)?,
false => db::get_items_matching(conn, tags, &meta)?
};
debug!("MAIN: Items: {:?}", items);
let mut tags_by_item: HashMap<i64, Vec<String>> = HashMap::new();
let mut meta_by_item: HashMap<i64, HashMap<String, String>> = HashMap::new();
let mut meta_columns = HashSet::new();
for item in items.iter() {
let item_id = item.id.unwrap();
let item_tags: Vec<String> = db::get_item_tags(conn, item)?
.into_iter()
.map(|x| {x.name})
.collect();
tags_by_item.insert(item_id, item_tags);
let mut item_meta: HashMap<String, String> = HashMap::new();
for meta in db::get_item_meta(conn, item)? {
meta_columns.insert(meta.name.clone());
item_meta.insert(meta.name.clone(), meta.value);
}
meta_by_item.insert(item_id, item_meta);
};
let mut meta_columns_sorted = Vec::from_iter(meta_columns);
meta_columns_sorted.sort();
let mut table = Table::new();
if std::io::stdout().is_terminal() {
table.set_format(*FORMAT_BOX_CHARS_NO_BORDER_LINE_SEPARATOR);
} else {
table.set_format(*FORMAT_NO_BORDER_LINE_SEPARATOR);
}
let mut title_row = row!(
b->"ID",
b->"Time",
b->"Stream Size",
b->"Comp",
b->"File Size",
b->"Tags",
);
for name in &meta_columns_sorted {
title_row.add_cell(Cell::new(name).with_style(Attr::Bold));
}
table.set_titles(title_row);
for item in items {
let item_id = item.id.unwrap();
let mut item_path = data_path.clone();
item_path.push(item.id.unwrap().to_string());
let id_cell = Cell::new_align(&item.id.unwrap_or(0).to_string(), Alignment::RIGHT);
let ts_cell = Cell::new(&item.ts.with_timezone(&Local).format("%F %T").to_string());
let size_cell = match item.size {
Some(size) => Cell::new_align(format_size(size as u64, BINARY).as_str(), Alignment::RIGHT),
None => Cell::new_align("Missing", Alignment::RIGHT).with_style(Attr::ForegroundColor(color::RED)).with_style(Attr::Bold)
};
let compression_cell = Cell::new(&item.compression);
let file_size_cell = match item_path.metadata() {
Ok(metadata) => Cell::new_align(format_size(metadata.len(), BINARY).as_str(), Alignment::RIGHT),
Err(_) => Cell::new_align("Missing", Alignment::RIGHT).with_style(Attr::ForegroundColor(color::RED)).with_style(Attr::Bold)
};
let item_tags = tags_by_item.get(&item_id).unwrap();
let tags_cell = Cell::new(&item_tags.join(" "));
let mut table_row = Row::new(vec![id_cell,ts_cell,size_cell, compression_cell, file_size_cell, tags_cell]);
let item_meta = meta_by_item.get(&item_id).unwrap();
for name in &meta_columns_sorted {
match item_meta.get(name) {
Some(value) => table_row.add_cell(Cell::new(value)),
None => table_row.add_cell(Cell::new(""))
};
}
table.add_row(table_row);
}
table.printstd();
Ok(())
}
fn mode_update(cmd: &mut Command, args: Args, ids: &mut Vec<u32>, tags: &mut Vec<String>) {
fn mode_update(cmd: &mut Command, args: Args, ids: &mut Vec<i64>, tags: &mut Vec<String>, conn: &mut Connection) -> Result<()> {
if ids.is_empty() {
cmd.error(ErrorKind::InvalidValue, "No ID given, you must supply one ID when using --update").exit();
}
let item_id = ids.iter().next().expect("Unable to determine item id");
let item_maybe = db::get_item(conn, *item_id)?;
let item = item_maybe.expect("Unable to find item in database");
debug!("MAIN: Found item {:?}", item);
if ! tags.is_empty() {
debug!("MAIN: Updating item tags");
db::set_item_tags(conn, item.clone(), tags)?;
}
if args.item.meta.is_some() {
debug!("MAIN: Updating item meta");
for kv in args.item.meta.unwrap().iter() {
let meta = db::Meta {
id: item.id.unwrap(),
name: kv.key.to_string(),
value: kv.value.to_string()
};
db::store_meta(conn, meta)?;
}
}
Ok(())
}
fn mode_delete(cmd: &mut Command, args: Args, ids: &mut Vec<u32>, tags: &mut Vec<String>) {
fn mode_delete(cmd: &mut Command, _args: Args, ids: &mut Vec<i64>, tags: &mut Vec<String>, conn: &mut Connection, data_path: PathBuf) -> Result<()> {
if ids.is_empty() {
cmd.error(ErrorKind::InvalidValue, "No ID given, you must supply one ID when using --delete").exit();
} else if ! tags.is_empty() {
@@ -206,19 +584,105 @@ fn mode_delete(cmd: &mut Command, args: Args, ids: &mut Vec<u32>, tags: &mut Vec
} else if ids.len() > 1 {
cmd.error(ErrorKind::InvalidValue, "More than one ID given, you must supply one ID when using --delete").exit();
}
let item_id = ids.iter().next().expect("Unable to determine item id");
let item_maybe = db::get_item(conn, *item_id)?;
let item = item_maybe.expect("Unable to find item in database");
debug!("MAIN: Found item {:?}", item);
db::delete_item(conn, item)?;
let mut item_path = data_path.clone();
item_path.push(item_id.to_string());
fs::remove_file(&item_path).context(anyhow!("Unable to remove item file {:?}", item_path))?;
Ok(())
}
fn mode_status_show_compression() {
let compression_types = compression::supported_compression_types();
println!("compression_types:");
for compression_type in compression_types.into_iter() {
println!(" {}", compression_type);
fn mode_status(_cmd: &mut Command, args: Args, data_path: PathBuf, db_path: PathBuf) -> Result<()> {
let mut path_table = Table::new();
if std::io::stdout().is_terminal() {
path_table.set_format(*FORMAT_BOX_CHARS_NO_BORDER_LINE_SEPARATOR);
} else {
path_table.set_format(*FORMAT_NO_BORDER_LINE_SEPARATOR);
}
}
path_table.set_titles(Row::new(vec![
Cell::new("Type").with_style(Attr::Bold),
Cell::new("Path").with_style(Attr::Bold),
]));
path_table.add_row(Row::new(vec![
Cell::new("Data"),
Cell::new(&data_path.into_os_string().into_string().expect("Unable to convert data path to string"))
]));
path_table.add_row(Row::new(vec![
Cell::new("Database"),
Cell::new(&db_path.into_os_string().into_string().expect("Unable to convert DB path to string"))
]));
fn mode_status(cmd: &mut Command, args: Args) {
mode_status_show_compression();
let mut compression_table = Table::new();
if std::io::stdout().is_terminal() {
compression_table.set_format(*FORMAT_BOX_CHARS_NO_BORDER_LINE_SEPARATOR);
} else {
compression_table.set_format(*FORMAT_NO_BORDER_LINE_SEPARATOR);
}
compression_table.set_titles(row!(
b->"Type",
b->"Found",
b->"Default",
b->"Binary",
b->"Compress",
b->"Decompress"));
let default_type = match args.item.compression {
Some(compression_name) => CompressionType::from_str(&compression_name)
.context(anyhow!("Invalid compression type {}", compression_name))?,
None => compression::default_type()
};
for compression_type in CompressionType::iter() {
let compression_program = match compression_type {
CompressionType::None => compression::CompressionProgram {
program: "".to_string(),
compress: Vec::new(),
decompress: Vec::new(),
supported: true
},
_ => compression::get_program(compression_type.clone())?
};
let is_default = compression_type == default_type;
compression_table.add_row(Row::new(vec![
Cell::new(&compression_type.to_string()),
match compression_program.supported {
true => Cell::new("Yes").with_style(Attr::ForegroundColor(color::GREEN)),
false => Cell::new("No").with_style(Attr::ForegroundColor(color::RED))
},
match is_default {
true => Cell::new("Yes").with_style(Attr::ForegroundColor(color::GREEN)),
false => Cell::new("No")
},
Cell::new(&compression_program.program),
Cell::new(&compression_program.compress.join(" ")),
Cell::new(&compression_program.decompress.join(" ")),
]));
}
println!("PATHS:");
path_table.printstd();
println!();
println!("COMPRESSION:");
compression_table.printstd();
Ok(())
}