diff --git a/db_info/Cargo.lock b/db_info/Cargo.lock new file mode 100644 index 0000000..a6545ac --- /dev/null +++ b/db_info/Cargo.lock @@ -0,0 +1,25 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "db_info" +version = "0.1.0" +dependencies = [ + "redb", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "redb" +version = "3.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ba239c1c1693315d3cc0e601db3b3965543afbf48c41730fdca2f069f510f4a" +dependencies = [ + "libc", +] diff --git a/db_info/Cargo.toml b/db_info/Cargo.toml new file mode 100644 index 0000000..573cfb2 --- /dev/null +++ b/db_info/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "db_info" +version = "0.1.0" +edition = "2024" + +[dependencies] +redb = "3.1" diff --git a/db_info/src/database.rs b/db_info/src/database.rs new file mode 100644 index 0000000..950493d --- /dev/null +++ b/db_info/src/database.rs @@ -0,0 +1,348 @@ +use redb::{Database, ReadableDatabase, ReadableTable, ReadableTableMetadata, TableDefinition}; +// use std::collections::HashMap; +use std::{path::Path, u64, u128}; + +// const RANGE_STATUS: MultimapTableDefinition = +// MultimapTableDefinition::new("range_status"); + +const JOBS_TABLE: TableDefinition = TableDefinition::new("jobs"); +const JOBS_FREE_IDS: TableDefinition = TableDefinition::new("jobs_free_ids"); + +const FOUND_KEYS_TABLE: TableDefinition<(u128, u128), u64> = TableDefinition::new("found_keys"); + +// get time is seconds +fn get_timestump() -> u64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs() +} + +#[derive(Copy, Clone, Debug)] +pub struct JobRecord { + pub id: u64, + pub range_id: u32, + pub tweak_key: u128, + pub start_key: u128, + pub len: u64, + pub start_time: u64, +} + +impl JobRecord { + pub fn from_data(k: u64, v: &(u32, u128, u128, u64, u64)) -> Self { + Self { + id: k, + range_id: v.0, + tweak_key: v.1, + start_key: v.2, + len: v.3, + start_time: v.4, + } + } + + pub fn from_iter_item( + item: Result< + ( + redb::AccessGuard<'_, u64>, + redb::AccessGuard<'_, (u32, u128, u128, u64, u64)>, + ), + redb::StorageError, + >, + ) -> Result { + let ag = item?; + Ok(Self::from_data(ag.0.value(), &ag.1.value())) + } + + pub fn get_value(&self) -> (u32, u128, u128, u64, u64) { + return ( + self.range_id, + self.tweak_key, + self.start_key, + self.len, + self.start_time, + ); + } +} + +pub fn list_jobs( + jobs_table: &mut redb::Table<'_, u64, (u32, u128, u128, u64, u64)>, +) -> Result, redb::Error> { + let mut res: Vec = Vec::new(); + + for job_res in jobs_table.iter()? { + let job_rec = JobRecord::from_iter_item(job_res)?; + res.push(job_rec); + } + + Ok(res) +} + +const RANGES: TableDefinition = TableDefinition::new("ranges"); +const RANGES_TO_USE: TableDefinition = TableDefinition::new("ranges_to_use"); + +// to not owerflow u128 we calculate it in range 0..max and then substruct +// const SUB_DIV: u128 = 0x1000100010001000100010001; +// TWEAKS_PER_RANGE: u128 = 5192296858534827628530496329220096 +const TWEAKS_PER_RANGE: u128 = (u128::MAX / (u16::MAX as u128 + 1)) + 1; +const TWEAKS_PER_RANGE_F64: f64 = TWEAKS_PER_RANGE as f64; +//assert!(TWEAKS_PER_RANGE == 5192296858534827628530496329220096u128); + +const TWEAKS_PER_RANGE_ID_VICE: u128 = TWEAKS_PER_RANGE - 1; +//const HALF_TWEAKS_PER_RANGE: u128 = TWEAKS_PER_RANGE / 2; + +#[derive(Copy, Clone, Debug)] +struct RangeRecord { + id: u32, + + tweak_current: u128, + tweak_end: u128, + key_progress: u128, + key_committed_progress: u128, +} + +impl RangeRecord { + fn from_value(k: u32, v: &(u128, u128, u128, u128)) -> Self { + Self { + id: k, + tweak_current: v.0, + tweak_end: v.1, + key_progress: v.2, + key_committed_progress: v.3, + } + } + + fn new(id: u32) -> Self { + let tweak_start: u128 = id as u128 * TWEAKS_PER_RANGE; + Self { + id: id, + tweak_current: tweak_start, + tweak_end: tweak_start.strict_add(TWEAKS_PER_RANGE_ID_VICE), + key_progress: 0, + key_committed_progress: 0, + } + } + + // fn tweaks_left(&self) -> u128 { + // let t_left = (self.tweak_end - self.tweak_current) + 1; + // if t_left == 1 && self.key_committed_progress == u128::MAX { + // 0u128 + // } + // else { + // t_left + // } + // } + + fn completed_ratio(&self) -> f64 { + let t_left = self.tweak_end - self.tweak_current; + if t_left == 0 && self.key_committed_progress == u128::MAX { + 1.0f64 + } else { + 1.0f64 - ((t_left as f64 + 1.0) / TWEAKS_PER_RANGE_F64) + } + } + + fn tweak_processed(&self) -> u128 { + let t_left = self.tweak_end - self.tweak_current; + return TWEAKS_PER_RANGE - t_left; + } + + fn value(&self) -> (u128, u128, u128, u128) { + ( + self.tweak_current, + self.tweak_end, + self.key_progress, + self.key_committed_progress, + ) + } +} + +#[derive(Debug)] +struct RangesTable<'a> { + ranges: redb::Table<'a, u32, (u128, u128, u128, u128)>, + ranges_to_use: redb::Table<'a, u32, ()>, +} + +impl<'a> RangesTable<'a> { + // fn new( + // ranges: redb::Table<'a, u16, (u128, u128, u128, u128)>, + // ranges_to_use: redb::Table<'a, u16, ()>, + // ) -> Self { + // RangesTable { + // ranges: ranges, + // ranges_to_use: ranges_to_use, + // } + // } + + fn open_rw(trx: &'a redb::WriteTransaction) -> Result { + Ok(RangesTable { + ranges: trx.open_table(RANGES)?, + ranges_to_use: trx.open_table(RANGES_TO_USE)?, + }) + } + + fn get_range(&self, k: u32) -> Result, redb::Error> { + Ok(self + .ranges + .get(k)? + .and_then(|ag_v| Some(RangeRecord::from_value(k, &ag_v.value())))) + } + + fn get_useful_ranges(&self) -> Result, redb::Error> { + let mut u_ranges: Vec = Vec::with_capacity(self.ranges_to_use.len()? as usize); + for kres in self.ranges_to_use.iter()? { + u_ranges.push( + self.get_range(kres?.0.value())?.expect( + "Randge id from ranges_to_use should allways be present in ranges table", + ), + ); + } + + Ok(u_ranges) + } + + fn update_range(&mut self, range: &RangeRecord) -> Result<(), redb::Error> { + self.ranges.insert(range.id, range.value())?; + // if range key in progress reach max values - remove it from to_use table + if range.key_progress == u128::MAX { + self.ranges_to_use.remove(range.id)?; + } + Ok(()) + } +} + +#[derive(Copy, Clone, Debug)] +pub struct FoundKey { + pub tweak_key: u128, + pub enc_key: u128, + pub timestump: u64, +} + +pub fn db_get_found_keys(db: &Database) -> Result, redb::Error> { + let trx: redb::ReadTransaction = db.begin_read()?; + let mut res: Vec = Vec::new(); + + { + let found_keys_t = trx.open_table(FOUND_KEYS_TABLE)?; + + for found_key_res in found_keys_t.iter()? { + let ag = found_key_res?; + res.push(FoundKey { + tweak_key: ag.0.value().0, + enc_key: ag.0.value().1, + timestump: ag.1.value(), + }); + } + } + + return Ok(res); +} + +pub fn db_put_found_key(db: &Database, tw_k: u128, en_k: u128) -> Result { + let trx: redb::WriteTransaction = db.begin_write()?; + let new_keys_count = { + let mut found_keys_t = trx.open_table(FOUND_KEYS_TABLE)?; + found_keys_t.insert((tw_k, en_k), get_timestump())?; + found_keys_t.len()? + }; + trx.commit()?; + + return Ok(new_keys_count); +} + +pub fn db_get_progress(db: &Database) -> Result { + let trx: redb::ReadTransaction = db.begin_read()?; + let progress = { + let ranges_t = trx.open_table(RANGES)?; + let mut mean_prog: f64 = 0.0; + for range_res in ranges_t.iter()? { + let ag = range_res?; + let range = RangeRecord::from_value(ag.0.value(), &ag.1.value()); + mean_prog += range.completed_ratio(); + } + + mean_prog / 65536.0f64 + }; + trx.close()?; + return Ok(progress); +} + +pub fn db_get_progress_details(db: &Database) -> Result<(u128, u128, u128, u128, f64), redb::Error> { + let trx: redb::ReadTransaction = db.begin_read()?; + + //let mut mean_prog: f64 = 0.0; + let mut min_proc_keys: u128 = u128::MAX; + let mut max_proc_keys: u128 = 0; + let mut avrg_proc_keys: u128 = 0; + + let mut max_proc_tweks: u128 = 0; + + let mut avg_proc_tweks: f64 = 0.0; + + { + let ranges_t = trx.open_table(RANGES)?; + for range_res in ranges_t.iter()? { + let ag = range_res?; + let range = RangeRecord::from_value(ag.0.value(), &ag.1.value()); + + avg_proc_tweks += range.tweak_processed() as f64 / 65536.0f64; + if range.tweak_processed() > max_proc_tweks { + max_proc_tweks = range.tweak_processed(); + } + if range.key_committed_progress > max_proc_keys { + max_proc_keys = range.key_committed_progress + } + if range.key_committed_progress < min_proc_keys { + min_proc_keys = range.key_committed_progress; + } + avrg_proc_keys += range.key_committed_progress / 65536; + } + } + trx.close()?; + return Ok((min_proc_keys, max_proc_keys, avrg_proc_keys, max_proc_tweks, avg_proc_tweks)); +} + +fn db_init(db: &Database) -> Result<(), redb::Error> { + println!("Start DB init."); + + let trx: redb::WriteTransaction = db.begin_write()?; + { + let mut ranges = trx.open_table(RANGES)?; + let mut ranges_to_use = trx.open_table(RANGES_TO_USE)?; + for k in 0..=u16::MAX { + // create new ranges + let r = RangeRecord::new(k as u32); + ranges.insert(k as u32, r.value())?; + ranges_to_use.insert(k as u32, ())?; + } + + println!("ensure that all tables is created"); + // ensure that all tables is created + trx.open_table(FOUND_KEYS_TABLE)?.len()?; + trx.open_table(JOBS_TABLE)?.len()?; + trx.open_table(JOBS_FREE_IDS)?.len()?; + } + println!("Commit transaction"); + trx.commit()?; + println!("Done!"); + + Ok(()) +} + +pub fn db_open(db_file_path: &Path) -> Result { + let db = Database::create(db_file_path)?; + let trx = db.begin_write()?; + let ranges_len = { + println!("Opening RANGES to get len"); + let ranges_table = trx.open_table(RANGES)?; + println!("Try to get len"); + ranges_table.len()? + }; + println!("Commit transaction"); + trx.commit()?; + if ranges_len == 0 { + db_init(&db)?; + } + + Ok(db) +} diff --git a/db_info/src/main.rs b/db_info/src/main.rs new file mode 100644 index 0000000..ffbb4fc --- /dev/null +++ b/db_info/src/main.rs @@ -0,0 +1,17 @@ + +use std::{path::Path, u64, u128}; +mod database; + +fn main() { + println!("Hello, world!"); + let db_path = Path::new("backup_data.rdb"); + let db = database::db_open(db_path).unwrap(); + let found_keys = database::db_get_found_keys(&db).unwrap(); + let pr = database::db_get_progress(&db).unwrap(); + let d_pr = database::db_get_progress_details(&db).unwrap(); + println!("found_keys: {}", found_keys.len()); + println!("progress: {:.1074}", pr); + println!("progress det: {:?}", d_pr); + +} +