Add simple DB stat utility

2026-08-15 17:25:33 +02:00
parent a6e64f4d3d
commit ab67f69b21
4 changed files with 397 additions and 0 deletions
@@ -0,0 +1,348 @@
+use redb::{Database, ReadableDatabase, ReadableTable, ReadableTableMetadata, TableDefinition};
+// use std::collections::HashMap;
+use std::{path::Path, u64, u128};
+
+// const RANGE_STATUS: MultimapTableDefinition<bool, u128> =
+//     MultimapTableDefinition::new("range_status");
+
+const JOBS_TABLE: TableDefinition<u64, (u32, u128, u128, u64, u64)> = TableDefinition::new("jobs");
+const JOBS_FREE_IDS: TableDefinition<u64, ()> = TableDefinition::new("jobs_free_ids");
+
+const FOUND_KEYS_TABLE: TableDefinition<(u128, u128), u64> = TableDefinition::new("found_keys");
+
+// get time is seconds
+fn get_timestump() -> u64 {
+    use std::time::{SystemTime, UNIX_EPOCH};
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap()
+        .as_secs()
+}
+
+#[derive(Copy, Clone, Debug)]
+pub struct JobRecord {
+    pub id: u64,
+    pub range_id: u32,
+    pub tweak_key: u128,
+    pub start_key: u128,
+    pub len: u64,
+    pub start_time: u64,
+}
+
+impl JobRecord {
+    pub fn from_data(k: u64, v: &(u32, u128, u128, u64, u64)) -> Self {
+        Self {
+            id: k,
+            range_id: v.0,
+            tweak_key: v.1,
+            start_key: v.2,
+            len: v.3,
+            start_time: v.4,
+        }
+    }
+
+    pub fn from_iter_item(
+        item: Result<
+            (
+                redb::AccessGuard<'_, u64>,
+                redb::AccessGuard<'_, (u32, u128, u128, u64, u64)>,
+            ),
+            redb::StorageError,
+        >,
+    ) -> Result<Self, redb::StorageError> {
+        let ag = item?;
+        Ok(Self::from_data(ag.0.value(), &ag.1.value()))
+    }
+
+    pub fn get_value(&self) -> (u32, u128, u128, u64, u64) {
+        return (
+            self.range_id,
+            self.tweak_key,
+            self.start_key,
+            self.len,
+            self.start_time,
+        );
+    }
+}
+
+pub fn list_jobs(
+    jobs_table: &mut redb::Table<'_, u64, (u32, u128, u128, u64, u64)>,
+) -> Result<Vec<JobRecord>, redb::Error> {
+    let mut res: Vec<JobRecord> = Vec::new();
+
+    for job_res in jobs_table.iter()? {
+        let job_rec = JobRecord::from_iter_item(job_res)?;
+        res.push(job_rec);
+    }
+
+    Ok(res)
+}
+
+const RANGES: TableDefinition<u32, (u128, u128, u128, u128)> = TableDefinition::new("ranges");
+const RANGES_TO_USE: TableDefinition<u32, ()> = TableDefinition::new("ranges_to_use");
+
+// to not owerflow u128 we calculate it in range 0..max and then substruct
+// const SUB_DIV: u128 = 0x1000100010001000100010001;
+// TWEAKS_PER_RANGE: u128 = 5192296858534827628530496329220096
+const TWEAKS_PER_RANGE: u128 = (u128::MAX / (u16::MAX as u128 + 1)) + 1;
+const TWEAKS_PER_RANGE_F64: f64 = TWEAKS_PER_RANGE as f64;
+//assert!(TWEAKS_PER_RANGE == 5192296858534827628530496329220096u128);
+
+const TWEAKS_PER_RANGE_ID_VICE: u128 = TWEAKS_PER_RANGE - 1;
+//const HALF_TWEAKS_PER_RANGE: u128 = TWEAKS_PER_RANGE / 2;
+
+#[derive(Copy, Clone, Debug)]
+struct RangeRecord {
+    id: u32,
+
+    tweak_current: u128,
+    tweak_end: u128,
+    key_progress: u128,
+    key_committed_progress: u128,
+}
+
+impl RangeRecord {
+    fn from_value(k: u32, v: &(u128, u128, u128, u128)) -> Self {
+        Self {
+            id: k,
+            tweak_current: v.0,
+            tweak_end: v.1,
+            key_progress: v.2,
+            key_committed_progress: v.3,
+        }
+    }
+
+    fn new(id: u32) -> Self {
+        let tweak_start: u128 = id as u128 * TWEAKS_PER_RANGE;
+        Self {
+            id: id,
+            tweak_current: tweak_start,
+            tweak_end: tweak_start.strict_add(TWEAKS_PER_RANGE_ID_VICE),
+            key_progress: 0,
+            key_committed_progress: 0,
+        }
+    }
+
+    // fn tweaks_left(&self) -> u128 {
+    //     let t_left = (self.tweak_end - self.tweak_current) + 1;
+    //     if t_left == 1 && self.key_committed_progress == u128::MAX {
+    //         0u128
+    //     }
+    //     else {
+    //         t_left
+    //     }
+    // }
+
+    fn completed_ratio(&self) -> f64 {
+        let t_left = self.tweak_end - self.tweak_current;
+        if t_left == 0 && self.key_committed_progress == u128::MAX {
+            1.0f64
+        } else {
+            1.0f64 - ((t_left as f64 + 1.0) / TWEAKS_PER_RANGE_F64)
+        }
+    }
+
+    fn tweak_processed(&self) -> u128 {
+        let t_left = self.tweak_end - self.tweak_current;
+        return TWEAKS_PER_RANGE - t_left;
+    }
+
+    fn value(&self) -> (u128, u128, u128, u128) {
+        (
+            self.tweak_current,
+            self.tweak_end,
+            self.key_progress,
+            self.key_committed_progress,
+        )
+    }
+}
+
+#[derive(Debug)]
+struct RangesTable<'a> {
+    ranges: redb::Table<'a, u32, (u128, u128, u128, u128)>,
+    ranges_to_use: redb::Table<'a, u32, ()>,
+}
+
+impl<'a> RangesTable<'a> {
+    // fn new(
+    //     ranges: redb::Table<'a, u16, (u128, u128, u128, u128)>,
+    //     ranges_to_use: redb::Table<'a, u16, ()>,
+    // ) -> Self {
+    //     RangesTable {
+    //         ranges: ranges,
+    //         ranges_to_use: ranges_to_use,
+    //     }
+    // }
+
+    fn open_rw(trx: &'a redb::WriteTransaction) -> Result<Self, redb::Error> {
+        Ok(RangesTable {
+            ranges: trx.open_table(RANGES)?,
+            ranges_to_use: trx.open_table(RANGES_TO_USE)?,
+        })
+    }
+
+    fn get_range(&self, k: u32) -> Result<Option<RangeRecord>, redb::Error> {
+        Ok(self
+            .ranges
+            .get(k)?
+            .and_then(|ag_v| Some(RangeRecord::from_value(k, &ag_v.value()))))
+    }
+
+    fn get_useful_ranges(&self) -> Result<Vec<RangeRecord>, redb::Error> {
+        let mut u_ranges: Vec<RangeRecord> = Vec::with_capacity(self.ranges_to_use.len()? as usize);
+        for kres in self.ranges_to_use.iter()? {
+            u_ranges.push(
+                self.get_range(kres?.0.value())?.expect(
+                    "Randge id from ranges_to_use should allways be present in ranges table",
+                ),
+            );
+        }
+
+        Ok(u_ranges)
+    }
+
+    fn update_range(&mut self, range: &RangeRecord) -> Result<(), redb::Error> {
+        self.ranges.insert(range.id, range.value())?;
+        // if range key in progress reach max values - remove it from to_use table
+        if range.key_progress == u128::MAX {
+            self.ranges_to_use.remove(range.id)?;
+        }
+        Ok(())
+    }
+}
+
+#[derive(Copy, Clone, Debug)]
+pub struct FoundKey {
+    pub tweak_key: u128,
+    pub enc_key: u128,
+    pub timestump: u64,
+}
+
+pub fn db_get_found_keys(db: &Database) -> Result<Vec<FoundKey>, redb::Error> {
+    let trx: redb::ReadTransaction = db.begin_read()?;
+    let mut res: Vec<FoundKey> = Vec::new();
+
+    {
+        let found_keys_t = trx.open_table(FOUND_KEYS_TABLE)?;
+
+        for found_key_res in found_keys_t.iter()? {
+            let ag = found_key_res?;
+            res.push(FoundKey {
+                tweak_key: ag.0.value().0,
+                enc_key: ag.0.value().1,
+                timestump: ag.1.value(),
+            });
+        }
+    }
+
+    return Ok(res);
+}
+
+pub fn db_put_found_key(db: &Database, tw_k: u128, en_k: u128) -> Result<u64, redb::Error> {
+    let trx: redb::WriteTransaction = db.begin_write()?;
+    let new_keys_count = {
+        let mut found_keys_t = trx.open_table(FOUND_KEYS_TABLE)?;
+        found_keys_t.insert((tw_k, en_k), get_timestump())?;
+        found_keys_t.len()?
+    };
+    trx.commit()?;
+
+    return Ok(new_keys_count);
+}
+
+pub fn db_get_progress(db: &Database) -> Result<f64, redb::Error> {
+    let trx: redb::ReadTransaction = db.begin_read()?;
+    let progress = {
+        let ranges_t = trx.open_table(RANGES)?;
+        let mut mean_prog: f64 = 0.0;
+        for range_res in ranges_t.iter()? {
+            let ag = range_res?;
+            let range = RangeRecord::from_value(ag.0.value(), &ag.1.value());
+            mean_prog += range.completed_ratio();
+        }
+
+        mean_prog / 65536.0f64
+    };
+    trx.close()?;
+    return Ok(progress);
+}
+
+pub fn db_get_progress_details(db: &Database) -> Result<(u128, u128, u128, u128, f64), redb::Error> {
+    let trx: redb::ReadTransaction = db.begin_read()?;
+
+    //let mut mean_prog: f64 = 0.0;
+    let mut min_proc_keys: u128 = u128::MAX;
+    let mut max_proc_keys: u128 = 0;
+    let mut avrg_proc_keys: u128 = 0;
+
+    let mut max_proc_tweks: u128 = 0;
+
+    let mut avg_proc_tweks: f64 = 0.0;
+
+    {
+        let ranges_t = trx.open_table(RANGES)?;
+        for range_res in ranges_t.iter()? {
+            let ag = range_res?;
+            let range = RangeRecord::from_value(ag.0.value(), &ag.1.value());
+            
+            avg_proc_tweks += range.tweak_processed() as f64 / 65536.0f64;
+            if range.tweak_processed() > max_proc_tweks {
+                max_proc_tweks = range.tweak_processed();
+            }
+            if range.key_committed_progress > max_proc_keys {
+                max_proc_keys = range.key_committed_progress
+            }
+            if range.key_committed_progress < min_proc_keys {
+                min_proc_keys = range.key_committed_progress;
+            }
+            avrg_proc_keys += range.key_committed_progress / 65536;
+        }
+    }
+    trx.close()?;
+    return Ok((min_proc_keys, max_proc_keys, avrg_proc_keys, max_proc_tweks, avg_proc_tweks));
+}
+
+fn db_init(db: &Database) -> Result<(), redb::Error> {
+    println!("Start DB init.");
+
+    let trx: redb::WriteTransaction = db.begin_write()?;
+    {
+        let mut ranges = trx.open_table(RANGES)?;
+        let mut ranges_to_use = trx.open_table(RANGES_TO_USE)?;
+        for k in 0..=u16::MAX {
+            // create new ranges
+            let r = RangeRecord::new(k as u32);
+            ranges.insert(k as u32, r.value())?;
+            ranges_to_use.insert(k as u32, ())?;
+        }
+
+        println!("ensure that all tables is created");
+        // ensure that all tables is created
+        trx.open_table(FOUND_KEYS_TABLE)?.len()?;
+        trx.open_table(JOBS_TABLE)?.len()?;
+        trx.open_table(JOBS_FREE_IDS)?.len()?;
+    }
+    println!("Commit transaction");
+    trx.commit()?;
+    println!("Done!");
+
+    Ok(())
+}
+
+pub fn db_open(db_file_path: &Path) -> Result<Database, redb::Error> {
+    let db = Database::create(db_file_path)?;
+    let trx = db.begin_write()?;
+    let ranges_len = {
+        println!("Opening RANGES to get len");
+        let ranges_table = trx.open_table(RANGES)?;
+        println!("Try to get len");
+        ranges_table.len()?
+    };
+    println!("Commit transaction");
+    trx.commit()?;
+    if ranges_len == 0 {
+        db_init(&db)?;
+    }
+
+    Ok(db)
+}
@@ -0,0 +1,17 @@
+
+use std::{path::Path, u64, u128};
+mod database;
+
+fn main() {
+    println!("Hello, world!");
+    let db_path = Path::new("backup_data.rdb");
+    let db = database::db_open(db_path).unwrap();
+    let found_keys = database::db_get_found_keys(&db).unwrap();
+    let pr = database::db_get_progress(&db).unwrap();
+    let d_pr = database::db_get_progress_details(&db).unwrap();
+    println!("found_keys: {}", found_keys.len());
+    println!("progress: {:.1074}", pr);
+    println!("progress det: {:?}", d_pr);
+
+}
+