diff --git a/nyash_server/Cargo.lock b/nyash_server/Cargo.lock index a705d68..8a32620 100644 --- a/nyash_server/Cargo.lock +++ b/nyash_server/Cargo.lock @@ -2,6 +2,157 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "libc" +version = "0.2.180" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" + [[package]] name = "nyash_server" version = "0.1.0" +dependencies = [ + "rand", + "redb", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74d9a594b72ae6656596548f56f667211f8a97b3d4c3d467150794690dc40a" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "redb" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae323eb086579a3769daa2c753bb96deb95993c534711e0dbe881b5192906a06" +dependencies = [ + "libc", +] + +[[package]] +name = "syn" +version = "2.0.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" + +[[package]] +name = "zerocopy" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668f5168d10b9ee831de31933dc111a459c97ec93225beb307aed970d1372dfd" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c7962b26b0a8685668b671ee4b54d007a67d4eaf05fda79ac0ecf41e32270f1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/nyash_server/Cargo.toml b/nyash_server/Cargo.toml index 8d1e1a7..15f43d6 100644 --- a/nyash_server/Cargo.toml +++ b/nyash_server/Cargo.toml @@ -4,3 +4,5 @@ version = "0.1.0" edition = "2024" [dependencies] +rand = "0.9.2" +redb = "3.1.0" diff --git a/nyash_server/src/database.rs b/nyash_server/src/database.rs new file mode 100644 index 0000000..ab54824 --- /dev/null +++ b/nyash_server/src/database.rs @@ -0,0 +1,424 @@ +use std::{u64, u128}; + +use crate::num_utils; +use rand::{self, Rng}; +use redb::{ + Database, MultimapTableDefinition, ReadableMultimapTable, ReadableTable, ReadableTableMetadata, + TableDefinition, +}; + +//tables defenition +const DB_FILE: &str = "./data"; +const RANGE_INFO: TableDefinition = TableDefinition::new("range_info"); + +const RANGE_STATUS: MultimapTableDefinition = + MultimapTableDefinition::new("range_status"); + +const JOBS_TABLE: TableDefinition = TableDefinition::new("jobs_state"); +const JOBS_FREE_IDS: TableDefinition = TableDefinition::new("jobs_free_ids"); + +// -- main_range -- +// (start_tweak_key) (end_tweak) (committed) (progress) +// (u128) (u128, u128, u128) +// (u32,u32,u32,u32,u32,u32,u32,u32) + +// -- jobs_state -- +//(job_id) (tweak_key, start_key, len, start_time) +// u64 (u128, u128, u64, u64) + +// get time is seconds +fn get_timestump() -> u64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs() +} + +#[derive(Copy, Clone, Debug)] +struct JobRecord { + job_id: u64, + tweak_key: u128, + start_key: u128, + len: u64, + start_time: u64, +} + +impl JobRecord { + pub fn from_data(rk: u64, rv: (u128, u128, u64, u64)) -> Self { + Self { + job_id: rk, + tweak_key: rv.0, + start_key: rv.1, + len: rv.2, + start_time: rv.3, + } + } + + pub fn from_acces_guard( + ag: ( + redb::AccessGuard<'_, u64>, + redb::AccessGuard<'_, (u128, u128, u64, u64)>, + ), + ) -> Self { + Self::from_data(ag.0.value(), ag.1.value()) + } + + pub fn from_option( + op: Option<( + redb::AccessGuard<'_, u64>, + redb::AccessGuard<'_, (u128, u128, u64, u64)>, + )>, + ) -> Option { + match op { + Some(ag) => Some(Self::from_acces_guard(ag)), + None => None, + } + } + + pub fn from_result( + res: Result< + ( + redb::AccessGuard<'_, u64>, + redb::AccessGuard<'_, (u128, u128, u64, u64)>, + ), + redb::StorageError, + >, + ) -> Result { + let ag_res = res?; + Ok(Self::from_data(ag_res.0.value(), ag_res.1.value())) + } + + pub fn get_value(&self) -> (u128, u128, u64, u64) { + (self.tweak_key, self.start_key, self.len, self.start_time) + } +} + +#[derive(Copy, Clone, Debug)] +struct RangeRecord { + start_tweak: u128, + end_tweak: u128, + committed: u128, + in_progress: u128, +} + +impl RangeRecord { + pub fn create_job(&mut self, job_id: u64, job_len: u64) -> JobRecord { + let job_rec = JobRecord { + job_id: job_id, + tweak_key: self.end_tweak, + start_key: self.in_progress, + len: job_len, + start_time: get_timestump(), + }; + + self.in_progress = self.in_progress.strict_add(job_len as u128); // panic if owerflow + return job_rec; + } + + pub fn new(start_tweak: u128) -> Self { + Self { + start_tweak: start_tweak, + end_tweak: start_tweak, + committed: 0, + in_progress: 0, + } + } + + pub fn from_data(rk: u128, rv: (u128, u128, u128)) -> Self { + Self { + start_tweak: rk, + end_tweak: rv.0, + committed: rv.1, + in_progress: rv.2, + } + } + + pub fn from_acces_guard( + ag: ( + redb::AccessGuard<'_, u128>, + redb::AccessGuard<'_, (u128, u128, u128)>, + ), + ) -> Self { + Self::from_data(ag.0.value(), ag.1.value()) + } + + pub fn from_option( + op: Option<( + redb::AccessGuard<'_, u128>, + redb::AccessGuard<'_, (u128, u128, u128)>, + )>, + ) -> Option { + match op { + Some(ag) => Some(Self::from_acces_guard(ag)), + None => None, + } + } + + pub fn from_result( + res: Result< + ( + redb::AccessGuard<'_, u128>, + redb::AccessGuard<'_, (u128, u128, u128)>, + ), + redb::StorageError, + >, + ) -> Result { + let ag_res = res?; + Ok(Self::from_data(ag_res.0.value(), ag_res.1.value())) + } + + pub fn get_value(&self) -> (u128, u128, u128) { + (self.end_tweak, self.committed, self.in_progress) + } + + pub fn increase_progress(self, job_len: u64) -> Self { + let (res, carry) = self.in_progress.carrying_add(job_len as u128, false); + if carry == true { + panic!("Increase progress resulted in u128 integer overflow, this should not happen!"); + } + Self { + in_progress: res, + ..self + } + } +} + +fn get_range( + range_info: &mut redb::Table<'_, u128, (u128, u128, u128)>, + range_status: &mut redb::MultimapTable<'_, bool, u128>, +) -> Result, redb::Error> { + // 1. Go tru active ranges and check if their in_progress less than u128::MAX + let mut ranges_list: Vec = Vec::new(); + for wrap_id in range_status.get(false)? { + let key = wrap_id?.value(); + let val = range_info + .get(key)? + .expect("range_table shuld contain key from range_active table") + .value(); + + let range = RangeRecord::from_data(key, val); + + if range.in_progress < u128::MAX { + ranges_list.push(range); + } + } + + // If found active ranges to use, choose one randomly from them + if ranges_list.len() > 0 { + let r_idx: usize = rand::rng().random_range(0..ranges_list.len()); + return Ok(Some(ranges_list[r_idx])); + } + + // Othervice, we need to create new range + // If no ranges in table at all, we creating first one + if range_info.len()? == 0 { + let start_tweak = u128::MAX / 2; + let new_range = RangeRecord::new(start_tweak); + range_info.insert(start_tweak, new_range.get_value())?; + range_status.insert(false, start_tweak)?; + return Ok(Some(new_range)); + } + + // + // if some records already exist, but no that we can use, + // we should add new range + let first_rec = + RangeRecord::from_option(range_info.first()?).expect("Table should not be empty"); + let last_rec = RangeRecord::from_option(range_info.last()?).expect("Table should not be empty"); + + let tweak_max = last_rec.end_tweak; + let tweak_min = first_rec.start_tweak; + + // check if we are done! + if tweak_max == u128::MAX && tweak_min == 0 { + return Ok(None); + } + + // randomly choosing to add new range to low or high part of the range + let start_tweak = if rand::rng().random_bool(0.5) { + // new range on upper + if tweak_max < u128::MAX { + tweak_max + 1 + } else { + tweak_min - 1 + } + } else { + // new range on lower + if tweak_min > 0 { + tweak_min - 1 + } else { + tweak_max + 1 + } + }; + + let new_range = RangeRecord::new(start_tweak); + range_info.insert(start_tweak, new_range.get_value())?; + range_status.insert(false, start_tweak)?; + return Ok(Some(new_range)); +} + +// search for abandoned job +// returns first job older than 3 hours +fn find_abandoned_job( + jobs_table: &redb::Table<'_, u64, (u128, u128, u64, u64)>, +) -> Result, redb::Error> { + let current_time = get_timestump(); + for job_res in jobs_table.iter()? { + let job_rec = JobRecord::from_result(job_res)?; + // if job is older that 3 hours + if (current_time - job_rec.start_time) > 10800 { + return Ok(Some(job_rec)); + } + } + Ok(None) +} + +// 1. Check if there is free ids to reuse +// If there is, returns first free ID +// and deletes it from jobs_free_ids_table +fn get_job_free_id( + jobs_free_ids_table: &mut redb::Table<'_, u64, ()>, +) -> Result, redb::Error> { + // 1. Check if there is free ids to reuse + if jobs_free_ids_table.len()? > 0 { + match jobs_free_ids_table.pop_first()? { + Some(ag) => Ok(Some(ag.0.value())), + None => Ok(None), + } + } else { + Ok(None) + } +} + +// calling this if there was no free ids. +// What we want is to check max and min ids +// and return id lower than min or grater than max +fn get_new_job_id( + jobs_table: &redb::Table<'_, u64, (u128, u128, u64, u64)>, +) -> Result { + // If table is empty return 0 ID + if jobs_table.len()? == 0 { + return Ok(0); + } + + // if min key greater than 0 + if let Some(ag) = jobs_table.first()? { + let min_kv = ag.0.value(); + if min_kv > 0 { + return Ok(min_kv.strict_sub(1u64)); + } + } else { + panic!("Because we checked for empty table, we should never get None here"); + } + + // now try to get new key from maximum key value + if let Some(ag) = jobs_table.last()? { + let max_kv = ag.0.value(); + if max_kv < u64::MAX { + return Ok(max_kv.strict_add(1u64)); + } else { + panic!("Key value should never reach u64 max value"); + } + } else { + panic!("Because we checked for empty table, we should never get None here"); + } +} + +const MAX_U64_AS_U128: u128 = u64::MAX as u128; +fn db_get_job(db: Database, pref_job_size: u64) -> Result, redb::Error> { + let transct = db.begin_write()?; + + // 1. look for abandonent jobs + let new_job_opt = { + let mut jobs_table = transct.open_table(JOBS_TABLE)?; + + if let Some(mut abandoned_job) = find_abandoned_job(&jobs_table)? { + // update job start time + abandoned_job.start_time = get_timestump(); + jobs_table.insert(abandoned_job.job_id, abandoned_job.get_value())?; + Some(abandoned_job) + } else { + None + } + }; + + if new_job_opt.is_some() { + transct.commit()?; + return Ok(new_job_opt); + } + + // 2. look for free ids to reuse or generate new id + let new_job_opt = { + let mut range_info = transct.open_table(RANGE_INFO)?; + let mut range_status = transct.open_multimap_table(RANGE_STATUS)?; + let mut jobs_table = transct.open_table(JOBS_TABLE)?; + let mut jobs_free_ids = transct.open_table(JOBS_FREE_IDS)?; + + // look for free ids to reuse + let job_id_to_use = match get_job_free_id(&mut jobs_free_ids)? { + Some(v) => v, + None => { + // need to generate ID + get_new_job_id(&jobs_table)? + } + }; + + // 3 Now get available range + let range_opt = get_range(&mut range_info, &mut range_status)?; + // if we found range - proceed. If None - means we already done. + match range_opt { + Some(mut range) => { + let job_len = + pref_job_size.min(MAX_U64_AS_U128.min(u128::MAX - range.in_progress) as u64); + if job_len == 0 { + panic!("Job len should not ended as 0"); + } + let new_job = range.create_job(job_id_to_use, job_len); + jobs_table.insert(new_job.job_id, new_job.get_value())?; + range_info.insert(range.start_tweak, range.get_value())?; + Some(new_job) + } + None => None, // We already done. All key space is processed. + } + }; + + if new_job_opt.is_some() { + // Commit transaction if Job was created + transct.commit()?; + } + + // If None - we already done. All key space is processed. + // Return new_job_opt that contains Some if Job was created + // or None if all ranges are searched + return Ok(new_job_opt); + + // ********************************************** +} + +// fn test_db() -> Result<(), Error> { + +// let db = Database::create(DB_FILE)?; +// let write_txn = db.begin_write()?; +// { +// let mut table = write_txn.open_table(RANGE_TABLE)?; +// table.insert([0,1,2,3,0,124,6,7], (42u128, 42u128, false))?; + +// } +// write_txn.commit()?; + +// let read_txn = db.begin_read()?; +// let table = read_txn.open_table(RANGE_TABLE)?; +// // println!("{}", table.get([0,1,2,3,3,5,6,7])?.unwrap().value()); +// let a = table.get([0,1,2,1,4,5,6,7]); +// match a { +// Ok(data) => match data { +// Some(d) => println!("some data {}", d.value().0), +// None => println!("None") +// } +// Err(_) => println!("Error") +// } +// assert_eq!(table.get([0,1,2,3,3,5,6,7])?.unwrap().value().0, 42u128); + +// Ok(()) +// } diff --git a/nyash_server/src/launch.json b/nyash_server/src/launch.json new file mode 100644 index 0000000..d3094cd --- /dev/null +++ b/nyash_server/src/launch.json @@ -0,0 +1,14 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Debug executable", + "type": "lldb", + "request": "launch", + "program": "${workspaceFolder}/target/debug/nyash_server", + "args": [], + "cwd": "${workspaceFolder}/src", // Set to your desired directory + "stopAtEntry": false + } + ] +} \ No newline at end of file diff --git a/nyash_server/src/main.rs b/nyash_server/src/main.rs index e7a11a9..e016073 100644 --- a/nyash_server/src/main.rs +++ b/nyash_server/src/main.rs @@ -1,3 +1,33 @@ + +mod num_utils; +mod database; + + + + + + + fn main() { println!("Hello, world!"); + let a = 1855454745u128.to_le_bytes(); + let b = 1u32.to_le_bytes(); + //let c = u32::from_le_bytes(a[0..4].try_into().unwrap()); + println!("{:?}", a); + println!("{:?}", b); + + let (c,r) = a.as_chunks::<4>(); + println!("{:?}", c); + println!("{:?}", r); + let a1 = 128u32; + let e = a1.min(u32::MAX); + println!("e = {}",e); + + } + + +// Так. Нам нужно база данныъ с дипазонами всей фигни. И сервить её через tls или что-то типо того. + + + diff --git a/nyash_server/src/num_utils.rs b/nyash_server/src/num_utils.rs new file mode 100644 index 0000000..7e9df54 --- /dev/null +++ b/nyash_server/src/num_utils.rs @@ -0,0 +1,157 @@ +// union U64orU32 { +// l: u64, +// i: [u32; 2], +// } + +fn u128_to_u32arr(a: u128) -> [u32;4] { + let mut res = [0u32;4]; + let a_bytes = a.to_le_bytes(); + let chunks = a_bytes.as_chunks::<4>().0; + for i in 0..4 { + res[i] = u32::from_le_bytes(chunks[i]); + } + return res; +} + +pub fn add_u128_to_u256(a: &[u32; 8], b: u128) -> ([u32; 8], bool) { + let mut res: [u32; 8] = [0; 8]; + let mut carry = false; + + // convert b u128 value to bytes then to u32 and then add it to a + b.to_le_bytes() + .as_chunks::<4>() + .0 + .iter() + .enumerate() + .for_each(|(i, sl)| { + (res[i], carry) = a[i].carrying_add(u32::from_le_bytes(*sl), carry); + }); + + // propagate carry till the end of a + for idx in 4..8 { + (res[idx], carry) = a[idx].carrying_add(0, carry); + } + + return (res, carry); +} + +fn add_u32_to_u256(a: &[u32; 8], b: u32) -> ([u32; 8], bool) { + let mut res: [u32; 8] = [0; 8]; + let mut carry = false; + (res[0], carry) = a[0].carrying_add(b, carry); + + for idx in 1..8 { + (res[idx], carry) = a[idx].carrying_add(0, carry); + } + + return (res, carry); +} + +fn add_u32_to_u256_(a: &mut [u32; 8], b: u32) -> bool { + let mut carry = false; + (a[0], carry) = a[0].carrying_add(b, carry); + + for idx in 1..8 { + (a[idx], carry) = a[idx].carrying_add(0, carry); + } + + return carry; +} + +fn bytes_from_chars(chars_chunk: &[char]) -> [u8; 4] { + let mut res: [u8; 4] = [0; 4]; + let mut idx: usize = 0; + chars_chunk.chunks_exact(2).for_each(|b_c| { + if idx < 4 { + match u8::from_str_radix(&b_c.iter().collect::(), 16) { + Ok(n) => res[idx] = n, + Err(_) => (), + } + idx += 1; + } + }); + return res; +} + +fn bignum_from_hex(hex: &str) -> [u32; 8] { + let mut res: [u32; 8] = [0; 8]; + let mut idx: usize = 0; + let chars_hex = hex.chars().collect::>(); + chars_hex + .chunks_exact(8) + .rev() + .map(|chunk| bytes_from_chars(chunk)) + .for_each(|b_arr| { + if idx < 8 { + res[idx] = u32::from_be_bytes(b_arr); + idx += 1; + } + }); + return res; +} + +fn hex_fmt_byte(n: u32) -> String { + let res: String = n + .to_be_bytes() + .iter() + .map(|b| format!("{:02x}", b)) + .collect(); + return res; +} + +fn bignum_to_hex(a: &[u32; 8]) -> String { + let res: String = a + .iter() + .rev() + .map(|n| hex_fmt_byte(*n)) + .collect::>() + .join(""); + return res; +} + +#[cfg(test)] +mod num_utils_tests { + use std::io::Read; + + use super::*; + + #[test] + fn test_add() { + use std::io::{BufRead, BufReader}; + use std::process::{Command, Stdio}; + + let test_gen_cmd = "/home/kira/Development/Rust/nyash-aes-xts256-plain64/nyash_client/src/tests/gen_test_data.py"; + let mut child = Command::new(test_gen_cmd) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + + let gen_stdout = child + .stdout + .take() + .ok_or("Failed to capture stdout") + .unwrap(); + let gen_reader = BufReader::new(gen_stdout); + + for r_line in gen_reader.lines() { + let test_line: String = r_line.unwrap(); // Handle any I/O errors + let test_data_line = test_line.split(' ').collect::>(); + let num_to_add = u32::from_str_radix(test_data_line[0], 10).unwrap(); + let t0 = bignum_from_hex(test_data_line[1]); + + let t1_test = add_u32_to_u256(&t0, 1).0; + let t2_test = add_u32_to_u256(&t0, num_to_add).0; + + let res_actual = format!( + "{} {} {} {}", + num_to_add, + bignum_to_hex(&t0), + bignum_to_hex(&t1_test), + bignum_to_hex(&t2_test) + ); + assert_eq!(test_line, res_actual); + } + + let _ = child.wait().unwrap(); + } +}