bitsquid_dt/dtmt
bitsquid_dt/dtmt
2
Fork 0
Code Issues 34 Pull requests 11 Projects 1 Releases 3 Packages 1 Wiki Activity
Var Dump: 
dumpVar: only available in dev mode
Mailing List

sdk: Reimplement logging current word

Browse source
This commit is contained in:
Lucas Schwiderski 2023-09-19 16:15:22 +02:00
parent b366185a63
commit 6449354714
Signed by: lucas
GPG key ID: AA12679AAA6DF4D8
1 changed files with 36 additions and 50 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

86
crates/dtmt/src/cmd/experiment/brute_force_words.rs
View file

@ -1,6 +1,6 @@
use std::collections::HashSet;
use std::fs;
use std::io::Write;
use std::io::{BufWriter, Write};
use std::path::PathBuf;
use std::sync::Arc;
use std::thread::JoinHandle;
@ -122,35 +122,30 @@ const PREFIXES: [&str; 29] = [
"wwise/world_sound_fx/",
];
fn make_info_printer(rx: Receiver<(usize, usize)>, hash_count: usize) -> JoinHandle<()> {
fn make_info_printer(rx: Receiver<(usize, usize, String)>, hash_count: usize) -> JoinHandle<()> {
std::thread::spawn(move || {
let mut writer = std::io::stderr();
let mut total_count = 0;
let mut total_found = 0;
let start = Instant::now();
let mut start = Instant::now();
while let Ok((count, found)) = rx.recv() {
while let Ok((count, found, last)) = rx.recv() {
total_count += count;
total_found += found;
let dur = Instant::now() - start;
if dur.as_secs() > 1 {
let s = format!("\r{total_count} per second | {total_found:6}/{hash_count} found",);
// let s = String::from_utf8_lossy(&buf);
// // The last prefix in the set is the one that will stay in the buffer
// // when we're about to print here.
// // So we strip that, to show just the generated part.
// // We also restrict the length to stay on a single line.
// let prefix_len = prefixes[28].len();
// let s = s[prefix_len..std::cmp::min(s.len(), prefix_len + 60)]
// .trim_end()
// .to_string();
let now = Instant::now();
if (now - start).as_millis() > 250 {
let s = &last[0..std::cmp::min(last.len(), 60)];
let s = format!(
"\r{:12} per second | {total_found:6}/{hash_count} found | {s:<60}",
total_count * 4
);
writer.write_all(s.as_bytes()).unwrap();
total_count = 0;
start = now;
}
}
})
@ -158,7 +153,7 @@ fn make_info_printer(rx: Receiver<(usize, usize)>, hash_count: usize) -> JoinHan
fn make_stdout_printer(rx: Receiver<Vec<u8>>) -> JoinHandle<()> {
std::thread::spawn(move || {
let mut writer = std::io::stdout();
let mut writer = BufWriter::new(std::io::stdout());
while let Ok(buf) = rx.recv() {
writer.write_all(&buf).unwrap();
@ -172,7 +167,7 @@ struct State {
words: Arc<Vec<String>>,
delimiters_len: usize,
stdout_tx: Sender<Vec<u8>>,
info_tx: Sender<(usize, usize)>,
info_tx: Sender<(usize, usize, String)>,
}
fn make_worker(rx: Receiver<Vec<usize>>, state: State) -> JoinHandle<()> {
@ -186,7 +181,6 @@ fn make_worker(rx: Receiver<Vec<usize>>, state: State) -> JoinHandle<()> {
let mut found = 0;
let mut buf = Vec::with_capacity(1024);
// while let Some(indices) = find_task(local, global, &[]) {
while let Ok(indices) = rx.recv() {
let sequence = indices.iter().map(|i| words[*i].as_str());
@ -224,7 +218,7 @@ fn make_worker(rx: Receiver<Vec<usize>>, state: State) -> JoinHandle<()> {
found += 1;
buf.push(LINE_FEED);
if let Err(_) = state.stdout_tx.send(buf.clone()) {
if state.stdout_tx.send(buf.clone()).is_err() {
return;
}
} else {
@ -250,7 +244,7 @@ fn make_worker(rx: Receiver<Vec<usize>>, state: State) -> JoinHandle<()> {
found += 1;
buf.push(LINE_FEED);
if let Err(_) = state.stdout_tx.send(buf.clone()) {
if state.stdout_tx.send(buf.clone()).is_err() {
return;
}
} else {
@ -261,30 +255,22 @@ fn make_worker(rx: Receiver<Vec<usize>>, state: State) -> JoinHandle<()> {
}
}
if count >= 1024 * 1024 {
let _ = state.info_tx.send((count, found));
if count >= 2 * 1024 * 1024 {
// The last prefix in the set is the one that will stay in the buffer
// when we're about to print here.
// So we strip that, to show just the generated part.
// We also restrict the length to stay on a single line.
let prefix_len = PREFIXES[28].len();
// No need to wait for this
let _ = state.info_tx.try_send((
count,
found,
String::from_utf8_lossy(&buf[prefix_len..]).to_string(),
));
count = 0;
found = 0;
}
// let dur = Instant::now() - start;
// if dur.as_secs() >= 1 {
// let hashes_len = hashes.len();
// let s = String::from_utf8_lossy(&buf);
// // The last prefix in the set is the one that will stay in the buffer
// // when we're about to print here.
// // So we strip that, to show just the generated part.
// // We also restrict the length to stay on a single line.
// let prefix_len = prefixes[28].len();
// let s = s[prefix_len..std::cmp::min(s.len(), prefix_len + 60)]
// .trim_end()
// .to_string();
// info_tx.send(format!(
// "\r{:8} hashes per second | {:6}/{} found | {:<60}",
// count, found, hashes_len, s
// ));
// start = Instant::now();
// count = 0;
// }
}
})
}
@ -430,9 +416,9 @@ pub(crate) fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()> {
};
tracing::debug!("{:?}", delimiter_lists);
let (info_tx, info_rx) = unbounded();
let (info_tx, info_rx) = bounded(100);
let (stdout_tx, stdout_rx) = unbounded::<Vec<u8>>();
let (task_tx, task_rx) = bounded::<Vec<usize>>(100);
let (task_tx, task_rx) = bounded::<Vec<usize>>(num_threads * 4);
let mut handles = Vec::new();
for _ in 0..num_threads {
@ -456,7 +442,7 @@ pub(crate) fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()> {
drop(stdout_tx);
drop(info_tx);
// handles.push(make_info_printer(info_rx, hashes.len()));
handles.push(make_info_printer(info_rx, hashes.len()));
handles.push(make_stdout_printer(stdout_rx));
let mut indices =
@ -508,8 +494,6 @@ pub(crate) fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()> {
// complete as well.
drop(task_tx);
tracing::debug!("Wainting for workers to finish.");
for handle in handles {
match handle.join() {
Ok(_) => {}
@ -523,5 +507,7 @@ pub(crate) fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()> {
}
}
let _ = std::io::stdout().write_all("\r".as_bytes());
Ok(())
}