Miscellaneous changes #266
4 changed files with 25 additions and 13 deletions
|
@ -1,4 +1,5 @@
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
use clap::{value_parser, Arg, ArgAction, ArgMatches, Command, ValueEnum};
|
use clap::{value_parser, Arg, ArgAction, ArgMatches, Command, ValueEnum};
|
||||||
use cli_table::{print_stdout, WithTitle};
|
use cli_table::{print_stdout, WithTitle};
|
||||||
|
@ -156,6 +157,8 @@ pub(crate) async fn run(mut ctx: sdk::Context, matches: &ArgMatches) -> Result<(
|
||||||
BufReader::new(Box::new(f))
|
BufReader::new(Box::new(f))
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let lookup = Arc::make_mut(&mut ctx.lookup);
|
||||||
|
|
||||||
let group = sdk::murmur::HashGroup::from(*group);
|
let group = sdk::murmur::HashGroup::from(*group);
|
||||||
|
|
||||||
let mut added = 0;
|
let mut added = 0;
|
||||||
|
@ -165,15 +168,15 @@ pub(crate) async fn run(mut ctx: sdk::Context, matches: &ArgMatches) -> Result<(
|
||||||
let total = {
|
let total = {
|
||||||
for line in lines.into_iter() {
|
for line in lines.into_iter() {
|
||||||
let value = line?;
|
let value = line?;
|
||||||
if ctx.lookup.find(&value, group).is_some() {
|
if lookup.find(&value, group).is_some() {
|
||||||
skipped += 1;
|
skipped += 1;
|
||||||
} else {
|
} else {
|
||||||
ctx.lookup.add(value, group);
|
lookup.add(value, group);
|
||||||
added += 1;
|
added += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx.lookup.len()
|
lookup.len()
|
||||||
};
|
};
|
||||||
|
|
||||||
let out_path = matches
|
let out_path = matches
|
||||||
|
@ -190,7 +193,7 @@ pub(crate) async fn run(mut ctx: sdk::Context, matches: &ArgMatches) -> Result<(
|
||||||
})
|
})
|
||||||
.with_section(|| out_path.display().to_string().header("Path:"))?;
|
.with_section(|| out_path.display().to_string().header("Path:"))?;
|
||||||
|
|
||||||
ctx.lookup
|
lookup
|
||||||
.to_csv(f)
|
.to_csv(f)
|
||||||
.await
|
.await
|
||||||
.wrap_err("Failed to write dictionary to disk")?;
|
.wrap_err("Failed to write dictionary to disk")?;
|
||||||
|
|
|
@ -12,6 +12,7 @@ use clap::value_parser;
|
||||||
use clap::{command, Arg};
|
use clap::{command, Arg};
|
||||||
use color_eyre::eyre;
|
use color_eyre::eyre;
|
||||||
use color_eyre::eyre::{Context, Result};
|
use color_eyre::eyre::{Context, Result};
|
||||||
|
use sdk::murmur::Dictionary;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use tokio::fs::File;
|
use tokio::fs::File;
|
||||||
use tokio::io::BufReader;
|
use tokio::io::BufReader;
|
||||||
|
@ -107,8 +108,9 @@ async fn main() -> Result<()> {
|
||||||
|
|
||||||
let r = BufReader::new(f);
|
let r = BufReader::new(f);
|
||||||
let mut ctx = ctx.write().await;
|
let mut ctx = ctx.write().await;
|
||||||
if let Err(err) = ctx.lookup.from_csv(r).await {
|
match Dictionary::from_csv(r).await {
|
||||||
tracing::error!("{:#}", err);
|
Ok(lookup) => ctx.lookup = Arc::new(lookup),
|
||||||
|
Err(err) => tracing::error!("{:#}", err),
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
};
|
};
|
||||||
|
|
|
@ -1,8 +1,11 @@
|
||||||
|
use std::ffi::OsString;
|
||||||
|
use std::path::PathBuf;
|
||||||
use std::process::Command;
|
use std::process::Command;
|
||||||
use std::{ffi::OsString, path::PathBuf};
|
use std::sync::Arc;
|
||||||
|
|
||||||
use crate::murmur::{Dictionary, HashGroup, IdString64, Murmur32, Murmur64};
|
use crate::murmur::{Dictionary, HashGroup, IdString64, Murmur32, Murmur64};
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
pub struct CmdLine {
|
pub struct CmdLine {
|
||||||
cmd: OsString,
|
cmd: OsString,
|
||||||
args: Vec<OsString>,
|
args: Vec<OsString>,
|
||||||
|
@ -52,7 +55,7 @@ impl From<&CmdLine> for Command {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct Context {
|
pub struct Context {
|
||||||
pub lookup: Dictionary,
|
pub lookup: Arc<Dictionary>,
|
||||||
pub ljd: Option<CmdLine>,
|
pub ljd: Option<CmdLine>,
|
||||||
pub revorb: Option<String>,
|
pub revorb: Option<String>,
|
||||||
pub ww2ogg: Option<String>,
|
pub ww2ogg: Option<String>,
|
||||||
|
@ -62,7 +65,7 @@ pub struct Context {
|
||||||
impl Context {
|
impl Context {
|
||||||
pub fn new() -> Self {
|
pub fn new() -> Self {
|
||||||
Self {
|
Self {
|
||||||
lookup: Dictionary::new(),
|
lookup: Arc::new(Dictionary::new()),
|
||||||
ljd: None,
|
ljd: None,
|
||||||
revorb: None,
|
revorb: None,
|
||||||
ww2ogg: None,
|
ww2ogg: None,
|
||||||
|
|
|
@ -48,6 +48,7 @@ struct Row {
|
||||||
group: HashGroup,
|
group: HashGroup,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
pub struct Entry {
|
pub struct Entry {
|
||||||
value: String,
|
value: String,
|
||||||
long: Murmur64,
|
long: Murmur64,
|
||||||
|
@ -73,6 +74,7 @@ impl Entry {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
pub struct Dictionary {
|
pub struct Dictionary {
|
||||||
entries: Vec<Entry>,
|
entries: Vec<Entry>,
|
||||||
}
|
}
|
||||||
|
@ -88,10 +90,12 @@ impl Dictionary {
|
||||||
Self { entries: vec![] }
|
Self { entries: vec![] }
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn from_csv<R>(&mut self, r: R) -> Result<()>
|
pub async fn from_csv<R>(r: R) -> Result<Self>
|
||||||
where
|
where
|
||||||
R: AsyncRead + std::marker::Unpin + std::marker::Send,
|
R: AsyncRead + std::marker::Unpin + std::marker::Send,
|
||||||
{
|
{
|
||||||
|
let mut entries = vec![];
|
||||||
|
|
||||||
let r = AsyncDeserializer::from_reader(r);
|
let r = AsyncDeserializer::from_reader(r);
|
||||||
let mut records = r.into_deserialize::<Row>();
|
let mut records = r.into_deserialize::<Row>();
|
||||||
|
|
||||||
|
@ -112,10 +116,10 @@ impl Dictionary {
|
||||||
group: record.group,
|
group: record.group,
|
||||||
};
|
};
|
||||||
|
|
||||||
self.entries.push(entry);
|
entries.push(entry);
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(Self { entries })
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn to_csv<W>(&self, w: W) -> Result<()>
|
pub async fn to_csv<W>(&self, w: W) -> Result<()>
|
||||||
|
@ -161,7 +165,7 @@ impl Dictionary {
|
||||||
self.entries.push(entry);
|
self.entries.push(entry);
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn find(&mut self, value: &String, group: HashGroup) -> Option<&Entry> {
|
pub fn find(&self, value: &String, group: HashGroup) -> Option<&Entry> {
|
||||||
self.entries
|
self.entries
|
||||||
.iter()
|
.iter()
|
||||||
.find(|e| e.value == *value && e.group == group)
|
.find(|e| e.value == *value && e.group == group)
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue