fix Rust warnings

Author: hipertracker

example-rust/src/main.rs

@@ -1,91 +1,62 @@
+use crossbeam_utils::sync::WaitGroup;
 use glob::glob;
-use itertools::Itertools;
 use lexical_sort::{natural_lexical_cmp, StringSort};
-use once_cell::sync::Lazy;
 use regex::Regex;
-use std::path::Path;
-use tokio::fs;
+use std::collections::HashSet;
+use std::fs;
+use std::thread;
+use time::Instant;
 use yaml_rust::YamlLoader;
 
-const SORT: bool = false;
-const OUTDIR: &str = "words_new";
-const FILE_DIR: &str = "../data/??/**/*.yml";
-static SEPARATOR_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(r"[\W\d]+").unwrap());
-
-async fn create_outdir() -> tokio::io::Result<()> {
-    fs::create_dir_all(OUTDIR).await
-}
-
-async fn read_file(path: &Path) -> String {
-    let raw = fs::read_to_string(path).await.unwrap();
-    raw.to_lowercase().replace('\n', " ")
-}
-
-fn get_unique_token(src: &str) -> Vec<&str> {
-    let mut data = SEPARATOR_REGEX.split(src).unique().collect::<Vec<_>>();
-
-    if SORT {
-        data.string_sort_unstable(natural_lexical_cmp);
+fn main() -> std::io::Result<()> {
+    let start = Instant::now();
+    let with_sorting = false;
+    let outdir = "words";
+    fs::create_dir_all(outdir)?;
+    let wg = WaitGroup::new();
+    let path = "../data/??/**/*.yml";
+    for entry in glob(path).expect("Failed to read glob pattern") {
+        match entry {
+            Ok(path) => {
+                // let separator = Regex::new(r"[^\p{L}]+").unwrap();
+                let separator = Regex::new(r"[\W\d]+").unwrap();
+                let wg = wg.clone();
+                thread::spawn(move || {
+                    let filepath = path.to_str().unwrap().replace(".yml", ".txt");
+                    // println!("{:?}", filepath);
+                    let text = fs::read_to_string(&filepath)
+                        .unwrap()
+                        .to_lowercase()
+                        .replace("\n", " ");
+                    let tokens: Vec<&str> = separator.split(&text).collect();
+                    let unique_tokens: HashSet<&str> = tokens.into_iter().collect();
+                    let mut words: Vec<&str>;
+                    if with_sorting {
+                        words = unique_tokens.into_iter().collect();
+                        words.string_sort_unstable(natural_lexical_cmp);
+                    } else {
+                        words = unique_tokens.into_iter().collect();
+                    }
+                    let yaml = fs::read_to_string(&path).unwrap();
+                    let docs = YamlLoader::load_from_str(&yaml).unwrap();
+                    let meta = &docs[0];
+                    let out = format!(
+                        "{}/{}-{}.txt",
+                        outdir,
+                        meta["lang"].as_str().unwrap(),
+                        meta["code"].as_str().unwrap()
+                    );
+                    if let Err(e) = fs::write(out, words.join("\n")) {
+                        println!("Writing error: {}", e.to_string());   
+                    }
+                    drop(wg);
+                });
+            }
+            Err(e) => println!("{:?}", e),
+        }
     }
-
-    data
-}
-
-async fn get_filename_from_meta(path: &Path) -> anyhow::Result<String> {
-    let yaml = fs::read_to_string(path).await?;
-    let docs = YamlLoader::load_from_str(&yaml)?;
-    let meta = &docs[0];
-
-    let label = meta["label"]
-        .as_str()
-        .ok_or_else(|| anyhow::anyhow!("label not found"))?;
-
-    Ok(format!("{}/extracted-words-for-{}.txt", OUTDIR, label))
-}
-
-#[tokio::main]
-async fn main() -> std::io::Result<()> {
-    let start = std::time::Instant::now();
-    let path = glob(FILE_DIR).expect("failed to read glob pattern");
-
-    let submissions = path.map(|entry| {
-        tokio::spawn(async {
-            let yaml_path = entry.expect("should be existed");
-            let txt_path = yaml_path.with_extension("txt");
-
-            let outdir_submission =
-                tokio::spawn(async { create_outdir().await.expect("unable to create outdir") });
-
-            let read_text_file_submission = tokio::spawn(async move {
-                let data = read_file(&txt_path).await;
-                let tokens = get_unique_token(&data);
-
-                tokens.join("\n")
-            });
-
-            let filename_submission = tokio::spawn(async move {
-                get_filename_from_meta(&yaml_path)
-                    .await
-                    .expect("should be existed")
-            });
-
-            let (tokens, filename, _) = tokio::join!(
-                read_text_file_submission,
-                filename_submission,
-                outdir_submission,
-            );
-
-            fs::write(
-                filename.expect("failed to run filename"),
-                tokens.expect("failed to get tokens"),
-            )
-            .await
-            .expect("failed to write");
-        })
-    });
-
-    futures::future::join_all(submissions).await;
-
-    println!("{:?}", start.elapsed());
+    wg.wait();
+    let end = Instant::now();
+    println!("{:?} seconds.", end - start);
     Ok(())
 }