update Rust example and fixing Golang result

Author: hipertracker

README.md

@@ -14,39 +14,35 @@ The following results are for 123 unique utf-8 Bible text files in 23 languages
 * Machine: MacBook Pro 16" 64GB 2TB M1Max 10 cores.
 
 <pre>
-1. Rust 1.58      = 0.38s (with sorting: 1.40s)
-2. Golang 1.17.6  = 0.61s (with sorting: 5.03s)
-3. Python 3.10.2  = 2.80s
+1. Golang 1.17.6  = 0.61s (with sorting: 5.03s)
+2. Rust 1.58      = 1.14s (with sorting: 1.62s) with tokyo (previous: 1.34s, with sorting: 1.79)
+3. Python 3.10.2  = 2.80s (with multiprocessing)
 4. Julia 1.7.1    = 4.522
 5. Crystal 1.3.2  = 5.72s
 6. Elixir 1.13.2  = 7.82s
-7. Ruby 3.1.0     = 8.31s
+7. Ruby 3.1.0     = 8.31s (with Parallel)
 </pre>
 
 ### Conclusion
 
-Rust is the fastest language beyond doubt. The new optimized Golang code version is very fast, slower than Rust but faster than other languages. Golang is the only language at the moment with full mature i18n support for arm64/M1 platform.
-
-* Rust = the current example uses [lexical-sort](https://lib.rs/crates/lexical-sort) which is not perfect. [There is no standard mature implementation of i18n in Rust](https://www.arewewebyet.org/topics/i18n/) at the moment. 
+The new optimized Golang code version is very fast, slower than Rust but faster than other languages. Golang is the only language at the moment with full mature i18n support for arm64/M1 platform.
 
+* Rust = the current example uses [lexical-sort](https://lib.rs/crates/lexical-sort) which is not perfect. [There is no standard mature implementation of i18n in Rust](https://www.arewewebyet.org/topics/i18n/) at the moment.
 
 * Python = has a great implementation of [ICU](https://icu.unicode.org/related) library however it does not support arm64/M1 platform, hence I couldn't use it in this comparison.
 
-
 * Ruby = same as Python, no ICU for M1.
 
-
 * Elixir = same as Python, no ICU for M1.
 
-
 * Julia = I couldn't find a good i18 library supporting many languages.
 
-
 * Crystal = currently supports only Turkish collations. Probably because the language is young and does not have a large enough community or company behind it.
 
-
 * Golang = has rules for many languages. You can see the influence of a large company and community which makes Golang a mature solution. Sorting slowed the whole task down significantly, but the result is correct (in this case I only checked the results for the Polish language)
 
 ### Kudos
 
 [@romanatnews](https://github.com/romanatnews) (Golang example refactoring)
+
+[@pan93412](https://github.com/pan93412) (Rust example refactoring using Tokyo runtime)

example-rust/src/main.rs

@@ -1,62 +1,95 @@
-use crossbeam_utils::sync::WaitGroup;
 use glob::glob;
+use itertools::Itertools;
 use lexical_sort::{natural_lexical_cmp, StringSort};
+use once_cell::sync::Lazy;
 use regex::Regex;
-use std::collections::HashSet;
-use std::fs;
-use std::thread;
-use time::Instant;
+use std::path::Path;
+use tokio::fs;
 use yaml_rust::YamlLoader;
 
-fn main() -> std::io::Result<()> {
-    let start = Instant::now();
-    let with_sorting = false;
-    let outdir = "words";
-    fs::create_dir_all(outdir)?;
-    let wg = WaitGroup::new();
-    let path = "../data/??/**/*.yml";
-    for entry in glob(path).expect("Failed to read glob pattern") {
-        match entry {
-            Ok(path) => {
-                // let separator = Regex::new(r"[^\p{L}]+").unwrap();
-                let separator = Regex::new(r"[\W\d]+").unwrap();
-                let wg = wg.clone();
-                thread::spawn(move || {
-                    let filepath = path.to_str().unwrap().replace(".yml", ".txt");
-                    // println!("{:?}", filepath);
-                    let text = fs::read_to_string(&filepath)
-                        .unwrap()
-                        .to_lowercase()
-                        .replace("\n", " ");
-                    let tokens: Vec<&str> = separator.split(&text).collect();
-                    let unique_tokens: HashSet<&str> = tokens.into_iter().collect();
-                    let mut words: Vec<&str>;
-                    if with_sorting {
-                        words = unique_tokens.into_iter().collect();
-                        words.string_sort_unstable(natural_lexical_cmp);
-                    } else {
-                        words = unique_tokens.into_iter().collect();
-                    }
-                    let yaml = fs::read_to_string(&path).unwrap();
-                    let docs = YamlLoader::load_from_str(&yaml).unwrap();
-                    let meta = &docs[0];
-                    let out = format!(
-                        "{}/{}-{}.txt",
-                        outdir,
-                        meta["lang"].as_str().unwrap(),
-                        meta["code"].as_str().unwrap()
-                    );
-                    if let Err(e) = fs::write(out, words.join("\n")) {
-                        println!("Writing error: {}", e.to_string());   
-                    }
-                    drop(wg);
-                });
-            }
-            Err(e) => println!("{:?}", e),
-        }
+const SORT: bool = true;
+const OUTDIR: &str = "words";
+const FILE_DIR: &str = "../data/??/**/*.yml";
+static SEPARATOR_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(r"[\W\d]+").unwrap());
+
+async fn create_outdir() -> tokio::io::Result<()> {
+    fs::create_dir_all(OUTDIR).await
+}
+
+async fn read_file(path: &Path) -> String {
+    let raw = fs::read_to_string(path).await.unwrap();
+    raw.to_lowercase().replace('\n', " ")
+}
+
+fn get_unique_token(src: &str) -> Vec<&str> {
+    let mut data = SEPARATOR_REGEX.split(src).unique().collect::<Vec<_>>();
+
+    if SORT {
+        data.string_sort_unstable(natural_lexical_cmp);
     }
-    wg.wait();
-    let end = Instant::now();
-    println!("{:?} seconds.", end - start);
-    Ok(())
+
+    data
+}
+
+async fn get_filename_from_meta(path: &Path) -> anyhow::Result<String> {
+    let yaml = fs::read_to_string(path).await?;
+    let docs = YamlLoader::load_from_str(&yaml)?;
+    let meta = &docs[0];
+
+    let lang = meta["lang"]
+        .as_str()
+        .ok_or_else(|| anyhow::anyhow!("code not found"))?;
+
+    let code = meta["code"]
+        .as_str()
+        .ok_or_else(|| anyhow::anyhow!("code not found"))?;
+
+    Ok(format!("{}/{}-{}.txt", OUTDIR, lang, code))
 }
+
+#[tokio::main]
+async fn main() -> std::io::Result<()> {
+    let start = std::time::Instant::now();
+    let path = glob(FILE_DIR).expect("failed to read glob pattern");
+
+    let submissions = path.map(|entry| {
+        tokio::spawn(async {
+            let yaml_path = entry.expect("should be existed");
+            let txt_path = yaml_path.with_extension("txt");
+
+            let outdir_submission =
+                tokio::spawn(async { create_outdir().await.expect("unable to create outdir") });
+
+            let read_text_file_submission = tokio::spawn(async move {
+                let data = read_file(&txt_path).await;
+                let tokens = get_unique_token(&data);
+
+                tokens.join("\n")
+            });
+
+            let filename_submission = tokio::spawn(async move {
+                get_filename_from_meta(&yaml_path)
+                    .await
+                    .expect("should be existed")
+            });
+
+            let (tokens, filename, _) = tokio::join!(
+                read_text_file_submission,
+                filename_submission,
+                outdir_submission,
+            );
+
+            fs::write(
+                filename.expect("failed to run filename"),
+                tokens.expect("failed to get tokens"),
+            )
+            .await
+            .expect("failed to write");
+        })
+    });
+
+    futures::future::join_all(submissions).await;
+
+    println!("{:?}", start.elapsed());
+    Ok(())
+}