Merge branch 'master' into golang-example-refactor

Author: roman-kulish

.gitignore

@@ -12,4 +12,5 @@
 /**/main.dwarf
 /**/main
 /**/target
-/**/.idea
+/**/.idea
+/**/.venv

README.md

@@ -1,26 +1,48 @@
 # words_extractor
 
 ### Info
+
 Example of a text file parsing in several programming languages. The goal is to extract unique words from utf-8 files and save results them into separate files.
 
+The difficulty in sorting words is due to the need to handle sorting rules according to the the different languages grammary. This is quite a complex problem that does not exist for the English language where the character set does not exceed the basic ASCII standard.
+
 ### Results
 
-The following results are for 936 files (2745 MB) on MacOS 12.2 and MacBook Pro 16" 64GB 2TB M1Max 10 cores. (For more text files go into data/pl/* and duplicate files several times.) All examples are using a similar logic and approach.
+The following results are for 123 unique utf-8 Bible text files in 23 languages (used at mybible.pl site) They take 504MB. (The repo contains only a few sample files in the 'data' folder. For testing more data you could multiple files by cloning *.txt (and the associated*.yml) file under different names)
+
+* Platform: MacOS 12.2
+* Machine: MacBook Pro 16" 64GB 2TB M1Max 10 cores.
 
 <pre>
-1. Rust v1.58.1      =  7.54s
-2. Python v3.10.2    = 15.34s (with multiprocessing)
-3. Julia v1.7.1      = 17.00s
-4. Crystal v1.3.2    = 26.32s 
-5. Ruby v3.1.0       = 40.94s (with Parallel)
-6. Golang v1.18beta1 = 73.00s
-7. Elixir v1.13.2    = 2m43s 
+1. Rust 1.58      = 1.14s (with sorting: 1.59s) with tokyo (previous: 1.34s, with sorting: 1.79)
+2. Golang 1.17.6  = 1.34s (with sorting: 6.55s)
+3. Python 3.10.2  = 2.80s (with multiprocessing)
+4. Julia 1.7.1    = 4.522
+5. Crystal 1.3.2  = 5.72s
+6. Elixir 1.13.2  = 7.82s
+7. Ruby 3.1.0     = 8.31s (with Parallel)
 </pre>
 
 ### Conclusion
 
-Rust is the fastest language beyond doubt.
+The new optimized Golang code version is very fast, slower than Rust but faster than other languages. Golang is the only language at the moment with full mature i18n support for arm64/M1 platform.
+
+* Rust = the current example uses [lexical-sort](https://lib.rs/crates/lexical-sort) which is not perfect. [There is no standard mature implementation of i18n in Rust](https://www.arewewebyet.org/topics/i18n/) at the moment.
+
+* Python = has a great implementation of [ICU](https://icu.unicode.org/related) library however it does not support arm64/M1 platform, hence I couldn't use it in this comparison.
+
+* Ruby = same as Python, no ICU for M1.
+
+* Elixir = same as Python, no ICU for M1.
+
+* Julia = I couldn't find a good i18 library supporting many languages.
+
+* Crystal = currently supports only Turkish collations. Probably because the language is young and does not have a large enough community or company behind it.
+
+* Golang = has rules for many languages. You can see the influence of a large company and community which makes Golang a mature solution. Sorting slowed the whole task down significantly, but the result is correct (in this case I only checked the results for the Polish language)
+
+### Kudos
 
-What is surprised is pretty poor Golang's performance on this task. Crystal is faster than Golang but in this task it is still slower than Python which is also surprising. (Neither Golang nor Crystal is my main field of expertise so maybe there is some room for improvement. Although I showed this code to people and nobody so far could improve it in any significant way. But if I find a better implementation I will update this comparison.)
+[@romanatnews](https://github.com/romanatnews) (Golang example refactoring)
 
-The high Python performance is interesting. Although it is using a multiprocessing standard library for full CPU cores utilization this is still dynamic interpreted language after all, which is rather expected to be slower than statically typed languages. 
+[@pan93412](https://github.com/pan93412) (Rust example refactoring using Tokyo runtime)

example-crystal/src/example-crystal.cr

@@ -1,47 +1,46 @@
 require "json"
 require "yaml"
 
-# TODO: Write documentation for `FastWordsCr`
-
 module Example::Crystal
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
   CHARSET = "aąbcćdeęfghijklłmnńoópqrsśtuvwxyzźż"
 
-  def self.main(outpath = "words")
+  def self.main(outdir = "words")
     with_sorting = false
     concurrent = true
 
-    prepare_folder(outpath, "*.txt")
+    prepare_folder(outdir, "*.txt")
 
     file_count = 0
     total_size = 0
 
     channel = Channel(Tuple(String, Int64)).new
     srcPath = "../data/??/**/*.yml"
-    # srcPath = "./bibles/??/**/*.yml"
-    Dir.glob(srcPath, follow_symlinks: true).each do |path|
+    paths = Dir.glob(srcPath, follow_symlinks: true)
+    count = paths.size
+    paths.each do |path|
       if concurrent
         spawn do
-          channel.send worker(path, outpath, with_sorting)
+          channel.send worker(path, outdir, with_sorting)
         end
         file_count += 1
       else
-        worker(path, outpath, with_sorting)
+        worker(path, outdir, with_sorting)
       end
     end
     if concurrent
       file_count.times do |i|
         path, size = channel.receive
         total_size += size
-        # puts("[#{i + 1}/#{file_count}] #{path}")
+        puts(::sprintf("[%3d/%d] %s", i + 1, file_count, path))
       end
     end
-    total_size = total_size / 1024 / 1024
-    puts("Total size: #{total_size} MB")
+    puts("Total size: #{(total_size / 1024 / 1024).round} MB")
   end
 
-  def self.worker(path, outpath, with_sorting)
+  def self.worker(path, outdir, with_sorting)
     filepath = path.gsub(".yml", ".txt")
+    filesize = File.size(filepath)
     text = File.read(filepath).gsub("\n", " ").downcase
 
     words = text.split(/[^\p{L}]+/).to_set
@@ -51,10 +50,9 @@ module Example::Crystal
     end
 
     meta = File.open(path) { |file| YAML.parse(file) }
-    filepath = %Q(#{outpath}/#{meta["lang"]}-#{meta["code"]}.txt)
-    File.write(filepath, words.join("\n"))
+    outfilepath = %Q(#{outdir}/#{meta["lang"]}-#{meta["code"]}.txt)
+    File.write(outfilepath, words.join("\n"))
     filesize = File.size(filepath)
-    puts([filepath, filesize])
     {filepath, filesize}
   end
 
@@ -78,7 +76,8 @@ module Example::Crystal
   end
 end
 
-elapsed_time = Time.measure do
+elapsed = Time.measure do
   Example::Crystal.main
 end
-puts elapsed_time
+
+puts("Total time: #{elapsed}")

example-elixir/lib/words_extractor.ex

@@ -1,47 +1,57 @@
+use Timex
+
 defmodule WordsExtractor do
   @moduledoc nil
 
+  @pat Regex.compile!("[\W\d]+/u")
+
   def run do
     outdir = "words"
-    clean_dir(outdir)
-
-    walk("../data/pl/", ".yml")
-    |> Task.async_stream(
-      WordsExtractor,
-      :worker,
-      [outdir],
-      ordered: false,
-      timeout: :infinity
-    )
-    |> Enum.to_list()
+    File.rm_rf!(outdir)
+    File.mkdir!(outdir)
+    t = Duration.now()
+    paths = Path.wildcard("../data/??/**/*.yml")
+    count = length(paths)
+
+    total_size =
+      paths
+      |> Enum.with_index(1)
+      |> Task.async_stream(
+        WordsExtractor,
+        :worker,
+        [outdir, count],
+        ordered: false,
+        timeout: :infinity
+      )
+      |> Enum.to_list()
+      |> Enum.reduce(0, fn {:ok, num}, acc -> acc + num end)
+
+    elapsed = Duration.diff(Duration.now(), t, :milliseconds)
+    IO.puts("Total time: #{elapsed / 1000}s")
+    IO.puts("Total size: #{(total_size / 1024 / 1024) |> round} MB")
   end
 
-  def clean_dir(path) do
-    File.rm_rf!(path)
-    File.mkdir!(path)
-  end
+  def worker({path, i}, outdir, count) do
+    %{"code" => code, "lang" => lang} = YamlElixir.read_from_file!(path)
 
-  def worker(path, outdir) do
-    %{"code" => code} = YamlElixir.read_from_file!(path)
+    filepath = String.replace(path, ".yml", ".txt")
+    %File.Stat{:size => filesize} = File.stat!(filepath)
 
-    words =
-      File.read!(String.replace(path, ".yml", ".txt"))
+    content =
+      File.read!(filepath)
       |> String.downcase()
       |> String.trim()
-      |> then(&Regex.split(~r/[\W\d]+/u, &1))
+      |> then(&Regex.split(@pat, &1))
       |> MapSet.new()
-      # sorting does not respect collation
-      |> Enum.sort()
+      |> Enum.join("\n")
 
-    File.write!("#{outdir}/extracted-#{code}.txt", Enum.join(words, "\n"))
-    IO.puts(path)
-  end
+    # sorting does not respect collation so it is ignored
+    # |> Enum.sort()
 
-  def walk(path, pattern) do
-    dir = String.to_charlist(path)
-    regexp = String.to_charlist(pattern)
+    it = i |> Integer.to_string() |> String.pad_leading(3)
+    IO.puts("[#{it}/#{count}] #{path}/#{lang}-#{code}.txt")
 
-    :filelib.fold_files(dir, regexp, true, fn file, acc -> [file | acc] end, [])
-    |> Enum.map(fn filepath -> to_string(filepath) end)
+    File.write!("./#{outdir}/#{lang}-#{code}.txt", content)
+    filesize
   end
 end

example-elixir/mix.exs

@@ -4,7 +4,7 @@ defmodule WordsExtractor.MixProject do
   def project do
     [
       app: :words_extractor_ex,
-      version: "0.1.0",
+      version: "0.2.0",
       elixir: "~> 1.13",
       start_permanent: Mix.env() == :prod,
       deps: deps()
@@ -21,7 +21,8 @@ defmodule WordsExtractor.MixProject do
   # Run "mix help deps" to learn about dependencies.
   defp deps do
     [
-      {:yaml_elixir, "~> 2.8"}
+      {:yaml_elixir, "~> 2.8"},
+      {:timex, "~> 3.7"}
       # {:dep_from_hexpm, "~> 0.3.0"},
       # {:dep_from_git, git: "https://github.com/elixir-lang/my_dep.git", tag: "0.1.0"}
     ]

example-elixir/mix.lock

@@ -1,4 +1,16 @@
 %{
+  "certifi": {:hex, :certifi, "2.8.0", "d4fb0a6bb20b7c9c3643e22507e42f356ac090a1dcea9ab99e27e0376d695eba", [:rebar3], [], "hexpm", "6ac7efc1c6f8600b08d625292d4bbf584e14847ce1b6b5c44d983d273e1097ea"},
+  "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"},
+  "gettext": {:hex, :gettext, "0.19.1", "564953fd21f29358e68b91634799d9d26989f8d039d7512622efb3c3b1c97892", [:mix], [], "hexpm", "10c656c0912b8299adba9b061c06947511e3f109ab0d18b44a866a4498e77222"},
+  "hackney": {:hex, :hackney, "1.18.0", "c4443d960bb9fba6d01161d01cd81173089686717d9490e5d3606644c48d121f", [:rebar3], [{:certifi, "~>2.8.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "9afcda620704d720db8c6a3123e9848d09c87586dc1c10479c42627b905b5c5e"},
+  "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},
+  "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"},
+  "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"},
+  "parse_trans": {:hex, :parse_trans, "3.3.1", "16328ab840cc09919bd10dab29e431da3af9e9e7e7e6f0089dd5a2d2820011d8", [:rebar3], [], "hexpm", "07cd9577885f56362d414e8c4c4e6bdf10d43a8767abb92d24cbe8b24c54888b"},
+  "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"},
+  "timex": {:hex, :timex, "3.7.6", "502d2347ec550e77fdf419bc12d15bdccd31266bb7d925b30bf478268098282f", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 1.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "a296327f79cb1ec795b896698c56e662ed7210cc9eb31f0ab365eb3a62e2c589"},
+  "tzdata": {:hex, :tzdata, "1.1.1", "20c8043476dfda8504952d00adac41c6eda23912278add38edc140ae0c5bcc46", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "a69cec8352eafcd2e198dea28a34113b60fdc6cb57eb5ad65c10292a6ba89787"},
+  "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"},
   "yamerl": {:hex, :yamerl, "0.10.0", "4ff81fee2f1f6a46f1700c0d880b24d193ddb74bd14ef42cb0bcf46e81ef2f8e", [:rebar3], [], "hexpm", "346adb2963f1051dc837a2364e4acf6eb7d80097c0f53cbdc3046ec8ec4b4e6e"},
   "yaml_elixir": {:hex, :yaml_elixir, "2.8.0", "c7ff0034daf57279c2ce902788ce6fdb2445532eb4317e8df4b044209fae6832", [:mix], [{:yamerl, "~> 0.8", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "4b674bd881e373d1ac6a790c64b2ecb69d1fd612c2af3b22de1619c15473830b"},
 }

example-golang/.gitignore

@@ -0,0 +1 @@
+/coverage.out

example-golang/Makefile

@@ -8,14 +8,22 @@ build:
 run: build
 	./${BINARY_NAME}
 
+run-sort: build
+	./${BINARY_NAME} -n 10 -s
+
 test:
+	@go test ./... -v
+
+coverage:
 	@go test ./... -v -coverprofile=coverage.out  
-	
+
+
 cover: test
 	@go tool cover -html=coverage.out
 
 clean:
 	@go clean
 	rm -f coverage.out 
 	rm -f ./${BINARY_NAME}
-	rm -rf ./words
+	rm -rf ./words
+

example-golang/README.md

@@ -4,5 +4,12 @@
 
 ```
 make build
-./main [-n=NUMBER_OF_WORKERS, integer] [-s] 
+./main -n 8
 ```
+
+<pre>
+Usage of ./main:
+  -n int
+     Number of workers to run (zero to match the number of available CPUs) (default 10)
+  -s Sort results
+</pre>

example-golang/app/app.go

@@ -5,6 +5,8 @@ import (
 	"os"
 	"path/filepath"
 	"sync"
+
+	"github.com/bmatcuk/doublestar"
 )
 
 const dirPerms = 0755
@@ -15,7 +17,7 @@ type empty struct{}
 // No error handling, no context cancellation is implemented to match implementations
 // in other languages.
 func Run(srcDir, outDir string, numWorkers int, sortResults bool) error {
-	files, err := filepath.Glob(srcDir)
+	files, err := doublestar.Glob(srcDir)
 	if err != nil {
 		return fmt.Errorf(`app: getting list of files "%s": %w`, srcDir, err)
 	}
@@ -41,7 +43,7 @@ func Run(srcDir, outDir string, numWorkers int, sortResults bool) error {
 		}
 
 		src := file[:len(file)-3] + "txt"
-		dst := filepath.Join(outDir, "extracted-words-for-"+spec.Code+".txt")
+		dst := filepath.Join(outDir, spec.Lang+"-"+spec.Code+".txt")
 
 		wg.Add(1)
 		go extract(src, dst, sortResults, spec.Tag, sem, &wg)