Merge branch 'master' of github.com:hipertracker/words_extractor

Author: pan93412

.gitignore

@@ -12,4 +12,5 @@
 /**/main.dwarf
 /**/main
 /**/target
-/**/.idea
+/**/.idea
+/**/.venv

README.md

@@ -1,26 +1,29 @@
 # words_extractor
 
 ### Info
+
 Example of a text file parsing in several programming languages. The goal is to extract unique words from utf-8 files and save results them into separate files.
 
 ### Results
 
-The following results are for 936 files (2745 MB) on MacOS 12.2 and MacBook Pro 16" 64GB 2TB M1Max 10 cores. (For more text files go into data/pl/* and duplicate files several times.) All examples are using a similar logic and approach.
+The following results are for 123 unique utf-8 Bible text files in 23 languages (used at mybible.pl site) They take 504MB. (The repo contains only a few sample files in the 'data' folder. For testing more data you could multiple files by cloning *.txt (and the associated*.yml) file under different names)
+
+* Platform: MacOS 12.2
+* Machine: MacBook Pro 16" 64GB 2TB M1Max 10 cores.
 
 <pre>
-1. Rust v1.58.1      =  7.54s
-2. Python v3.10.2    = 15.34s (with multiprocessing)
-3. Julia v1.7.1      = 17.00s
-4. Crystal v1.3.2    = 26.32s 
-5. Ruby v3.1.0       = 40.94s (with Parallel)
-6. Golang v1.18beta1 = 73.00s
-7. Elixir v1.13.2    = 2m43s 
+1. Rust 1.58      = 0.38s
+2. Python 3.10.2  = 2.80s
+3. Julia 1.7.1    = 4.522
+4. Crystal 1.3.2  = 5.72s
+5. Elixir 1.13.2  = 8.37s
+6. Ruby 3.1.0     = 8.31s
+
+Golang 1.17    = UNDER REFACTORING, stay tuned
 </pre>
 
 ### Conclusion
 
 Rust is the fastest language beyond doubt.
 
-What is surprised is pretty poor Golang's performance on this task. Crystal is faster than Golang but in this task it is still slower than Python which is also surprising. (Neither Golang nor Crystal is my main field of expertise so maybe there is some room for improvement. Although I showed this code to people and nobody so far could improve it in any significant way. But if I find a better implementation I will update this comparison.)
-
-The high Python performance is interesting. Although it is using a multiprocessing standard library for full CPU cores utilization this is still dynamic interpreted language after all, which is rather expected to be slower than statically typed languages. 
+The high Python performance is interesting. Although it is using a multiprocessing standard library for full CPU cores utilization this is still dynamic interpreted language after all, which is rather expected to be slower than statically typed languages.

example-crystal/src/example-crystal.cr

@@ -1,47 +1,46 @@
 require "json"
 require "yaml"
 
-# TODO: Write documentation for `FastWordsCr`
-
 module Example::Crystal
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
   CHARSET = "aąbcćdeęfghijklłmnńoópqrsśtuvwxyzźż"
 
-  def self.main(outpath = "words")
+  def self.main(outdir = "words")
     with_sorting = false
     concurrent = true
 
-    prepare_folder(outpath, "*.txt")
+    prepare_folder(outdir, "*.txt")
 
     file_count = 0
     total_size = 0
 
     channel = Channel(Tuple(String, Int64)).new
     srcPath = "../data/??/**/*.yml"
-    # srcPath = "./bibles/??/**/*.yml"
-    Dir.glob(srcPath, follow_symlinks: true).each do |path|
+    paths = Dir.glob(srcPath, follow_symlinks: true)
+    count = paths.size
+    paths.each do |path|
       if concurrent
         spawn do
-          channel.send worker(path, outpath, with_sorting)
+          channel.send worker(path, outdir, with_sorting)
         end
         file_count += 1
       else
-        worker(path, outpath, with_sorting)
+        worker(path, outdir, with_sorting)
       end
     end
     if concurrent
       file_count.times do |i|
         path, size = channel.receive
         total_size += size
-        # puts("[#{i + 1}/#{file_count}] #{path}")
+        puts(::sprintf("[%3d/%d] %s", i + 1, file_count, path))
       end
     end
-    total_size = total_size / 1024 / 1024
-    puts("Total size: #{total_size} MB")
+    puts("Total size: #{(total_size / 1024 / 1024).round} MB")
   end
 
-  def self.worker(path, outpath, with_sorting)
+  def self.worker(path, outdir, with_sorting)
     filepath = path.gsub(".yml", ".txt")
+    filesize = File.size(filepath)
     text = File.read(filepath).gsub("\n", " ").downcase
 
     words = text.split(/[^\p{L}]+/).to_set
@@ -51,10 +50,9 @@ module Example::Crystal
     end
 
     meta = File.open(path) { |file| YAML.parse(file) }
-    filepath = %Q(#{outpath}/#{meta["lang"]}-#{meta["code"]}.txt)
-    File.write(filepath, words.join("\n"))
+    outfilepath = %Q(#{outdir}/#{meta["lang"]}-#{meta["code"]}.txt)
+    File.write(outfilepath, words.join("\n"))
     filesize = File.size(filepath)
-    puts([filepath, filesize])
     {filepath, filesize}
   end
 
@@ -78,7 +76,8 @@ module Example::Crystal
   end
 end
 
-elapsed_time = Time.measure do
+elapsed = Time.measure do
   Example::Crystal.main
 end
-puts elapsed_time
+
+puts("Total time: #{elapsed}")

example-elixir/lib/words_extractor.ex

@@ -1,47 +1,56 @@
+use Timex
+
 defmodule WordsExtractor do
   @moduledoc nil
 
   def run do
     outdir = "words"
-    clean_dir(outdir)
-
-    walk("../data/pl/", ".yml")
-    |> Task.async_stream(
-      WordsExtractor,
-      :worker,
-      [outdir],
-      ordered: false,
-      timeout: :infinity
-    )
-    |> Enum.to_list()
+    File.rm_rf!(outdir)
+    File.mkdir!(outdir)
+    t = Duration.now()
+    paths = Path.wildcard("../data/??/**/*.yml")
+    count = length(paths)
+
+    total_size =
+      paths
+      |> Enum.with_index(1)
+      |> Task.async_stream(
+        WordsExtractor,
+        :worker,
+        [outdir, count],
+        ordered: false,
+        timeout: :infinity
+      )
+      |> Enum.to_list()
+      |> Enum.reduce(0, fn {:ok, num}, acc -> acc + num end)
+
+    elapsed = Duration.diff(Duration.now(), t, :milliseconds)
+    IO.puts("Total time: #{elapsed / 1000}s")
+    IO.puts("Total size: #{(total_size / 1024 / 1024) |> round} MB")
   end
 
-  def clean_dir(path) do
-    File.rm_rf!(path)
-    File.mkdir!(path)
-  end
+  def worker({path, i}, outdir, count) do
+    %{"code" => code, "lang" => lang} = YamlElixir.read_from_file!(path)
+    pat = Regex.compile!("[\W\d]+/u")
 
-  def worker(path, outdir) do
-    %{"code" => code} = YamlElixir.read_from_file!(path)
+    filepath = String.replace(path, ".yml", ".txt")
+    %File.Stat{:size => filesize} = File.stat!(filepath)
 
-    words =
-      File.read!(String.replace(path, ".yml", ".txt"))
+    content =
+      File.read!(filepath)
       |> String.downcase()
       |> String.trim()
-      |> then(&Regex.split(~r/[\W\d]+/u, &1))
+      |> then(&Regex.split(pat, &1))
       |> MapSet.new()
-      # sorting does not respect collation
-      |> Enum.sort()
+      |> Enum.join("\n")
 
-    File.write!("#{outdir}/extracted-#{code}.txt", Enum.join(words, "\n"))
-    IO.puts(path)
-  end
+    # sorting does not respect collation so it is ignored
+    # |> Enum.sort()
 
-  def walk(path, pattern) do
-    dir = String.to_charlist(path)
-    regexp = String.to_charlist(pattern)
+    it = i |> Integer.to_string() |> String.pad_leading(3)
+    IO.puts("[#{it}/#{count}] #{path}/#{lang}-#{code}.txt")
 
-    :filelib.fold_files(dir, regexp, true, fn file, acc -> [file | acc] end, [])
-    |> Enum.map(fn filepath -> to_string(filepath) end)
+    File.write!("./#{outdir}/#{lang}-#{code}.txt", content)
+    filesize
   end
 end

example-elixir/mix.exs

@@ -4,7 +4,7 @@ defmodule WordsExtractor.MixProject do
   def project do
     [
       app: :words_extractor_ex,
-      version: "0.1.0",
+      version: "0.2.0",
       elixir: "~> 1.13",
       start_permanent: Mix.env() == :prod,
       deps: deps()
@@ -21,7 +21,8 @@ defmodule WordsExtractor.MixProject do
   # Run "mix help deps" to learn about dependencies.
   defp deps do
     [
-      {:yaml_elixir, "~> 2.8"}
+      {:yaml_elixir, "~> 2.8"},
+      {:timex, "~> 3.7"}
       # {:dep_from_hexpm, "~> 0.3.0"},
       # {:dep_from_git, git: "https://github.com/elixir-lang/my_dep.git", tag: "0.1.0"}
     ]

example-elixir/mix.lock

@@ -1,4 +1,16 @@
 %{
+  "certifi": {:hex, :certifi, "2.8.0", "d4fb0a6bb20b7c9c3643e22507e42f356ac090a1dcea9ab99e27e0376d695eba", [:rebar3], [], "hexpm", "6ac7efc1c6f8600b08d625292d4bbf584e14847ce1b6b5c44d983d273e1097ea"},
+  "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"},
+  "gettext": {:hex, :gettext, "0.19.1", "564953fd21f29358e68b91634799d9d26989f8d039d7512622efb3c3b1c97892", [:mix], [], "hexpm", "10c656c0912b8299adba9b061c06947511e3f109ab0d18b44a866a4498e77222"},
+  "hackney": {:hex, :hackney, "1.18.0", "c4443d960bb9fba6d01161d01cd81173089686717d9490e5d3606644c48d121f", [:rebar3], [{:certifi, "~>2.8.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "9afcda620704d720db8c6a3123e9848d09c87586dc1c10479c42627b905b5c5e"},
+  "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},
+  "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"},
+  "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"},
+  "parse_trans": {:hex, :parse_trans, "3.3.1", "16328ab840cc09919bd10dab29e431da3af9e9e7e7e6f0089dd5a2d2820011d8", [:rebar3], [], "hexpm", "07cd9577885f56362d414e8c4c4e6bdf10d43a8767abb92d24cbe8b24c54888b"},
+  "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"},
+  "timex": {:hex, :timex, "3.7.6", "502d2347ec550e77fdf419bc12d15bdccd31266bb7d925b30bf478268098282f", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 1.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "a296327f79cb1ec795b896698c56e662ed7210cc9eb31f0ab365eb3a62e2c589"},
+  "tzdata": {:hex, :tzdata, "1.1.1", "20c8043476dfda8504952d00adac41c6eda23912278add38edc140ae0c5bcc46", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "a69cec8352eafcd2e198dea28a34113b60fdc6cb57eb5ad65c10292a6ba89787"},
+  "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"},
   "yamerl": {:hex, :yamerl, "0.10.0", "4ff81fee2f1f6a46f1700c0d880b24d193ddb74bd14ef42cb0bcf46e81ef2f8e", [:rebar3], [], "hexpm", "346adb2963f1051dc837a2364e4acf6eb7d80097c0f53cbdc3046ec8ec4b4e6e"},
   "yaml_elixir": {:hex, :yaml_elixir, "2.8.0", "c7ff0034daf57279c2ce902788ce6fdb2445532eb4317e8df4b044209fae6832", [:mix], [{:yamerl, "~> 0.8", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "4b674bd881e373d1ac6a790c64b2ecb69d1fd612c2af3b22de1619c15473830b"},
 }

example-julia/src/words.jl

@@ -2,14 +2,15 @@ module words_extractor_jl
 
 using Distributed
 using YAML
+using Glob
 
-const folder = "words"
+const outdir = "words"
 
-function worker(yaml_path)
+function worker(yaml_path, i, count)
     path = get_filepath(yaml_path)
     words = get_words(yaml_path)
     write(path, join(words, "\n"))
-#     println(string("Saved...", path))
+    println("[$(lpad(i, 3, ' '))/$count] $path")
 end
 
 function get_words(yaml_path)
@@ -20,35 +21,37 @@ end
 
 function get_filepath(path)
     meta = YAML.load_file(path)
-    string(folder, "/extracted-words-for-", meta["label"], ".txt")
+    """./$outdir/$(meta["lang"])-$(meta["code"]).txt"""
 end
 
-function walk(path, file_ext)
-    res = []
-    for (root, _, files) in walkdir(path, topdown = true)
-        for file in files
-            if endswith(file, file_ext)
-                filepath = joinpath(root, file)
-                push!(res, filepath)
-            end
-        end
+function rdir(dir::AbstractString, pat::Glob.FilenameMatch)
+    result = String[]
+    for (root, dirs, files) in walkdir(dir)
+        filepaths = joinpath.(root, files)
+        append!(result, filter!(f -> occursin(pat, f), filepaths))
     end
-    res
+    return result
 end
 
+rdir(dir::AbstractString, pat::AbstractString) = rdir(dir, Glob.FilenameMatch(pat))
+
 function main()
-    if ispath(folder)
-        rm(folder, recursive = true)
+    if ispath(outdir)
+        rm(outdir, recursive = true)
     end
-    mkdir(folder)
-    Threads.@threads for path in walk("../data/pl/", ".yml")
-#         println("Spawn $path")
-        worker(path)
+    mkdir(outdir)
+    paths = rdir("../data", fn"../data/??/*.yml")
+    count = length(paths)
+    i = 1
+    Threads.@threads for path in paths
+        # println("Spawn $path")
+        worker(path, i, count)
+        i += 1
     end
 end
 
-# addprocs()
-# println(string("Workers ", nworkers()))
+addprocs()
+println(string("Workers ", nworkers()))
 println(string("Processing... using ", Threads.nthreads(), " threads"))
 @time main()
 end # module

example-python/words.py

@@ -51,7 +51,6 @@ def worker(path: str, outdir: str, sorting: bool = False) -> Tuple[str, int]:
 
     pool = mp.Pool(mp.cpu_count())
 
-    print("Processing")
     results = []
     paths = glob.glob(src_path, recursive=True)
     if not paths:

example-ruby/words.rb

@@ -1,29 +1,31 @@
 require 'yaml'
+require 'yaml'
 require 'parallel'
 require 'etc'
 require 'fileutils'
 
 outdir = 'words'
-start = Time.now
 
 FileUtils.rm_rf(outdir)
 Dir.mkdir(outdir)
 
+t = Time.now
+
 sorted = false
 
-paths = Dir['../data/pl/**/*.yml']
+paths = Dir['../data/??/**/*.yml']
+count = paths.count
 
-Parallel.each(paths, in_processes: Etc.nprocessors) do |yaml_path| 
+sizes = Parallel.map_with_index(paths, in_processes: Etc.nprocessors) do |yaml_path, i|
   meta = YAML.load_file(yaml_path)
-  words = IO.read(yaml_path.gsub('.yml', '.txt')).downcase.strip.split(/[^\p{word}]+/).uniq
-  if sorted
-    words = words.sort
-  end
+  filepath = yaml_path.gsub('.yml', '.txt')
+  words = IO.read(filepath).downcase.strip.split(/[^\p{word}]+/).uniq
+  words = words.sort if sorted
   outpath = "#{outdir}/#{meta['lang']}-#{meta['code']}.txt"
-  puts outpath
+  puts(format('[%3d/%d] %s/%s', i, count, yaml_path, outpath))
   File.write(outpath, words.join("\n"))
+  File.size(filepath)
 end
 
-secs = Time.now - start
-puts "Total time: #{secs} s"
-
+puts "Total size: #{(sizes.sum / 1024.0 / 1024).round} MB"
+puts "Total time: #{Time.now - t} s"

example-rust/README.md

@@ -7,10 +7,3 @@ cargo build --release
 target/release/words_extractor_rs
 
 ```
-
-MacOS 12.2
-Rust 1.58.1
-MBP 16" M1Max 10 cores
-Total files: 123
-Total size: 504 MB
-Total time: 0.3521 s