Merge branch 'm1'

# Conflicts:
#	README.md
#	words_extractor_go/main.go

Author: hipertracker

.gitignore

@@ -1,6 +1,10 @@
 /.history/
 /.vscode/
-/.DS_Store
+/**/.DS_Store
 
 /words_extractor_pypy/
 /.elixir_ls/
+/.idea/
+/old/
+/**/.mypy_cache
+/**/*.pyc

README.md

@@ -1,78 +1,22 @@
 # words_extractor
 
-Example of text file parsing in Python, Golang, Elixir, Rust, Crystal and Julia
-
-Text source: 79.4MB in 30 files
-
-- Rust 1.51.0 (parallel) with sorting: 0.91s, without sorting: 0.49s
-- Python 3.9.5 (parallel) with sorting: 3.42s, without sorting: 2.47
-- Crystal 1.0.0 (parallel) with sorting: 5.34s, withouts sorting: 2.48
-- Go 1.16.4 (parallel) with sorting: 6.41s, without sorting: 3.75s
-- Rust 1.51.0 with sorting: 7s, without sorting: 5s (no parallelism)
-- Julia 1.6.1 (8 threads) 8.7s, (1 thread) 9.7s without sorting
-- Python 3.9.5 with sorting: 10s, without sorting 8.32s (no parallelism)
-- Crystal 1.0.0 with sorting: 13s, without sorting: 7s (no parallelism)
-- Go 1.16.4 with sorting: 21s, without sorting: 11s (no parallelism)
-- Elixir 1.12 (parallel) with sorting: 33s (without release build)
-
-macOS 11.3.1, MacBook Pro (Retina, 15-inch, Late 2013)
-
-Python
-
-```bash
-cd words_extractor_py
-python words.py
-```
-
-Rust
-
-```
-cd words_extractor_rs
-cargo build --release
-target/release/words_extractor_rs
-```
-Golang
-
-```
-cd words_extractor_go
-make build
-GOGC=2000 ./main
-
-Crystal
-
-```
-cd words_extractor_cr
-crystal build --release -Dpreview_mt src/fast_words_cr.cr -o main
-CRYSTAL_WORKES=8 ./main
-```
-
-Julia
-
-```
-julia -t 8 src/words_extractor_jl.jl
-```
-
-Elixir
-
-```
-cd words_extractor_ex
-mix run -e "WordsExtractor.run"
-```
-
-## Running Python
-
-1. Install the latest Python 3.9.5
-2. Create venv and dependencies
-
-```bash
-cd words_extractor_py
-python -m venv venv
-source venv/bin/activate
-pip install -r requirements.txt
-```
-
-3. Run the code
-
-```bash
-python words_parallel.py
-```
+|   |   |   |   |   |
+|--- |--- |--- |--- |--- |
+|   |   |   |   |   |
+|   |   |   |   |   |
+|   |   |   |   |   |
+
+Example of a text file parsing in several programming languages
+
+MacOS 12.2
+Rust 1.58.1
+MBP 16" 64GB 2TB M1Max 10 cores
+Tested on 123 files (504MB)
+
+Results:
+
+1. Rust 1.58.1 -> 0.3521 s
+2. Ruby 3.1 with Parallel -> 2.0542 s
+3. Python 3.10.2 with multiprocessing -> 2.9403 s
+4. Crystal 1.3.2 with channels ->  6.0035 s
+5. Go 1.18beta1 with waitgroup -> 7.2166 s

words_extractor_cr/.editorconfig example-crystal/.editorconfigwords_extractor_cr/.editorconfig renamed to example-crystal/.editorconfig

@@ -1,6 +1,6 @@
-The MIT License (MIT)
+MIT License
 
-Copyright (c) 2021 Jaroslaw Zabiello <hipertracker@gmail.com>
+Copyright (c) 2022 Jaroslaw Zabiello <jaroslaw.zabiello@easyfairs.com>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -9,13 +9,13 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 copies of the Software, and to permit persons to whom the Software is
 furnished to do so, subject to the following conditions:
 
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
 
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

words_extractor_cr/LICENSE example-crystal/LICENSEwords_extractor_cr/LICENSE renamed to example-crystal/LICENSE

@@ -0,0 +1,15 @@
+# Crystal
+
+## setup and run
+
+```
+crystal build --release -Dpreview_mt src/example-crystal.cr -o main
+CRYSTAL_WORKES=10 ./main
+```
+
+MacOS 12.2
+Crystal 1.3.2
+MBP 16" M1Max 10 cores
+Total files: 123
+Total size: 504 MB
+Total time: 6.0035 s

example-crystal/README.md

@@ -0,0 +1,13 @@
+name: example-crystal
+version: 0.1.0
+
+authors:
+  - Jaroslaw Zabiello <[email protected]>
+
+targets:
+  example-crystal:
+    main: src/example-crystal.cr
+
+crystal: 1.3.2
+
+license: MIT

words_extractor_cr/shard.lock example-crystal/shard.lockwords_extractor_cr/shard.lock renamed to example-crystal/shard.lock

@@ -1,21 +1,21 @@
 require "./spec_helper"
 
-describe FastWordsCr do
+describe Example::Crystal do
   it "should be understand PL collations" do
     words = %w(ala ąla ćma ciemno ęle element łódź lody śmierć serial źdźbło żółw zawias)
     expected = %w(ala ąla ciemno ćma element ęle lody łódź serial śmierć zawias źdźbło żółw)
-    words.sort { |x, y| FastWordsCr.cmp(x, y) }.should eq(expected)
+    words.sort { |x, y| Example::Crystal.word_cmp(x, y) }.should eq(expected)
   end
 
   it "should be understand PL collations for q" do
     words = %w(ala ąla ćma ciemno ęle element łódź lody śmierć serial źdźbło żółw zawias querty)
     expected = %w(ala ąla ciemno ćma element ęle lody łódź querty serial śmierć zawias źdźbło żółw)
-    words.sort { |x, y| FastWordsCr.cmp(x, y) }.should eq(expected)
+    words.sort { |x, y| Example::Crystal.word_cmp(x, y) }.should eq(expected)
   end
 
   it "should understand PL collations for upper chars" do
     words = %w(ala ąla Ćma ciemno Ęle Element łódź lody śmierć serial źdźbło żółw zawias querty)
     expected = %w(ala ąla ciemno Ćma Element Ęle lody łódź querty serial śmierć zawias źdźbło żółw)
-    words.sort { |x, y| FastWordsCr.cmp(x, y) }.should eq(expected)
+    words.sort { |x, y| Example::Crystal.word_cmp(x.downcase, y.downcase) }.should eq(expected)
   end
 end

example-crystal/shard.yml

@@ -0,0 +1,2 @@
+require "spec"
+require "../src/example-crystal"

…_extractor_cr/spec/fast_words_cr_spec.cr …ple-crystal/spec/example-crystal_spec.crwords_extractor_cr/spec/fast_words_cr_spec.cr renamed to example-crystal/spec/example-crystal_spec.cr words_extractor_cr/spec/fast_words_cr_spec.cr renamed to example-crystal/spec/example-crystal_spec.cr

@@ -3,39 +3,46 @@ require "yaml"
 
 # TODO: Write documentation for `FastWordsCr`
 
-module FastWordsCr
+module Example::Crystal
   VERSION = "0.2.0"
   CHARSET = "aąbcćdeęfghijklłmnńoópqrsśtuvwxyzźż"
 
   def self.main(outpath = "words")
-    with_sorting = true
+    with_sorting = false
     concurrent = true
 
     prepare_folder(outpath, "*.txt")
 
     file_count = 0
+    total_size = 0
 
-    processed_files = Channel(Bool).new
-    Dir.glob("../data/pl/**/*.yml").each do |path|
+    channel = Channel(Tuple(String, Int64)).new
+    srcPath = "../data/??/**/*.yml"
+    # srcPath = "./bibles/??/**/*.yml"
+    Dir.glob(srcPath, follow_symlinks: true).each do |path|
       if concurrent
         spawn do
-          worker(path, outpath, with_sorting)
-          processed_files.send true
+          channel.send worker(path, outpath, with_sorting)
         end
         file_count += 1
       else
         worker(path, outpath, with_sorting)
       end
     end
     if concurrent
-      file_count.times do
-        processed_files.receive
+      file_count.times do |i|
+        path, size = channel.receive
+        total_size += size
+        # puts("[#{i + 1}/#{file_count}] #{path}")
       end
     end
+    total_size = total_size / 1024 / 1024
+    puts("Total size: #{total_size} MB")
   end
 
   def self.worker(path, outpath, with_sorting)
-    text = File.read(path.gsub(".yml", ".txt")).gsub("\n", " ").downcase
+    filepath = path.gsub(".yml", ".txt")
+    text = File.read(filepath).gsub("\n", " ").downcase
 
     words = text.split(/[^\p{L}]+/).to_set
 
@@ -44,9 +51,11 @@ module FastWordsCr
     end
 
     meta = File.open(path) { |file| YAML.parse(file) }
-    filepath = %Q(#{outpath}/extracted-words-for-#{meta["label"]}.txt)
+    filepath = %Q(#{outpath}/#{meta["lang"]}-#{meta["code"]}.txt)
     File.write(filepath, words.join("\n"))
-    puts "Saved #{filepath}"
+    filesize = File.size(filepath)
+    puts([filepath, filesize])
+    {filepath, filesize}
   end
 
   def self.prepare_folder(folder : String, pattern : String)
@@ -70,6 +79,6 @@ module FastWordsCr
 end
 
 elapsed_time = Time.measure do
-  FastWordsCr.main
+  Example::Crystal.main
 end
 puts elapsed_time