update Ruby 3.3.0

Author: hipertracker

README.md

@@ -14,13 +14,13 @@ The following results are for 123 unique utf-8 Bible text files in 23 languages
 * Machine: MacBook Pro 16" 64GB 2TB M1Max 10 cores.
 
 <pre>
-1. Rust 1.63           = 1.15s, with sorting: 1.64s
-2. Golang 1.19.1       = 1.38s, with sorting: 1.71s
+1. Rust 1.63           = 1.15s, sorting: 1.64s
+2. Golang 1.22.0       = 1.40s, sorting with collations: 1.71s
 3. Python 3.12.2       = 5.69, sorting with collations: 6.04s
 4. Crystal 1.5.1       = 5.61s
 5. Elixir 1.14.0       = 7.34s
-6. Julia 1.8.1         = 12.13s, with sorting: 12.22s
-7. Ruby 3.3.0          = 12.63s, sorting with collations: 12.87s
+6. Julia 1.8.1         = 12.13s, sorting: 12.22s
+7. Ruby 3.3.0          = 12.63s, sorting with collations: 22.00s
 </pre>
 
 ### Conclusion
@@ -52,6 +52,8 @@ The new optimized Golang code version is very fast, slower than Rust but faster
 2024-03-02
 
 * Updated Python version to 3.12.2, added poetry, solved missing icu4 collations for M1 processors, added a fancy progress bar
+* Updated Golang version to 1.22.0
+* Updated Ruby version to 3.3.0, added sorting with collations for many languages (this slowed the code almost 2x)
 
 2022-09-17
 

example-ruby/.tool-versions

@@ -1 +1 @@
-ruby 3.2.0-preview2
+ruby 3.3.0

example-ruby/Gemfile

@@ -1,11 +1,13 @@
 source 'https://rubygems.org'
 git_source(:github) { |repo| "https://github.com/#{repo}.git" }
 
-ruby '3.2.0.preview2'
+ruby '3.3.0'
 
+gem 'bigdecimal', require: false
+gem 'ffi-icu'
 gem 'parallel'
 gem 'rubocop', require: false
 # gem 'irbtools', require: 'irbtools/binding'
-gem 'pry'
-gem 'pry-doc'
-gem 'pry-gem', '~> 1.0'
+# gem 'pry'
+# gem 'pry-doc'
+# gem 'pry-gem'

example-ruby/Gemfile.lock

@@ -2,53 +2,47 @@ GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.2)
-    coderay (1.1.3)
-    json (2.6.2)
-    method_source (1.0.0)
-    parallel (1.22.1)
-    parser (3.1.2.1)
+    bigdecimal (3.1.6)
+    ffi (1.16.3)
+    ffi-icu (0.5.3)
+      ffi (~> 1.0, >= 1.0.9)
+    json (2.7.1)
+    language_server-protocol (3.17.0.3)
+    parallel (1.24.0)
+    parser (3.3.0.5)
       ast (~> 2.4.1)
-    pry (0.14.1)
-      coderay (~> 1.1)
-      method_source (~> 1.0)
-    pry-doc (1.3.0)
-      pry (~> 0.11)
-      yard (~> 0.9.11)
-    pry-gem (1.0.0)
-      pry (~> 0.12)
+      racc
+    racc (1.7.3)
     rainbow (3.1.1)
-    regexp_parser (2.5.0)
-    rexml (3.2.5)
-    rubocop (1.36.0)
+    regexp_parser (2.9.0)
+    rexml (3.2.6)
+    rubocop (1.61.0)
       json (~> 2.3)
+      language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
-      parser (>= 3.1.2.1)
+      parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.20.1, < 2.0)
+      rubocop-ast (>= 1.30.0, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.21.0)
-      parser (>= 3.1.1.0)
-    ruby-progressbar (1.11.0)
-    unicode-display_width (2.3.0)
-    webrick (1.7.0)
-    yard (0.9.28)
-      webrick (~> 1.7.0)
+      unicode-display_width (>= 2.4.0, < 3.0)
+    rubocop-ast (1.31.1)
+      parser (>= 3.3.0.4)
+    ruby-progressbar (1.13.0)
+    unicode-display_width (2.5.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
+  bigdecimal
+  ffi-icu
   parallel
-  pry
-  pry-doc
-  pry-gem (~> 1.0)
   rubocop
 
 RUBY VERSION
-   ruby 3.2.0.preview2
+   ruby 3.3.0p0
 
 BUNDLED WITH
-   2.4.0.dev
+   2.5.6

example-ruby/words.rb

@@ -3,9 +3,11 @@
 require 'etc'
 require 'fileutils'
 require 'optparse'
+require 'uri'
+require 'ffi-icu'
 
 class WordsExtractor
-  def initialize(cores: Etc.nprocessors, sorting: false, outdir: 'words', source: '../data/??/**/*.yml')
+  def initialize(outdir:, source:, cores: Etc.nprocessors, sorting: false)
     @cores = cores
     @sorting = sorting
     @outdir = outdir
@@ -17,6 +19,35 @@ def clear_output
     Dir.mkdir(@outdir)
   end
 
+  def get_collation(lang)
+    mapper = {
+      'ar' => 'ar_SA', # Arabic, Saudi Arabia
+      'cs' => 'cs_CZ', # Czech, Czech Republic
+      'da' => 'da_DK', # Danish, Denmark
+      'de' => 'de_DE', # German, Germany
+      'el' => 'el_GR', # Greek, Greece
+      'en' => 'en_EN', # English
+      'eo' => 'eo',    # Esperanto, not country-specific
+      'es' => 'es_ES', # Spanish, Spain
+      'fi' => 'fi_FI', # Finnish, Finland
+      'fr' => 'fr_FR', # French, France
+      'he' => 'he_IL', # Hebrew, Israel
+      'hr' => 'hr_HR', # Croatian, Croatia
+      'hu' => 'hu_HU', # Hungarian, Hungary
+      'it' => 'it_IT', # Italian, Italy
+      'lt' => 'lt_LT', # Lithuanian, Lithuania
+      'la' => 'en_EN', # Latin locale is the same as English
+      'nl' => 'nl_NL', # Dutch, Netherlands
+      'pl' => 'pl_PL', # Polish, Poland
+      'pt' => 'pt_PT', # Portuguese, Portugal
+      'ru' => 'ru_RU', # Russian, Russia
+      'sk' => 'sk_SK', # Slovak, Slovakia
+      'sv' => 'sv_SE', # Swedish, Sweden
+      'uk' => 'uk_UA'  # Ukrainian, Ukraine
+    }
+    mapper[lang]
+  end
+
   def get_words(filepath)
     IO.readlines(filepath).map do |line|
       line.strip.downcase.split(' ')[2...-1].join(' ').split(/[^\p{L}]/).uniq.select { |s| s.size > 1 }
@@ -34,15 +65,21 @@ def run
     print ' with sorting' if @sorting
     puts '...'
     clear_output
+
     start = Time.now
     paths = Dir[@source]
     count = paths.count
+
     sizes = Parallel.map_with_index(paths, in_processes: @cores) do |yaml_path, i|
       meta = YAML.load_file(yaml_path)
       filepath = yaml_path.gsub('.yml', '.txt')
       words = get_words(filepath)
-      words.sort! if @sorting
+      if @sorting
+        collator = ICU::Collation::Collator.new(get_collation(meta['lang']))
+        words = words.sort { |a, b| collator.compare(a, b) }
+      end
       save_words(words:, meta:, yaml_path:, count:, i:)
+
       File.size(filepath)
     end
     puts "Total size: #{(sizes.sum / 1024.0 / 1024).round} MB"
@@ -64,5 +101,10 @@ def run
     end
     opts.on('-s', 'Sort results') { |v| options[:s] = v }
   end.parse!
-  WordsExtractor.new(cores: options[:n], sorting: options[:s]).run
+  WordsExtractor.new(
+    cores: options[:n],
+    sorting: options[:s],
+    outdir: 'words',
+    source: '../data/??/**/*.yml'
+  ).run
 end