add crystal example

Author: Jaroslaw Zabiello

example-crystal/.editorconfig

@@ -0,0 +1,9 @@
+root = true
+
+[*.cr]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+indent_style = space
+indent_size = 2
+trim_trailing_whitespace = true

example-crystal/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Jaroslaw Zabiello <[email protected]>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

example-crystal/README.md

@@ -0,0 +1,35 @@
+# example-crystal
+
+TODO: Write a description here
+
+## Installation
+
+TODO: Write installation instructions here
+
+## Usage
+
+crystal build --release -Dpreview_mt src/example-crystal.cr -o main
+CRYSTAL_WORKES=10 ./main
+
+MacOS 12.2
+Crystal 1.3.2
+MBP 16" M1Max 10 cores
+Total files: 123
+Total size: 504 MB
+Total time: 6.0035 s
+
+## Development
+
+TODO: Write development instructions here
+
+## Contributing
+
+1. Fork it (<https://github.com/your-github-user/example-crystal/fork>)
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request
+
+## Contributors
+
+- [Jaroslaw Zabiello](https://github.com/your-github-user) - creator and maintainer

example-crystal/main

@@ -0,0 +1,2 @@
+version: 2.0
+shards: {}

example-crystal/shard.lock

@@ -0,0 +1,13 @@
+name: example-crystal
+version: 0.1.0
+
+authors:
+  - Jaroslaw Zabiello <[email protected]>
+
+targets:
+  example-crystal:
+    main: src/example-crystal.cr
+
+crystal: 1.3.2
+
+license: MIT

example-crystal/shard.yml

@@ -0,0 +1,21 @@
+require "./spec_helper"
+
+describe Example::Crystal do
+  it "should be understand PL collations" do
+    words = %w(ala ąla ćma ciemno ęle element łódź lody śmierć serial źdźbło żółw zawias)
+    expected = %w(ala ąla ciemno ćma element ęle lody łódź serial śmierć zawias źdźbło żółw)
+    words.sort { |x, y| Example::Crystal.word_cmp(x, y) }.should eq(expected)
+  end
+
+  it "should be understand PL collations for q" do
+    words = %w(ala ąla ćma ciemno ęle element łódź lody śmierć serial źdźbło żółw zawias querty)
+    expected = %w(ala ąla ciemno ćma element ęle lody łódź querty serial śmierć zawias źdźbło żółw)
+    words.sort { |x, y| Example::Crystal.word_cmp(x, y) }.should eq(expected)
+  end
+
+  it "should understand PL collations for upper chars" do
+    words = %w(ala ąla Ćma ciemno Ęle Element łódź lody śmierć serial źdźbło żółw zawias querty)
+    expected = %w(ala ąla ciemno Ćma Element Ęle lody łódź querty serial śmierć zawias źdźbło żółw)
+    words.sort { |x, y| Example::Crystal.word_cmp(x.downcase, y.downcase) }.should eq(expected)
+  end
+end

example-crystal/spec/example-crystal_spec.cr

@@ -0,0 +1,2 @@
+require "spec"
+require "../src/example-crystal"

example-crystal/spec/spec_helper.cr

@@ -0,0 +1,84 @@
+require "json"
+require "yaml"
+
+# TODO: Write documentation for `FastWordsCr`
+
+module Example::Crystal
+  VERSION = "0.2.0"
+  CHARSET = "aąbcćdeęfghijklłmnńoópqrsśtuvwxyzźż"
+
+  def self.main(outpath = "words")
+    with_sorting = false
+    concurrent = true
+
+    prepare_folder(outpath, "*.txt")
+
+    file_count = 0
+    total_size = 0
+
+    channel = Channel(Tuple(String, Int64)).new
+    srcPath = "../data/??/**/*.yml"
+    # srcPath = "./bibles/??/**/*.yml"
+    Dir.glob(srcPath, follow_symlinks: true).each do |path|
+      if concurrent
+        spawn do
+          channel.send worker(path, outpath, with_sorting)
+        end
+        file_count += 1
+      else
+        worker(path, outpath, with_sorting)
+      end
+    end
+    if concurrent
+      file_count.times do |i|
+        path, size = channel.receive
+        total_size += size
+        # puts("[#{i + 1}/#{file_count}] #{path}")
+      end
+    end
+    total_size = total_size / 1024 / 1024
+    puts("Total size: #{total_size} MB")
+  end
+
+  def self.worker(path, outpath, with_sorting)
+    filepath = path.gsub(".yml", ".txt")
+    text = File.read(filepath).gsub("\n", " ").downcase
+
+    words = text.split(/[^\p{L}]+/).to_set
+
+    if with_sorting
+      words = words.to_a.sort { |x, y| self.word_cmp(x, y) }
+    end
+
+    meta = File.open(path) { |file| YAML.parse(file) }
+    filepath = %Q(#{outpath}/#{meta["lang"]}-#{meta["code"]}.txt)
+    File.write(filepath, words.join("\n"))
+    filesize = File.size(filepath)
+    puts([filepath, filesize])
+    {filepath, filesize}
+  end
+
+  def self.prepare_folder(folder : String, pattern : String)
+    Dir.mkdir_p(folder) unless File.exists?(folder)
+    Dir.glob("#{folder}/#{pattern}").each do |path|
+      File.delete path
+    end
+  end
+
+  def self.word_cmp(str1 : String, str2 : String)
+    tokens2 = str2.chars
+    str1.chars.each_with_index do |s1, i|
+      return 1 unless tokens2[i]?
+      idx1 = CHARSET.index(s1) || -1
+      idx2 = CHARSET.index(tokens2[i]) || -1
+      return 1 if idx1 > idx2
+      return -1 if idx1 < idx2
+    end
+    0
+  end
+end
+
+elapsed_time = Time.measure do
+  Example::Crystal.main
+end
+puts elapsed_time