load code

hipertracker · hipertracker · commit e097b8a08781 · 2021-05-08T16:19:37.000+01:00
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,4 @@
+/.history/
+/.vscode/
+/.DS_Store
+/data
diff --git a/README.md b/README.md
@@ -1,2 +1,26 @@
 # words_extractor
+
 Example of words extracting in Go, Crystal, Rust and Python
+
+Text source: 79.4MB in 20 files
+
+- Python 3.9.5 with sorting: 11s, without sorting 10s
+- Go 1.16.3 with sorting: 30s, without sorting: 20s
+- Rust 1.51.0 with sorting: 1m31s, without sorting: 1m10s
+- Crystal with sorting: 2m55s, without sorting: 27s
+
+macOS 11.3.1, MacBook Pro (Retina, 15-inch, Late 2013)
+
+```
+cd words_extractor_py
+python words.py
+
+cd words_extractor_rs
+cargo run
+
+cd words_extractor_go
+go run .
+
+cd words_extractor_cr
+crystal run src/fast_words_cr.cr
+```
diff --git a/words_extractor_cr/.editorconfig b/words_extractor_cr/.editorconfig
@@ -0,0 +1,9 @@
+root = true
+
+[*.cr]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+indent_style = space
+indent_size = 2
+trim_trailing_whitespace = true
diff --git a/words_extractor_cr/.gitignore b/words_extractor_cr/.gitignore
@@ -0,0 +1,10 @@
+/docs/
+/lib/
+/bin/
+/.shards/
+*.dwarf
+/.history/
+/words/
+/fast_words_cr
+/.idea/
+/.DS_Store
diff --git a/words_extractor_cr/.travis.yml b/words_extractor_cr/.travis.yml
@@ -0,0 +1,6 @@
+language: crystal
+
+# Uncomment the following if you'd like Travis to run specs and check code formatting
+# script:
+#   - crystal spec
+#   - crystal tool format --check
diff --git a/words_extractor_cr/LICENSE b/words_extractor_cr/LICENSE
@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2021 Jaroslaw Zabiello <hipertracker@gmail.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/words_extractor_cr/README.md b/words_extractor_cr/README.md
@@ -0,0 +1,27 @@
+# fast_words_cr
+
+TODO: Write a description here
+
+## Installation
+
+TODO: Write installation instructions here
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+TODO: Write development instructions here
+
+## Contributing
+
+1. Fork it (<https://github.com/your-github-user/fast_words_cr/fork>)
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request
+
+## Contributors
+
+- [Jaroslaw Zabiello](https://github.com/your-github-user) - creator and maintainer
diff --git a/words_extractor_cr/shard.lock b/words_extractor_cr/shard.lock
@@ -0,0 +1,2 @@
+version: 2.0
+shards: {}
diff --git a/words_extractor_cr/shard.yml b/words_extractor_cr/shard.yml
@@ -0,0 +1,13 @@
+name: fast_words_cr
+version: 0.1.0
+
+authors:
+  - Jaroslaw Zabiello <hipertracker@gmail.com>
+
+targets:
+  fast_words_cr:
+    main: src/fast_words_cr.cr
+
+crystal: 1.0.0
+
+license: MIT
diff --git a/words_extractor_cr/spec/fast_words_cr_spec.cr b/words_extractor_cr/spec/fast_words_cr_spec.cr
@@ -0,0 +1,21 @@
+require "./spec_helper"
+
+describe FastWordsCr do
+  it "should be understand PL collations" do
+    words = %w(ala ąla ćma ciemno ęle element łódź lody śmierć serial źdźbło żółw zawias)
+    expected = %w(ala ąla ciemno ćma element ęle lody łódź serial śmierć zawias źdźbło żółw)
+    words.sort { |x, y| FastWordsCr.cmp(x, y) }.should eq(expected)
+  end
+
+  it "should be understand PL collations for q" do
+    words = %w(ala ąla ćma ciemno ęle element łódź lody śmierć serial źdźbło żółw zawias querty)
+    expected = %w(ala ąla ciemno ćma element ęle lody łódź querty serial śmierć zawias źdźbło żółw)
+    words.sort { |x, y| FastWordsCr.cmp(x, y) }.should eq(expected)
+  end
+
+  it "should understand PL collations for upper chars" do
+    words = %w(ala ąla Ćma ciemno Ęle Element łódź lody śmierć serial źdźbło żółw zawias querty)
+    expected = %w(ala ąla ciemno Ćma Element Ęle lody łódź querty serial śmierć zawias źdźbło żółw)
+    words.sort { |x, y| FastWordsCr.cmp(x, y) }.should eq(expected)
+  end
+end
diff --git a/words_extractor_cr/spec/spec_helper.cr b/words_extractor_cr/spec/spec_helper.cr
@@ -0,0 +1,2 @@
+require "spec"
+require "../src/fast_words_cr"
diff --git a/words_extractor_cr/src/fast_words_cr.cr b/words_extractor_cr/src/fast_words_cr.cr
@@ -0,0 +1,54 @@
+require "json"
+require "yaml"
+
+# TODO: Write documentation for `FastWordsCr`
+
+module FastWordsCr
+  VERSION = "0.1.0"
+
+  def self.main(outpath = "words")
+    prepare_folder(outpath, "*.txt")
+    Dir.glob("../data/pl/**/*.yml").each do |path|
+      text = File.read(path.gsub(".yml", ".txt")).gsub("\n", " ").downcase
+
+      # 2m13s or 2m58 with verbose .downcase in self.word_cmp
+      words_json = (text.split(/[^\p{L}]+/).to_set - Set{""}).to_json.downcase
+      words = Array(String).from_json(words_json).sort { |x, y| self.word_cmp(x, y) }
+
+      # 3m8s
+      # words = (text.split(/[^\p{L}]+/).to_set - Set{""}).to_a.sort do |x, y|
+      #   self.word_cmp(x, y)
+      # end
+
+      # 27s (no sort)
+      # words = (text.split(/[^\p{L}]+/).to_set - Set{""}).to_a
+
+      meta = File.open(path) { |file| YAML.parse(file) }
+      filepath = %Q(#{outpath}/słowa - #{meta["label"]}.txt)
+      puts filepath
+      File.write(filepath, words.join("\n"))
+    end
+  end
+
+  def self.prepare_folder(folder : String, pattern : String)
+    Dir.mkdir_p(folder) unless File.exists?(folder)
+    Dir.glob("#{folder}/#{pattern}").each do |path|
+      File.delete path
+    end
+  end
+
+  def self.word_cmp(str1 : String, str2 : String, charset = "aąbcćdeęfghijklłmnńoópqrsśtuvwxyzźż")
+    tokens1 = str1.downcase.split("")
+    tokens2 = str2.downcase.split("")
+    tokens1.each_with_index do |s1, i|
+      return 1 unless tokens2[i]?
+      idx1 = charset.index(s1) || -1
+      idx2 = charset.index(tokens2[i]) || -1
+      return 1 if idx1 > idx2
+      return -1 if idx1 < idx2
+    end
+    0
+  end
+end
+
+FastWordsCr.main
diff --git a/words_extractor_go/.gitignore b/words_extractor_go/.gitignore
@@ -0,0 +1,5 @@
+.idea
+words
+.history
+/fast_words
+/.DS_Store
diff --git a/words_extractor_go/content.go b/words_extractor_go/content.go
@@ -0,0 +1,57 @@
+package main
+
+import (
+	"bytes"
+	"io/ioutil"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+func getRows(metaPath string) ListOfStrings {
+	path := strings.Replace(metaPath, ".yml", ".txt", -1)
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		panic(err)
+	}
+	rows := strings.Split(string(data), "\n")
+	if rows[len(rows)-1] == "" {
+		rows = rows[:len(rows)-1]
+	}
+	return rows
+}
+
+func (arr ListOfStrings) toString() string {
+	buf := bytes.Buffer{}
+	for _, row := range arr {
+		text := parseVerse(row).text
+		buf.WriteString(text + " ")
+	}
+	return buf.String()
+}
+
+type verseStruct struct {
+	book    string
+	chapter int
+	verse   int
+	text    string
+}
+
+func parseVerse(s string) verseStruct {
+	re := regexp.MustCompile("([^ ]+) ([^ ]+):([^ ]+)\\s?(.*)")
+	match := re.FindStringSubmatch(s)
+	chapter, chapterError := strconv.Atoi(match[2])
+	if chapterError != nil {
+		panic(chapterError)
+	}
+	verse, verseError := strconv.Atoi(match[3])
+	if verseError != nil {
+		panic(verseError)
+	}
+	return verseStruct{
+		book:    match[1],
+		chapter: chapter,
+		verse:   verse,
+		text:    match[4],
+	}
+}
diff --git a/words_extractor_go/main.go b/words_extractor_go/main.go
@@ -0,0 +1,50 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"sort"
+
+	"github.com/tidwall/collate"
+)
+
+func main() {
+	folder := "./words"
+	prepareFolder(folder, "*.txt")
+
+	for _, path := range getYamlFilepaths("../data/pl/") {
+		meta := getMeta(path)
+		filename := "słowa - " + meta.Label + ".txt"
+		fmt.Println("Parsing...", filename)
+
+		// set: extracted unique words normalized to lowercase
+		set := make(map[string]void)
+		extractWords(getRows(path).toString(), set)
+		delete(set, "")
+
+		// convert map[string]void to []string
+		var words []string
+		for word := range set {
+			words = append(words, word)
+		}
+
+		// sortArray(words, "POLISH_CI")
+
+		var data []byte
+		for _, word := range words {
+			bytes := []byte(word + "\n")
+			data = append(data, bytes...)
+		}
+
+		for err := ioutil.WriteFile(folder+"/"+filename, data, 0644); err != nil; {
+			panic(err)
+		}
+	}
+}
+
+func sortArray(arr []string, lang string) {
+	less := collate.IndexString(lang)
+	sort.SliceStable(arr, func(i, j int) bool {
+		return less(arr[i], arr[j])
+	})
+}
diff --git a/words_extractor_go/meta.go b/words_extractor_go/meta.go
@@ -0,0 +1,31 @@
+package main
+
+import (
+	"gopkg.in/yaml.v3"
+	"io/ioutil"
+)
+
+type metaConfig struct {
+	Lang   string
+	Code   string
+	Label  string
+	Year   string
+	Strong bool
+}
+
+func (m *metaConfig) Parse(data []byte) error {
+	return yaml.Unmarshal(data, m)
+}
+
+func getMeta(path string) metaConfig {
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		panic(err)
+	}
+	config := metaConfig{}
+	err = config.Parse(data)
+	if err != nil {
+		panic(err)
+	}
+	return config
+}
diff --git a/words_extractor_go/utils.go b/words_extractor_go/utils.go
diff --git a/words_extractor_py/.gitignore b/words_extractor_py/.gitignore
diff --git a/words_extractor_py/words.py b/words_extractor_py/words.py
diff --git a/words_extractor_rs/.gitignore b/words_extractor_rs/.gitignore
diff --git a/words_extractor_rs/Cargo.lock b/words_extractor_rs/Cargo.lock
diff --git a/words_extractor_rs/Cargo.toml b/words_extractor_rs/Cargo.toml
diff --git a/words_extractor_rs/src/main.rs b/words_extractor_rs/src/main.rs
