Merge pull request #2 from hipertracker/revert-1-main

Revert "Golang improvements."

Author: hipertracker

words_extractor_go/.gitignore

@@ -3,4 +3,3 @@ words
 .history
 /fast_words
 /.DS_Store
-sort-me-out

words_extractor_go/Makefile

@@ -1,26 +1,24 @@
 package main
 
 import (
-	"bufio"
 	"bytes"
-	"os"
+	"io/ioutil"
 	"regexp"
 	"strconv"
 	"strings"
 )
 
 func getRows(metaPath string) ListOfStrings {
 	path := strings.Replace(metaPath, ".yml", ".txt", -1)
-	data, _ := os.Open(path)
-	defer data.Close()
-
-	scanner := bufio.NewScanner(data)
-	scanner.Split(bufio.ScanLines)
-	var txtlines []string
-	for scanner.Scan() {
-		txtlines = append(txtlines, scanner.Text())
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		panic(err)
+	}
+	rows := strings.Split(string(data), "\n")
+	if rows[len(rows)-1] == "" {
+		rows = rows[:len(rows)-1]
 	}
-	return txtlines
+	return rows
 }
 
 func (arr ListOfStrings) toString() string {

words_extractor_go/content.go

@@ -3,28 +3,12 @@ package main
 import (
 	"fmt"
 	"io/ioutil"
-	"strings"
-	"time"
+	"sort"
 
-	"github.com/jfcg/sorty"
-	"github.com/thoas/go-funk"
+	"github.com/tidwall/collate"
 )
 
-type resultsArray struct {
-	Results []string
-}
-
-var (
-	res resultsArray
-)
-
-func timeTrack(start time.Time) {
-	fmt.Println("Total timing: ", time.Since(start))
-}
-
 func main() {
-	t1 := time.Now()
-	defer timeTrack(t1)
 	folder := "./words"
 	prepareFolder(folder, "*.txt")
 
@@ -33,13 +17,34 @@ func main() {
 		filename := "słowa - " + meta.Label + ".txt"
 		fmt.Println("Parsing...", filename)
 
-		res.extractWords(getRows(path).toString())
-		res.Results = funk.UniqString(res.Results)
-		sorty.SortS(res.Results)
-		data := strings.Join(res.Results, "\n")
+		// set: extracted unique words normalized to lowercase
+		set := make(map[string]void)
+		extractWords(getRows(path).toString(), set)
+		delete(set, "")
 
-		for err := ioutil.WriteFile(folder+"/"+filename, []byte(data), 0644); err != nil; {
+		// convert map[string]void to []string
+		var words []string
+		for word := range set {
+			words = append(words, word)
+		}
+
+		// sortArray(words, "POLISH_CI")
+
+		var data []byte
+		for _, word := range words {
+			bytes := []byte(word + "\n")
+			data = append(data, bytes...)
+		}
+
+		for err := ioutil.WriteFile(folder+"/"+filename, data, 0644); err != nil; {
 			panic(err)
 		}
 	}
 }
+
+func sortArray(arr []string, lang string) {
+	less := collate.IndexString(lang)
+	sort.SliceStable(arr, func(i, j int) bool {
+		return less(arr[i], arr[j])
+	})
+}

words_extractor_go/go.mod

@@ -1,9 +1,8 @@
 package main
 
 import (
-	"io/ioutil"
-
 	"gopkg.in/yaml.v3"
+	"io/ioutil"
 )
 
 type metaConfig struct {

words_extractor_go/go.sum

@@ -3,6 +3,7 @@ package main
 import (
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 )
 
@@ -25,19 +26,10 @@ func getYamlFilepaths(root string) []string {
 	return result
 }
 
-func removeCharacters(input string, characters string) string {
-	filter := func(r rune) rune {
-		if strings.IndexRune(characters, r) < 0 {
-			return r
-		}
-		return -1
-	}
-	return strings.Map(filter, input)
-}
-
-func (r *resultsArray) extractWords(s string) {
-	for _, word := range strings.Fields(s) {
-		r.Results = append(r.Results, strings.ToLower(removeCharacters(word, ".:,;()!?'-_")))
+func extractWords(s string, set map[string]void) {
+	re := regexp.MustCompile("[^\\p{L}]+")
+	for _, word := range re.Split(s, -1) {
+		set[strings.ToLower(word)] = member
 	}
 }