fix bug in Go not reading recursive folders, fix results, fix sorting rules for ASCII and non ASCII text files

Author: hipertracker

README.md

@@ -14,8 +14,8 @@ The following results are for 123 unique utf-8 Bible text files in 23 languages
 * Machine: MacBook Pro 16" 64GB 2TB M1Max 10 cores.
 
 <pre>
-1. Golang 1.17.6  = 0.47s (with sorting: 2.25s)
-2. Rust 1.58      = 1.14s (with sorting: 1.62s) with tokyo (previous: 1.34s, with sorting: 1.79)
+1. Rust 1.58      = 1.14s (with sorting: 1.62s) with tokyo (previous: 1.34s, with sorting: 1.79)
+2. Golang 1.17.6  = 1.34s (with sorting: 6.55s)
 3. Python 3.10.2  = 2.80s (with multiprocessing)
 4. Julia 1.7.1    = 4.522
 5. Crystal 1.3.2  = 5.72s

example-golang/app/app.go

@@ -5,6 +5,8 @@ import (
 	"os"
 	"path/filepath"
 	"sync"
+
+	"github.com/bmatcuk/doublestar"
 )
 
 const dirPerms = 0755
@@ -15,7 +17,7 @@ type empty struct{}
 // No error handling, no context cancellation is implemented to match implementations
 // in other languages.
 func Run(srcDir, outDir string, numWorkers int, sortResults bool) error {
-	files, err := filepath.Glob(srcDir)
+	files, err := doublestar.Glob(srcDir)
 	if err != nil {
 		return fmt.Errorf(`app: getting list of files "%s": %w`, srcDir, err)
 	}
@@ -44,6 +46,7 @@ func Run(srcDir, outDir string, numWorkers int, sortResults bool) error {
 		dst := filepath.Join(outDir, spec.Lang+"-"+spec.Code+".txt")
 
 		wg.Add(1)
+		// TODO: add more collations
 		go extract(src, dst, "POLISH_CI", sortResults, sem, &wg)
 	}
 

example-golang/app/extract.go

@@ -110,7 +110,7 @@ func extract(src, dst, lang string, sortResults bool, sem <-chan empty, wg *sync
 
 	// One of the possible optimisations here is to split file in chunks and process
 	// each chunk individually.
-	words, err := collectWords(fd, InitialDictSize)
+	words, err := collectWords(fd, lang, InitialDictSize)
 	if err != nil {
 		_, _ = fmt.Fprintf(os.Stderr, `extract: reading input "%s": %s`, src, err)
 		return
@@ -146,9 +146,14 @@ func extract(src, dst, lang string, sortResults bool, sem <-chan empty, wg *sync
 	_, _ = fmt.Fprintf(os.Stdout, "Saved %s\n", dst)
 }
 
-func collectWords(r io.Reader, sizeHint int) ([]string, error) {
+func collectWords(r io.Reader, lang string, sizeHint int) ([]string, error) {
 	scanner := bufio.NewScanner(r)
-	scanner.Split(splitWords)
+	ascii := []string{"en", "la"}
+	if stringInSlice(lang, ascii) {
+		scanner.Split(splitWords)
+	} else {
+		scanner.Split(splitWordsUnicode)
+	}
 
 	// map[uint64]empty should take less memory than map[string]empty and avoid
 	// GC checks.
@@ -200,7 +205,8 @@ func writeResults(w io.Writer, words []string) error {
 
 func ExtractUniqueWords(content string, lang string, sizeHint int) ([]string, error) {
 	r := strings.NewReader(content)
-	words, err := collectWords(r, sizeHint)
+	words, err := collectWords(r, lang, sizeHint)
+
 	if err != nil {
 		_, _ = fmt.Fprintf(os.Stderr, `collectWords error: %s`, err)
 		return nil, err
@@ -211,3 +217,12 @@ func ExtractUniqueWords(content string, lang string, sizeHint int) ([]string, er
 	})
 	return words, nil
 }
+
+func stringInSlice(a string, list []string) bool {
+	for _, b := range list {
+		if b == a {
+			return true
+		}
+	}
+	return false
+}

example-golang/main.go

@@ -7,7 +7,6 @@ import (
 	"path/filepath"
 	"runtime"
 	"time"
-
 	"wordextractor/app"
 )