add sorting in 23 languages for Golang example

Author: hipertracker

README.md

@@ -15,7 +15,7 @@ The following results are for 123 unique utf-8 Bible text files in 23 languages
 
 <pre>
 1. Rust 1.58      = 1.14s (with sorting: 1.59s) with tokyo (previous: 1.34s, with sorting: 1.79)
-2. Golang 1.17.6  = 1.34s (with sorting: 6.55s)
+2. Golang 1.17.6  = 1.34s (with sorting: 4.56s)
 3. Python 3.10.2  = 2.80s (with multiprocessing)
 4. Julia 1.7.1    = 4.522
 5. Crystal 1.3.2  = 5.72s

example-golang/app/app.go

@@ -46,8 +46,7 @@ func Run(srcDir, outDir string, numWorkers int, sortResults bool) error {
 		dst := filepath.Join(outDir, spec.Lang+"-"+spec.Code+".txt")
 
 		wg.Add(1)
-		// TODO: add more collations
-		go extract(src, dst, "POLISH_CI", sortResults, sem, &wg)
+		go extract(src, dst, langMap[spec.Lang], sortResults, sem, &wg)
 	}
 
 	wg.Wait()
@@ -66,3 +65,29 @@ func clearOutput(outDir string) error {
 
 	return nil
 }
+
+var langMap = map[string]string{
+	"en": "ENGLISH_CI", // The first language is used as fallback.
+	"la": "ENGLISH_CI", // Latin
+	"eo": "ENGLISH_CI", // Esperanto
+	"ar": "ARABIC_CI",
+	"cz": "CZECH_CI",
+	"da": "DANISH_CI", // ?
+	"de": "GERMAN_CI",
+	"el": "GREEK_CI",
+	"es": "SPANISH_CI",
+	"fi": "FINNISH_CI",
+	"fr": "FRENCH_CI",
+	"he": "HEBREW_CI",
+	"hr": "CROATIAN_CI",
+	"hu": "HUNGARIAN_CI",
+	"it": "ITALIAN_CI",
+	"lt": "LITHUANIAN_CI",
+	"nl": "DUTCH_CI",
+	"pl": "POLISH_CI",
+	"pt": "PORTUGUESE_CI",
+	"ru": "RUSSIAN_CI",
+	"sk": "SLOVAK_CI",
+	"sv": "SWEDISH_CI",
+	"uk": "UKRAINIAN_CI",
+}

example-golang/app/extract.go

@@ -27,7 +27,7 @@ const (
 // correctly.
 //
 // Rust and Python versions split text according to "[\W\d]+" - anything that is
-// not a word or a digit. WTF?
+// not a word or a digit. TODO: comfirm if some words contain digits
 func splitWordsUnicode(data []byte, atEOF bool) (advance int, token []byte, err error) {
 	var start int
 	var r rune

example-golang/go.mod

@@ -5,7 +5,6 @@ go 1.17
 require (
 	github.com/bmatcuk/doublestar v1.3.4
 	github.com/stretchr/testify v1.7.0
-	github.com/thoas/go-funk v0.9.1
 	github.com/tidwall/collate v1.0.0
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )

example-golang/go.sum

@@ -5,11 +5,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M=
-github.com/thoas/go-funk v0.9.1/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q=
 github.com/tidwall/collate v1.0.0 h1:xgvwO2UunUoXx3NS3UqHBX63l248ZApqo7mUe3NHy6I=
 github.com/tidwall/collate v1.0.0/go.mod h1:S56qxEr2ALVCaGY41npreOJ5lBIILSrxYLgEpxoHVIk=
 github.com/tidwall/gjson v1.3.4 h1:On5waDnyKKk3SWE4EthbjjirAWXp43xx5cKCUZY1eZw=
@@ -23,7 +20,6 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=