Update main.go

improve Go implementation

Author: hipertracker

words_extractor_go/main.go

@@ -10,7 +10,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/bmatcuk/doublestar/v2"
+	"github.com/bmatcuk/doublestar"
 	"github.com/thoas/go-funk"
 	"github.com/tidwall/collate"
 )
@@ -20,92 +20,56 @@ type Pair struct {
 	Size int64
 }
 
+// var srcPath = "../data/**/*.yml"
+var srcPath = "./bibles/??/**/*.yml"
+var outdir = "words"
+
 func main() {
-	with_channels()
-	without_channels()
-}
+	paths, _ := doublestar.Glob(srcPath)
 
-func without_channels() {
-	var wg sync.WaitGroup
+	clearResults()
+	runWithChannels(paths)
 
-	t := time.Now()
-	defer timeTrack(t)
+	clearResults()
+	runWithWaitGroups(paths)
+}
 
-	outdir := "words"
+func clearResults() {
 	os.RemoveAll(outdir)
 	os.Mkdir(outdir, 0777)
-
-	fmt.Println("Parsing without channels...")
-
-	paths, _ := doublestar.Glob("../data/**/*.yml")
-
-	items_count := len(paths)
-        wg.Add(items_count)
-	for _, path := range paths {
-		go processFile(&wg, outdir, path, false)
-	}
-	wg.Wait()
-
-	fmt.Printf("Total items: %d\n", items_count)
-
 }
 
-func with_channels() {
-	queue := make(chan string)
+func runWithChannels(paths []string) {
+	var ch = make(chan string)
 
 	t := time.Now()
 	defer timeTrack(t)
 
-	outdir := "words"
-	os.RemoveAll(outdir)
-	os.Mkdir(outdir, 0777)
-
-	fmt.Println("Parsing with channels...")
-
-	paths, _ := doublestar.Glob("../data/**/*.yml")
-
-	// total_size := int64(0)
-	items_count := len(paths)
 	for _, path := range paths {
-		go processFileWithChannels(queue, outdir, path, false)
-		path := <-queue
-		fmt.Println(path)
-		// total_size += res.Size
-		// fmt.Printf("[%d/%d] %s\n", i+1, items_count, res.Path)
+		go func(yamlPath string) {
+			ch <- parseFile(yamlPath, false)
+		}(path)
+	}
+	for range paths {
+		<-ch
 	}
-	fmt.Printf("Total items: %d\n", items_count)
-	// fmt.Printf("Total size: %d MB\n", total_size/(1024*1024))
 }
 
-func processFileWithChannels(queue chan string, outdir string, path string, sorting bool) {
-	meta := GetYAML(path)
-	// load text file
-	filepath := strings.Replace(path, ".yml", ".txt", -1)
-	// info, err := os.Stat(filepath)
-	// if err != nil {
-	// panic(err)
-	// }
-	content, err := ioutil.ReadFile(filepath)
-	if err != nil {
-		panic(err)
-	}
-	// extract and sort unique words
-	words := extractUniqueWords(content)
-	if sorting {
-		words = sortWords(words, "POLISH_CI")
-	}
-	text := strings.Join(words, "\n")
-	outpath := fmt.Sprintf("%s/%s-%s.txt", outdir, meta.Lang, meta.Code)
-	for err := ioutil.WriteFile(outpath, []byte(text), 0644); err != nil; {
-		panic(err)
+func runWithWaitGroups(paths []string) {
+	var wg sync.WaitGroup
+	t := time.Now()
+	defer timeTrack(t)
+	for _, path := range paths {
+		wg.Add(1)
+		go func(yamlPath string) {
+			parseFile(yamlPath, false)
+			wg.Done()
+		}(path)
 	}
-	// queue <- Pair{path, info.Size()}
-	queue <- path
-
+	wg.Wait()
 }
 
-func processFile(wg *sync.WaitGroup, outdir string, path string, sorting bool) {
-	defer wg.Done()
+func parseFile(path string, sorting bool) string {
 	// load YAML file
 	meta := GetYAML(path)
 	outfilepath := fmt.Sprintf("%s/extracted-words-for-%s.txt", outdir, meta.Code)
@@ -128,7 +92,7 @@ func processFile(wg *sync.WaitGroup, outdir string, path string, sorting bool) {
 	for err := ioutil.WriteFile(outfilepath, []byte(text), 0644); err != nil; {
 		panic(err)
 	}
-	fmt.Println(path)
+	return outfilepath
 }
 
 func timeTrack(start time.Time) {