add new code for Python and Go on M1

Author: hipertracker

words_extractor_go/main.go

@@ -15,8 +15,14 @@ import (
 	"github.com/tidwall/collate"
 )
 
+type Pair struct {
+	Path string
+	Size int64
+}
+
 func main() {
-	var wg sync.WaitGroup
+	// var wg sync.WaitGroup
+	queue := make(chan Pair)
 
 	t := time.Now()
 	defer timeTrack(t)
@@ -27,12 +33,43 @@ func main() {
 
 	fmt.Println("Parsing...")
 
-	paths, _ := doublestar.Glob("../data/pl/**/*.yml")
+	paths, _ := doublestar.Glob("../data/**/*.yml")
+
+	total_size := int64(0)
+	items_count := len(paths)
 	for i, path := range paths {
-		wg.Add(1)
-		go worker(i, &wg, path, outdir, true)
+		go processFile(queue, outdir, path, false)
+		res := <-queue
+		total_size += res.Size
+		fmt.Printf("[%d/%d] %s\n", i+1, items_count, res.Path)
+	}
+	fmt.Printf("Total items: %d\n", items_count)
+	fmt.Printf("Total size: %d MB\n", total_size/(1024*1024))
+}
+
+func processFile(queue chan Pair, outdir string, path string, sorting bool) {
+	meta := GetYAML(path)
+	// load text file
+	filepath := strings.Replace(path, ".yml", ".txt", -1)
+	info, err := os.Stat(filepath)
+	if err != nil {
+		panic(err)
+	}
+	content, err := ioutil.ReadFile(filepath)
+	if err != nil {
+		panic(err)
+	}
+	// extract and sort unique words
+	words := extractUniqueWords(content)
+	if sorting {
+		words = sortWords(words, "POLISH_CI")
+	}
+	text := strings.Join(words, "\n")
+	outpath := fmt.Sprintf("%s/%s-%s.txt", outdir, meta.Lang, meta.Code)
+	for err := ioutil.WriteFile(outpath, []byte(text), 0644); err != nil; {
+		panic(err)
 	}
-	wg.Wait()
+	queue <- Pair{path, info.Size()}
 }
 
 func worker(id int, wg *sync.WaitGroup, path, outdir string, verbose bool) {

words_extractor_go/main_test.go

@@ -19,3 +19,7 @@ func Test_sortWords(t *testing.T) {
 	given := sortWords(words, "POLISH_CI")
 	assert.Equal(t, expected, given, "words should be sorted out using Polish grammar rules")
 }
+
+// Total items: 123
+// Total size: 503 MB`
+// Total timing:  36.606038042s`

words_extractor_py/.gitignore

@@ -1,3 +1,5 @@
 /words/
 /.DS_Store
 /venv/
+/.idea/
+/.venv/

words_extractor_py/requirements.txt

@@ -1,2 +1,2 @@
-PyICU==2.7.3
-PyYAML==5.4.1
+#PyICU>=2.7.4
+PyYAML>=5.4.1

words_extractor_py/words.py

@@ -1,24 +1,42 @@
 import glob
-from icu import Collator, Locale
+from typing import Tuple
+import os
+
+try:
+    from icu import Collator, Locale
+
+    i18nsorting = True
+except ModuleNotFoundError:
+    i18nsorting = False
+
+import multiprocessing as mp
 import os
 import re
 import shutil
 import time
 import yaml
 
 
-def worker(path, collator, separator, outdir, with_sorting):
+def worker(path: str, outdir: str, sorting: bool = False) -> Tuple[str, int]:
+    if sorting:
+        if i18nsorting:
+            collator = Collator.createInstance(Locale("pl_PL.UTF-8"))
+        print("I18nN sorting not available")
+
+    separator = re.compile("[\W\d]+")
     filepath = path.replace(".yml", ".txt")
+    filesize = os.path.getsize(filepath)
     with open(filepath) as file:
+
         text = file.read().lower().rstrip()
         words = set(re.split(separator, text))
     with open(path) as file:
         meta = yaml.safe_load(file)
-    with open(f"{outdir}/extracted-words-for-{meta['label']}.txt", "w") as file:
-        if with_sorting:
+    with open(f"{outdir}/{meta['lang']}-{meta['code']}.txt", "w") as file:
+        if sorting and i18nsorting:
             words = sorted(words, key=collator.getSortKey)
         file.write("\n".join(words))
-    print(f"Saved: ", filepath)
+    return path, filesize
 
 
 if __name__ == "__main__":
@@ -29,16 +47,33 @@ def worker(path, collator, separator, outdir, with_sorting):
         shutil.rmtree(outdir)
     os.makedirs(outdir)
 
-    collator = Collator.createInstance(Locale("pl_PL.UTF-8"))
-    separator = re.compile("[\W\d]+")
+    pool = mp.Pool(mp.cpu_count())
+
     print("Processing")
-    for path in glob.glob("../data/pl/**/*.yml", recursive=True):
-        worker(
-            path=path,
-            collator=collator,
-            separator=separator,
-            outdir=outdir,
-            with_sorting=True,
+    results = []
+    for path in glob.glob("../data/**/*.yml", recursive=True):
+        res = pool.apply_async(
+            worker,
+            kwds=dict(
+                path=path,
+                outdir=outdir,
+                sorting=False,
+            ),
         )
+        results.append(res)
+    total_size = 0
+    items_count = len(results)
+    for i, res in enumerate(results):
+        path, size = res.get()
+        total_size += size
+        print(f"[{i+1}/{items_count}] {path}")
+    print(f"Total files: {len(results)}")
+    print(f"Total size: {round((total_size / 1024 / 1024))} MB")
+    t = time.time() - t
+    print(f"Total time: {t:.4f} s")
 
-    print("Total timing: ", time.time() - t)
+"""
+Total files: 123
+Total size: 504 MB
+Total time: 5.1153 s
+"""