Skip to content

Commit 8064ba7

Browse files
ccbrownccbrown
authored
locale support update (#37)
1 parent adeac9f commit 8064ba7

File tree

7 files changed

+113
-156
lines changed

7 files changed

+113
-156
lines changed

‎server/database.go‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,6 @@ func unmarshalActivity(key, value []byte) (Activity, error) {
4545
if err != nil {
4646
return nil, err
4747
}
48-
if post.Host == "" {
49-
post.Host = "www.pathofexile.com"
50-
}
5148
if post.Id != 0 {
5249
return post, nil
5350
}

‎server/database_test.go‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,12 @@ func testDatabase_ForumPosts(t *testing.T, db Database) {
2121
Id: 9000,
2222
Poster: "Chris",
2323
Time: time.Unix(1486332365, 0),
24-
Host: locale.ForumHost(),
2524
}
2625

2726
post2 := &ForumPost{
2827
Id: 9001,
2928
Poster: "Chris",
3029
Time: time.Unix(1486332364, 0),
31-
Host: locale.ForumHost(),
3230
}
3331

3432
db.AddActivity([]Activity{post1, post2})

‎server/forum_indexer.go‎

Lines changed: 34 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ import (
99
"regexp"
1010
"strconv"
1111
"strings"
12-
"sync"
1312
"time"
1413

1514
"github.com/PuerkitoBio/goquery"
@@ -73,33 +72,37 @@ func (indexer *ForumIndexer) run() {
7372
}
7473
}
7574

76-
var wg sync.WaitGroup
77-
wg.Add(len(Locales))
78-
79-
for _, l := range Locales {
80-
l := l
81-
go func() {
82-
for {
83-
for _, account := range accounts {
84-
select {
85-
case <-indexer.closeSignal:
86-
return
87-
default:
88-
if err := indexer.index(l, account, timezone); err != nil {
89-
log.WithError(err).Error("error indexing forum account: " + account)
90-
}
91-
time.Sleep(time.Second)
92-
}
75+
for {
76+
for _, locale := range Locales {
77+
select {
78+
case <-indexer.closeSignal:
79+
return
80+
default:
81+
logger := log.WithField("host", locale.ForumHost())
82+
if err := locale.RefreshForumIds(); err != nil {
83+
logger.WithError(err).Error("error refreshing forum ids")
84+
} else {
85+
logger.Info("refreshed forum ids")
9386
}
87+
time.Sleep(time.Second)
9488
}
95-
}()
89+
}
90+
for _, account := range accounts {
91+
select {
92+
case <-indexer.closeSignal:
93+
return
94+
default:
95+
if err := indexer.index(account, timezone); err != nil {
96+
log.WithError(err).Error("error indexing forum account: " + account)
97+
}
98+
time.Sleep(time.Second)
99+
}
100+
}
96101
}
97-
98-
wg.Wait()
99102
}
100103

101-
func (indexer *ForumIndexer) requestDocument(host, resource string) (*goquery.Document, error) {
102-
urlString := fmt.Sprintf("https://%v/%v", host, strings.TrimPrefix(resource, "/"))
104+
func (indexer *ForumIndexer) requestDocument(resource string) (*goquery.Document, error) {
105+
urlString := fmt.Sprintf("https://www.pathofexile.com/%v", strings.TrimPrefix(resource, "/"))
103106
jar, _ := cookiejar.New(nil)
104107
u, _ := url.Parse(urlString)
105108
jar.SetCookies(u, []*http.Cookie{
@@ -127,7 +130,7 @@ var postURLExpression = regexp.MustCompile("^/forum/view-post/([0-9]+)")
127130
var threadURLExpression = regexp.MustCompile("^/forum/view-thread/([0-9]+)")
128131
var forumURLExpression = regexp.MustCompile("^/forum/view-forum/([0-9]+)")
129132

130-
func ScrapeForumPosts(doc *goquery.Document, locale *Locale, timezone *time.Location) ([]*ForumPost, error) {
133+
func ScrapeForumPosts(doc *goquery.Document, timezone *time.Location) ([]*ForumPost, error) {
131134
posts := []*ForumPost(nil)
132135

133136
err := error(nil)
@@ -145,7 +148,7 @@ func ScrapeForumPosts(doc *goquery.Document, locale *Locale, timezone *time.Loca
145148

146149
timeText := sel.Find(".post_date").Text()
147150

148-
if post.Time, err = locale.ParseTime(timeText, timezone); err != nil {
151+
if post.Time, err = time.ParseInLocation("Jan _2, 2006, 3:04:05 PM", timeText, timezone); err != nil {
149152
log.WithField("text", timeText).Error("unable to parse time")
150153
return false
151154
}
@@ -177,24 +180,20 @@ func ScrapeForumPosts(doc *goquery.Document, locale *Locale, timezone *time.Loca
177180
return posts, nil
178181
}
179182

180-
func (indexer *ForumIndexer) forumPosts(locale *Locale, poster string, page int, timezone *time.Location) ([]*ForumPost, error) {
181-
doc, err := indexer.requestDocument(locale.ForumHost(), fmt.Sprintf("/account/view-posts/%v/page/%v", poster, page))
183+
func (indexer *ForumIndexer) forumPosts(poster string, page int, timezone *time.Location) ([]*ForumPost, error) {
184+
doc, err := indexer.requestDocument(fmt.Sprintf("/account/view-posts/%v/page/%v", poster, page))
182185
if err != nil {
183186
return nil, err
184187
}
185-
posts, err := ScrapeForumPosts(doc, locale, timezone)
188+
posts, err := ScrapeForumPosts(doc, timezone)
186189
if err != nil {
187190
return nil, err
188191
}
189-
for _, post := range posts {
190-
post.Host = locale.ForumHost()
191-
}
192192
return posts, nil
193193
}
194194

195-
func (indexer *ForumIndexer) index(locale *Locale, poster string, timezone *time.Location) error {
195+
func (indexer *ForumIndexer) index(poster string, timezone *time.Location) error {
196196
logger := log.WithFields(log.Fields{
197-
"host": locale.ForumHost(),
198197
"poster": poster,
199198
})
200199

@@ -203,7 +202,7 @@ func (indexer *ForumIndexer) index(locale *Locale, poster string, timezone *time
203202
activity := []Activity(nil)
204203

205204
for page := 1; ; page++ {
206-
posts, err := indexer.forumPosts(locale, poster, page, timezone)
205+
posts, err := indexer.forumPosts(poster, page, timezone)
207206
if err != nil {
208207
logger.WithError(err).Error("error requesting forum posts")
209208
}
@@ -243,7 +242,7 @@ func ScrapeForumTimezone(doc *goquery.Document) (*time.Location, error) {
243242
}
244243

245244
func (indexer *ForumIndexer) sessionTimezone() (*time.Location, error) {
246-
doc, err := indexer.requestDocument("www.pathofexile.com", "/my-account/preferences")
245+
doc, err := indexer.requestDocument("/my-account/preferences")
247246
if err != nil {
248247
return nil, err
249248
}

‎server/forum_indexer_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ func TestScrapeForumPosts(t *testing.T) {
2121
tz, err := time.LoadLocation("America/Los_Angeles")
2222
require.NoError(t, err)
2323

24-
posts, err := ScrapeForumPosts(doc, Locales[0], tz)
24+
posts, err := ScrapeForumPosts(doc, tz)
2525
require.NoError(t, err)
2626
assert.Equal(t, 10, len(posts))
2727

‎server/forum_post.go‎

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
package server
22

33
import (
4+
"encoding/json"
45
"fmt"
56
"time"
67
)
78

89
type ForumPost struct {
910
Id int `json:"id"`
10-
Host string `json:"host"`
1111
BodyHTML string `json:"body_html"`
1212
Time time.Time `json:"time"`
1313
Poster string `json:"poster"`
@@ -17,6 +17,32 @@ type ForumPost struct {
1717
ForumName string `json:"forum_name"`
1818
}
1919

20+
type forumPostWithHost struct {
21+
Id int `json:"id"`
22+
BodyHTML string `json:"body_html"`
23+
Time time.Time `json:"time"`
24+
Poster string `json:"poster"`
25+
ThreadId int `json:"thread_id"`
26+
ThreadTitle string `json:"thread_title"`
27+
ForumId int `json:"forum_id"`
28+
ForumName string `json:"forum_name"`
29+
Host string `json:"host"`
30+
}
31+
32+
func (p ForumPost) MarshalJSON() ([]byte, error) {
33+
return json.Marshal(&forumPostWithHost{
34+
Id: p.Id,
35+
BodyHTML: p.BodyHTML,
36+
Time: p.Time,
37+
Poster: p.Poster,
38+
ThreadId: p.ThreadId,
39+
ThreadTitle: p.ThreadTitle,
40+
ForumId: p.ForumId,
41+
ForumName: p.ForumName,
42+
Host: p.Host(),
43+
})
44+
}
45+
2046
func (p *ForumPost) ActivityTime() time.Time {
2147
return p.Time
2248
}
@@ -25,6 +51,15 @@ func (p *ForumPost) ActivityKey() uint32 {
2551
return uint32(p.Id)
2652
}
2753

54+
func (p *ForumPost) Host() string {
55+
for _, l := range Locales {
56+
if l.ForumIds()[p.ForumId] {
57+
return l.ForumHost()
58+
}
59+
}
60+
return "www.pathofexile.com"
61+
}
62+
2863
func (p *ForumPost) PostURL() string {
29-
return fmt.Sprintf("https://%v/forum/view-post/%v", p.Host, p.Id)
64+
return fmt.Sprintf("https://%v/forum/view-post/%v", p.Host(), p.Id)
3065
}

0 commit comments

Comments
 (0)