Skip to content

Commit 76d9298

Browse files
ccbrownccbrown
authored
update for post url change (ccbrown#24)
1 parent 7ce15ae commit 76d9298

File tree

4 files changed

+5
-9
lines changed

4 files changed

+5
-9
lines changed

‎server/forum_indexer.go‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ func (indexer *ForumIndexer) requestDocument(host, resource string) (*goquery.Do
121121
return goquery.NewDocumentFromReader(resp.Body)
122122
}
123123

124-
var postURLExpression = regexp.MustCompile("^/forum/view-thread/([0-9]+)/page/([0-9]+)#p([0-9]+)")
124+
var postURLExpression = regexp.MustCompile("^/forum/view-post/([0-9]+)")
125125
var threadURLExpression = regexp.MustCompile("^/forum/view-thread/([0-9]+)")
126126
var forumURLExpression = regexp.MustCompile("^/forum/view-forum/([0-9]+)")
127127

@@ -152,12 +152,10 @@ func ScrapeForumPosts(doc *goquery.Document, locale *Locale, timezone *time.Loca
152152
href := sel.AttrOr("href", "")
153153
if match := postURLExpression.FindStringSubmatch(href); match != nil {
154154
n, _ := strconv.Atoi(match[1])
155-
post.ThreadId = n
156-
n, _ = strconv.Atoi(match[2])
157-
post.PageNumber = n
158-
n, _ = strconv.Atoi(match[3])
159155
post.Id = n
160156
} else if match := threadURLExpression.FindStringSubmatch(href); match != nil {
157+
n, _ := strconv.Atoi(match[1])
158+
post.ThreadId = n
161159
post.ThreadTitle = sel.Text()
162160
} else if match := forumURLExpression.FindStringSubmatch(href); match != nil {
163161
n, _ := strconv.Atoi(match[1])

‎server/forum_indexer_test.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@ func TestScrapeForumPosts(t *testing.T) {
3333
assert.Equal(t, "Chris", p.Poster)
3434
assert.Equal(t, "Photos of the Fan Meetup", p.ThreadTitle)
3535
assert.Equal(t, "Announcements", p.ForumName)
36-
assert.Equal(t, 1, p.PageNumber)
3736
assert.Equal(t, "we had a great ti<strong>m</strong>e too!", p.BodyHTML)
3837
assert.Equal(t, int64(1486332365), p.Time.Unix())
3938
}

‎server/forum_post.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@ type ForumPost struct {
1313
Poster string `json:"poster"`
1414
ThreadId int `json:"thread_id"`
1515
ThreadTitle string `json:"thread_title"`
16-
PageNumber int `json:"page_number"`
1716
ForumId int `json:"forum_id"`
1817
ForumName string `json:"forum_name"`
1918
}

‎server/testdata/forum-posts.html‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,12 +69,12 @@
6969
<div class="layoutBox1 layoutBoxFull defaultTheme">
7070
<div class="topBar first last"><div class="breadcrumb"><a href="/account/view-profile/Chris">View Profile</a><span class="separator">»</span>Recent posts by Chris</div></div>
7171
<div class="layoutBoxContent"><table class="forumTable forumPostListTable"><tr><td class="content-container"><div class="content">we had a great ti<strong>m</strong>e too!</div><div class="signature">Lead Developer. Follow us on: <a href="http://www.twitter.com/pathofexile">Twitter</a> | <a href="http://www.youtube.com/grindinggear">YouTube</a> | <a href="http://www.facebook.com/pathofexile">Facebook</a> | Contact <a href="http://www.pathofexile.com/support">Support</a> if you need help!<br />
72-
<img src="https://p7p4m6s5.ssl.hwcdn.net/public/chris/maps3.png" alt="maps3" /></div></td><td class="post_info"><div><div class="centered"><a class="bright bold medium" href="/forum/view-thread/1830139/page/1#p14168107">View post</a></div> <div class="avatar frame1">
72+
<img src="https://p7p4m6s5.ssl.hwcdn.net/public/chris/maps3.png" alt="maps3" /></div></td><td class="post_info"><div><div class="centered"><a class="bright bold medium" href="/forum/view-post/14168107">View post</a></div> <div class="avatar frame1">
7373
<img src="https://web.poecdn.com/gen/image/YTozOntpOjA7aTowO2k6/MjthOjI6e3M6MjoiaWQi/O2k6Mjc3O3M6NDoic2l6/ZSI7czo2OiJhdmF0YXIi/O31pOjE7aToxO30,/f28981be51/Path_of_Exile_Gallery_Image.jpg" alt="Avatar" />
7474
</div><div class="posted-by"><a class="posted-by-link" href="#p14168107">Posted by</a> <span class="profile-link staff post_by_account" ><a href="/account/view-profile/Chris"><img class="achievement" src="https://web.poecdn.com/image/icons/achievements/7.png?v=11" alt="Completed 7 Challenges" title="Completed 7 Challenges" />Chris</a></span><br>on <span class="post_date">Feb 5, 2017, 2:06:05 PM</span><div class="roleLabel staffText">Grinding Gear Games</div></div><div><span class="bright">Forum:</span> <a href="/forum/view-forum/54">Announcements</a></div><div><span class="bright">Thread:</span> <a href="/forum/view-thread/1830139">Photos of the Fan Meetup</a></div></div></td></tr><tr class="even"><td class="content-container"><div class="content">Yesterday, we attended the San Francisco fan meetup and had a great time! It was our busiest one yet, and we were able to stay for three hours answering questions and meeting fans. Today's news post has some photos of the event! <a href="http://pathofexile.com/forum/view-thread/1830139">Read More</a>.
7575

7676
</div><div class="signature">Lead Developer. Follow us on: <a href="http://www.twitter.com/pathofexile">Twitter</a> | <a href="http://www.youtube.com/grindinggear">YouTube</a> | <a href="http://www.facebook.com/pathofexile">Facebook</a> | Contact <a href="http://www.pathofexile.com/support">Support</a> if you need help!<br />
77-
<img src="https://p7p4m6s5.ssl.hwcdn.net/public/chris/maps3.png" alt="maps3" /></div><div class="last_edited_by">Last edited by Chris on Feb 6, 2017 8:31:55 AM</div></td><td class="post_info"><div><div class="centered"><a class="bright bold medium" href="/forum/view-thread/1830139/page/1#p14168056">View post</a></div> <div class="avatar frame1">
77+
<img src="https://p7p4m6s5.ssl.hwcdn.net/public/chris/maps3.png" alt="maps3" /></div><div class="last_edited_by">Last edited by Chris on Feb 6, 2017 8:31:55 AM</div></td><td class="post_info"><div><div class="centered"><a class="bright bold medium" href="/forum/view-post/p14168056">View post</a></div> <div class="avatar frame1">
7878
<img src="https://web.poecdn.com/gen/image/YTozOntpOjA7aTowO2k6/MjthOjI6e3M6MjoiaWQi/O2k6Mjc3O3M6NDoic2l6/ZSI7czo2OiJhdmF0YXIi/O31pOjE7aToxO30,/f28981be51/Path_of_Exile_Gallery_Image.jpg" alt="Avatar" />
7979
</div><div class="posted-by"><a class="posted-by-link" href="#p14168056">Posted by</a> <span class="profile-link staff post_by_account" ><a href="/account/view-profile/Chris"><img class="achievement" src="https://web.poecdn.com/image/icons/achievements/7.png?v=11" alt="Completed 7 Challenges" title="Completed 7 Challenges" />Chris</a></span><br>on <span class="post_date">Feb 5, 2017, 1:48:29 PM</span><div class="roleLabel staffText">Grinding Gear Games</div></div><div><span class="bright">Forum:</span> <a href="/forum/view-forum/54">Announcements</a></div><div><span class="bright">Thread:</span> <a href="/forum/view-thread/1830139">Photos of the Fan Meetup</a></div></div></td></tr><tr><td class="content-container"><div class="content">Chris, Jonathan and Nick are going to be in San Francisco next week to hold a press tour for various upcoming announcements. If you're interested in meeting with them, they'll be hosting a fan meet-up on Saturday February 4th at 4pm (PST). If you're in or around the area, you should come on by! <a href="http://pathofexile.com/forum/view-thread/1827130">Read More</a>.
8080

0 commit comments

Comments
 (0)