support reddit again, add error handling for forum indexer

Author: ccbrown

main.go

@@ -20,6 +20,7 @@ func main() {
 	pflag.String("db", "./gggtracker.db", "the database file path")
 	pflag.String("dynamodb-table", "", "if given, DynamoDB will be used instead of a database file")
 	pflag.String("forumsession", "", "the POESESSID cookie for a forum session")
+	pflag.String("reddit-auth", "", "the APPLICATION:SECRET to use as Reddit auth")
 	viper.BindPFlags(pflag.CommandLine)
 	pflag.Parse()
 
@@ -43,13 +44,16 @@ func main() {
 	}
 	defer db.Close()
 
-	redditIndexer, err := server.NewRedditIndexer(server.RedditIndexerConfiguration{
-		Database: db,
-	})
-	if err != nil {
-		log.Fatal(err)
+	if viper.GetString("reddit-auth") != "" {
+		redditIndexer, err := server.NewRedditIndexer(server.RedditIndexerConfiguration{
+			Database: db,
+			Auth:     viper.GetString("reddit-auth"),
+		})
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer redditIndexer.Close()
 	}
-	defer redditIndexer.Close()
 
 	if viper.GetString("forumsession") != "" {
 		forumIndexer, err := server.NewForumIndexer(server.ForumIndexerConfiguration{

server/forum_indexer.go

@@ -147,6 +147,10 @@ func ScrapeForumPosts(doc *goquery.Document, timezone *time.Location) ([]*ForumP
 
 	err := error(nil)
 
+	if doc.Find(".forumPostListTable").Length() == 0 {
+		return nil, errors.New("forum post list not found")
+	}
+
 	doc.Find(".forumPostListTable > tbody > tr").EachWithBreak(func(i int, sel *goquery.Selection) bool {
 		post := &ForumPost{
 			Poster: sel.Find(".post_by_account").Text(),

server/forum_indexer_test.go

@@ -35,6 +35,22 @@ func TestScrapeForumPosts(t *testing.T) {
 	assert.Equal(t, "Announcements", p.ForumName)
 	assert.Equal(t, "we had a great ti<strong>m</strong>e too!", p.BodyHTML)
 	assert.Equal(t, int64(1486332365), p.Time.Unix())
+
+	t.Run("LoggedOut", func(t *testing.T) {
+		f, err := os.Open("testdata/forum-posts-logged-out.html")
+		require.NoError(t, err)
+		defer f.Close()
+
+		doc, err := goquery.NewDocumentFromReader(f)
+		require.NoError(t, err)
+
+		tz, err := time.LoadLocation("America/Los_Angeles")
+		require.NoError(t, err)
+
+		posts, err := ScrapeForumPosts(doc, tz)
+		require.Error(t, err)
+		assert.Equal(t, 0, len(posts))
+	})
 }
 
 func TestScrapeForumTimezone(t *testing.T) {

server/reddit_indexer.go

@@ -13,6 +13,7 @@ import (
 
 type RedditIndexerConfiguration struct {
 	Database Database
+	Auth     string
 }
 
 type RedditIndexer struct {
@@ -33,6 +34,8 @@ func (indexer *RedditIndexer) Close() {
 	indexer.closeSignal <- true
 }
 
+const redditRequestInterval = time.Second * 8
+
 func (indexer *RedditIndexer) run() {
 	log.Info("starting reddit indexer")
 
@@ -58,7 +61,7 @@ func (indexer *RedditIndexer) run() {
 			if next >= len(users) {
 				next = 0
 			}
-			time.Sleep(time.Second * 3)
+			time.Sleep(redditRequestInterval)
 		}
 	}
 }
@@ -136,6 +139,9 @@ func (indexer *RedditIndexer) redditActivity(user string, page string) ([]Activi
 		return nil, "", err
 	}
 	req.Header.Add("User-Agent", "GGG Tracker (https://github.com/ccbrown/gggtracker) by /u/rz2yoj")
+	if parts := strings.Split(indexer.configuration.Auth, ":"); len(parts) == 2 {
+		req.SetBasicAuth(parts[0], parts[1])
+	}
 
 	resp, err := client.Do(req)
 	if err != nil {
@@ -184,7 +190,7 @@ func (indexer *RedditIndexer) index(user string) error {
 		if done {
 			break
 		}
-		time.Sleep(time.Second * 3)
+		time.Sleep(redditRequestInterval)
 	}
 
 	if len(activity) == 0 {