From 7645e678e7cded75e5ce6c1f7487f71b273fe6c7 Mon Sep 17 00:00:00 2001 From: nethew Date: Wed, 1 Nov 2017 03:13:51 +0100 Subject: [PATCH 01/90] Update style.css to solve too wide title column (#13) * Update style.css to solve too wide title column Solves too wide title column like in the case of https://www.pathofexile.com/forum/view-post/14987094 Also makes the title always on one line - nowrap, which keeps the line height always the same. You can reconsider this if you want to. * remove nowrap for now * update css cache buster --- server/index_handler.go | 2 +- server/static/style.css | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/server/index_handler.go b/server/index_handler.go index 404cfb9..0b3df1f 100644 --- a/server/index_handler.go +++ b/server/index_handler.go @@ -11,7 +11,7 @@ var index = ` GGG Tracker - + diff --git a/server/static/style.css b/server/static/style.css index c03a29c..f319fe4 100644 --- a/server/static/style.css +++ b/server/static/style.css @@ -197,6 +197,11 @@ td.icon img { transform: translate(-50%, -50%); } +td.title { + max-width: 450px; + overflow: hidden; +} + td.body { background-color: #262626; padding: 10px; From 97f0d07230829c4f836a66419c61dd093eecb65b Mon Sep 17 00:00:00 2001 From: Chris Date: Tue, 31 Oct 2017 21:24:02 -0500 Subject: [PATCH 02/90] Change readme link from HTML to markdown --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 96d1ea7..992b553 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Welcome! [![Build Status](https://travis-ci.org/ccbrown/gggtracker.svg?branch=master)](https://travis-ci.org/ccbrown/gggtracker) -This is the repository for gggtracker.com. If there's something you think the site is missing, please either a.) open an issue to request the feature or b.) develop the feature yourself and put in a pull request. +This is the repository for [gggtracker.com](https://gggtracker.com). If there's something you think the site is missing, please either a.) open an issue to request the feature or b.) develop the feature yourself and put in a pull request. ### Development From f3a9d343bf3fef40356ad961973686dee37e31d5 Mon Sep 17 00:00:00 2001 From: Christopher Brown Date: Wed, 6 Dec 2017 02:32:31 -0600 Subject: [PATCH 03/90] prevent images from stretching the table width --- server/index_handler.go | 2 +- server/static/style.css | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/server/index_handler.go b/server/index_handler.go index 0b3df1f..8716dc7 100644 --- a/server/index_handler.go +++ b/server/index_handler.go @@ -11,7 +11,7 @@ var index = ` GGG Tracker - + diff --git a/server/static/style.css b/server/static/style.css index f319fe4..f7c23fd 100644 --- a/server/static/style.css +++ b/server/static/style.css @@ -214,6 +214,10 @@ td.body blockquote { padding: 0.5em 10px; } +td.body img { + max-width: 100%; +} + .spoilerHidden .spoilerContent { display: none; } From b548ffab61c16f262f5470aff3165ab0cd808fa0 Mon Sep 17 00:00:00 2001 From: Christopher Brown Date: Sat, 9 Dec 2017 13:27:19 -0600 Subject: [PATCH 04/90] update scraping for new forum HTML --- server/forum_indexer.go | 2 +- server/testdata/forum-preferences.html | 236 ++++++++++++------------- 2 files changed, 112 insertions(+), 126 deletions(-) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index ede70ad..98f9f5b 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -210,7 +210,7 @@ func (indexer *ForumIndexer) index(poster string, timezone *time.Location) { } func ScrapeForumTimezone(doc *goquery.Document) (*time.Location, error) { - sel := doc.Find("#timezone option[selected]") + sel := doc.Find(`select[name="preferences[timezone]"] option[selected]`) if sel == nil || sel.AttrOr("value", "") == "" { return nil, errors.New("unable to find timezone selection") } diff --git a/server/testdata/forum-preferences.html b/server/testdata/forum-preferences.html index 3ce51ab..a2ec6c2 100644 --- a/server/testdata/forum-preferences.html +++ b/server/testdata/forum-preferences.html @@ -70,131 +70,117 @@
My Account»Preferences

Account Preferences

Looking to change your Supporter Titles? You can now find them here.

-
-
- - - - - - - - - - - -
-
-
- -
Check this box to receive a Message when someone in the forum quotes you.
- -
Check this box to receive the Path of Exile newsletter via email
- -
Check this box to receive account status related email.
-
-
+
+
+
+
+
Check this box to receive a Message when someone in the forum quotes you.
+
Check this box to receive the Path of Exile newsletter via email.
+
Check this box to receive account status related email.
+
+
+
From ebdeff62ee35b6f0dc6d7503e7523b358c45b70d Mon Sep 17 00:00:00 2001 From: Christopher Brown Date: Fri, 19 Jan 2018 17:43:41 -0600 Subject: [PATCH 05/90] add forum user Jeff_GGG, reddit user Daniel_GGG --- server/forum_indexer.go | 1 + server/reddit_indexer.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index 98f9f5b..1405c6a 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -51,6 +51,7 @@ func (indexer *ForumIndexer) run() { "Alex_GGG", "Alexcc_GGG", "Andy", "CJ_GGG", "Eben_GGG", "Emma_GGG", "Ethan_GGG", "Fitzy_GGG", "Hartlin_GGG", "Jake_GGG", "Lionel_GGG", "Melissa_GGG", "MikeP_GGG", "Novynn", "Rachel_GGG", "Rob_GGG", "Roman_GGG", "Sarah_GGG", "SarahB_GGG", "Tom_GGG", "Natalia_GGG", + "Jeff_GGG", } next := 0 diff --git a/server/reddit_indexer.go b/server/reddit_indexer.go index 68ce28c..eab7688 100644 --- a/server/reddit_indexer.go +++ b/server/reddit_indexer.go @@ -39,7 +39,7 @@ func (indexer *RedditIndexer) run() { users := []string{ "chris_wilson", "Bex_GGG", "Negitivefrags", "Omnitect", "qarldev", "BrianWeissman_GGG", "Mark_GGG", "RhysGGG", "Dan_GGG", "Rory_Rackham", "Blake_GGG", "Fitzy_GGG", "Hartlin_GGG", - "Hrishi_GGG", "Baltic_GGG", "KamilOrmanJanowski", + "Hrishi_GGG", "Baltic_GGG", "KamilOrmanJanowski", "Daniel_GGG", } next := 0 From 3c881e33f11a4cf058ee6afd5d96e86a8f10b0f0 Mon Sep 17 00:00:00 2001 From: Chris Date: Mon, 5 Mar 2018 05:28:14 -0600 Subject: [PATCH 06/90] add support for multiple languages (#19) --- .dockerignore | 1 + .gitignore | 1 + Dockerfile | 9 +- Gopkg.lock | 224 +++++++++++++++++++++++++++++++++++++ Gopkg.toml | 58 ++++++++++ main.go | 2 +- server/activity.go | 42 +++++++ server/activity_handler.go | 2 +- server/database.go | 19 +++- server/database_test.go | 6 +- server/forum_indexer.go | 139 +++++++++++++++++------ server/forum_post.go | 3 +- server/index_handler.go | 2 +- server/reddit_indexer.go | 2 +- server/rss_handler.go | 2 +- server/static/index.js | 10 +- 16 files changed, 462 insertions(+), 60 deletions(-) create mode 100644 Gopkg.lock create mode 100644 Gopkg.toml diff --git a/.dockerignore b/.dockerignore index bbfd30f..88cef9a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,4 @@ * +!Gopkg.* !*.go !server/* diff --git a/.gitignore b/.gitignore index 3990fde..f744723 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.db gggtracker +/vendor diff --git a/Dockerfile b/Dockerfile index f94d04f..3c7f0b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,12 @@ -FROM golang:alpine +FROM golang:1.10-alpine WORKDIR /go/src/github.com/ccbrown/gggtracker -ADD . . -RUN apk add --no-cache git && go get -t ./... +RUN apk add --no-cache git +RUN wget -O - https://raw.githubusercontent.com/golang/dep/master/install.sh | sh + +ADD . . +RUN dep ensure RUN go vet . && go test -v ./... RUN go build . diff --git a/Gopkg.lock b/Gopkg.lock new file mode 100644 index 0000000..2c713f1 --- /dev/null +++ b/Gopkg.lock @@ -0,0 +1,224 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + name = "github.com/PuerkitoBio/goquery" + packages = ["."] + revision = "09540e565986583836b5fc792fe951ec5fd201dd" + version = "v1.3.0" + +[[projects]] + name = "github.com/andybalholm/cascadia" + packages = ["."] + revision = "901648c87902174f774fac311d7f176f8647bdaa" + version = "v1.0.0" + +[[projects]] + name = "github.com/boltdb/bolt" + packages = ["."] + revision = "2f1ce7a837dcb8da3ec595b1dac9d0632f0f99e8" + version = "v1.3.1" + +[[projects]] + name = "github.com/davecgh/go-spew" + packages = ["spew"] + revision = "346938d642f2ec3594ed81d874461961cd0faa76" + version = "v1.1.0" + +[[projects]] + name = "github.com/dgrijalva/jwt-go" + packages = ["."] + revision = "dbeaa9332f19a944acb5736b4456cfcc02140e29" + version = "v3.1.0" + +[[projects]] + name = "github.com/fsnotify/fsnotify" + packages = ["."] + revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" + version = "v1.4.7" + +[[projects]] + branch = "master" + name = "github.com/hashicorp/hcl" + packages = [ + ".", + "hcl/ast", + "hcl/parser", + "hcl/scanner", + "hcl/strconv", + "hcl/token", + "json/parser", + "json/scanner", + "json/token" + ] + revision = "23c074d0eceb2b8a5bfdbb271ab780cde70f05a8" + +[[projects]] + name = "github.com/labstack/echo" + packages = [ + ".", + "middleware" + ] + revision = "b338075a0fc6e1a0683dbf03d09b4957a289e26f" + version = "3.2.6" + +[[projects]] + name = "github.com/labstack/gommon" + packages = [ + "bytes", + "color", + "log", + "random" + ] + revision = "57409ada9da0f2afad6664c49502f8c50fbd8476" + version = "0.2.3" + +[[projects]] + name = "github.com/magiconair/properties" + packages = ["."] + revision = "c3beff4c2358b44d0493c7dda585e7db7ff28ae6" + version = "v1.7.6" + +[[projects]] + name = "github.com/mattn/go-colorable" + packages = ["."] + revision = "167de6bfdfba052fa6b2d3664c8f5272e23c9072" + version = "v0.0.9" + +[[projects]] + name = "github.com/mattn/go-isatty" + packages = ["."] + revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39" + version = "v0.0.3" + +[[projects]] + branch = "master" + name = "github.com/mitchellh/mapstructure" + packages = ["."] + revision = "00c29f56e2386353d58c599509e8dc3801b0d716" + +[[projects]] + name = "github.com/pelletier/go-toml" + packages = ["."] + revision = "acdc4509485b587f5e675510c4f2c63e90ff68a8" + version = "v1.1.0" + +[[projects]] + name = "github.com/pmezard/go-difflib" + packages = ["difflib"] + revision = "792786c7400a136282c1664665ae0a8db921c6c2" + version = "v1.0.0" + +[[projects]] + name = "github.com/sirupsen/logrus" + packages = ["."] + revision = "d682213848ed68c0a260ca37d6dd5ace8423f5ba" + version = "v1.0.4" + +[[projects]] + name = "github.com/spf13/afero" + packages = [ + ".", + "mem" + ] + revision = "bb8f1927f2a9d3ab41c9340aa034f6b803f4359c" + version = "v1.0.2" + +[[projects]] + name = "github.com/spf13/cast" + packages = ["."] + revision = "8965335b8c7107321228e3e3702cab9832751bac" + version = "v1.2.0" + +[[projects]] + branch = "master" + name = "github.com/spf13/jwalterweatherman" + packages = ["."] + revision = "7c0cea34c8ece3fbeb2b27ab9b59511d360fb394" + +[[projects]] + name = "github.com/spf13/pflag" + packages = ["."] + revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66" + version = "v1.0.0" + +[[projects]] + name = "github.com/spf13/viper" + packages = ["."] + revision = "25b30aa063fc18e48662b86996252eabdcf2f0c7" + version = "v1.0.0" + +[[projects]] + name = "github.com/stretchr/testify" + packages = [ + "assert", + "require" + ] + revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71" + version = "v1.2.1" + +[[projects]] + branch = "master" + name = "github.com/valyala/bytebufferpool" + packages = ["."] + revision = "e746df99fe4a3986f4d4f79e13c1e0117ce9c2f7" + +[[projects]] + branch = "master" + name = "github.com/valyala/fasttemplate" + packages = ["."] + revision = "dcecefd839c4193db0d35b88ec65b4c12d360ab0" + +[[projects]] + branch = "master" + name = "golang.org/x/crypto" + packages = [ + "acme", + "acme/autocert", + "ssh/terminal" + ] + revision = "91a49db82a88618983a78a06c1cbd4e00ab749ab" + +[[projects]] + branch = "master" + name = "golang.org/x/net" + packages = [ + "html", + "html/atom" + ] + revision = "22ae77b79946ea320088417e4d50825671d82d57" + +[[projects]] + branch = "master" + name = "golang.org/x/sys" + packages = [ + "unix", + "windows" + ] + revision = "dd2ff4accc098aceecb86b36eaa7829b2a17b1c9" + +[[projects]] + name = "golang.org/x/text" + packages = [ + "internal/gen", + "internal/triegen", + "internal/ucd", + "transform", + "unicode/cldr", + "unicode/norm" + ] + revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" + version = "v0.3.0" + +[[projects]] + name = "gopkg.in/yaml.v2" + packages = ["."] + revision = "7f97868eec74b32b0982dd158a51a446d1da7eb5" + version = "v2.1.1" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "fb3f71f8b31a4ef69f8c12df7c17800632ad541fb57db8b99989ec33570b63fd" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml new file mode 100644 index 0000000..e04d3c1 --- /dev/null +++ b/Gopkg.toml @@ -0,0 +1,58 @@ +# Gopkg.toml example +# +# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html +# for detailed Gopkg.toml documentation. +# +# required = ["github.com/user/thing/cmd/thing"] +# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] +# +# [[constraint]] +# name = "github.com/user/project" +# version = "1.0.0" +# +# [[constraint]] +# name = "github.com/user/project2" +# branch = "dev" +# source = "github.com/myfork/project2" +# +# [[override]] +# name = "github.com/x/y" +# version = "2.4.0" +# +# [prune] +# non-go = false +# go-tests = true +# unused-packages = true + + +[[constraint]] + name = "github.com/PuerkitoBio/goquery" + version = "1.3.0" + +[[constraint]] + name = "github.com/boltdb/bolt" + version = "1.3.1" + +[[constraint]] + name = "github.com/labstack/echo" + version = "3.2.6" + +[[constraint]] + name = "github.com/sirupsen/logrus" + version = "1.0.4" + +[[constraint]] + name = "github.com/spf13/pflag" + version = "1.0.0" + +[[constraint]] + name = "github.com/spf13/viper" + version = "1.0.0" + +[[constraint]] + name = "github.com/stretchr/testify" + version = "1.2.1" + +[prune] + go-tests = true + unused-packages = true diff --git a/main.go b/main.go index 7af8c20..56cd657 100644 --- a/main.go +++ b/main.go @@ -5,9 +5,9 @@ import ( "net/http" "path" - log "github.com/Sirupsen/logrus" "github.com/labstack/echo" "github.com/labstack/echo/middleware" + log "github.com/sirupsen/logrus" "github.com/spf13/pflag" "github.com/spf13/viper" diff --git a/server/activity.go b/server/activity.go index 0c8fbe4..427020b 100644 --- a/server/activity.go +++ b/server/activity.go @@ -1,6 +1,8 @@ package server import ( + "net/http" + "strings" "time" ) @@ -8,3 +10,43 @@ type Activity interface { ActivityTime() time.Time ActivityKey() uint32 } + +func ActivityFilterForRequest(r *http.Request) func(Activity) bool { + subdomain := "" + if r.Host != "" { + subdomain = strings.Split(r.Host, ".")[0] + } + + includeReddit := false + includeForumHost := map[string]bool{} + + switch subdomain { + case "br": + includeForumHost["br.pathofexile.com"] = true + case "ru": + includeForumHost["ru.pathofexile.com"] = true + case "th": + includeForumHost["th.pathofexile.com"] = true + case "de": + includeForumHost["de.pathofexile.com"] = true + case "fr": + includeForumHost["fr.pathofexile.com"] = true + case "es": + includeForumHost["es.pathofexile.com"] = true + default: + includeReddit = true + includeForumHost["www.pathofexile.com"] = true + } + + return func(a Activity) bool { + switch a := a.(type) { + case *ForumPost: + return includeForumHost[a.Host] + case *RedditComment: + return includeReddit + case *RedditPost: + return includeReddit + } + return false + } +} diff --git a/server/activity_handler.go b/server/activity_handler.go index 6530cc1..f75ebb9 100644 --- a/server/activity_handler.go +++ b/server/activity_handler.go @@ -16,7 +16,7 @@ type jsonResponse struct { func ActivityHandler(db *Database) echo.HandlerFunc { return func(c echo.Context) error { - activity, next := db.Activity(c.QueryParam("next"), 50) + activity, next := db.Activity(c.QueryParam("next"), 50, ActivityFilterForRequest(c.Request())) response := jsonResponse{ Next: next, } diff --git a/server/database.go b/server/database.go index 00a4891..a79679b 100644 --- a/server/database.go +++ b/server/database.go @@ -69,7 +69,7 @@ func (db *Database) AddActivity(activity []Activity) { } } -func (db *Database) Activity(start string, count int) ([]Activity, string) { +func (db *Database) Activity(start string, count int, filter func(Activity) bool) ([]Activity, string) { ret := []Activity(nil) next := "" err := db.db.View(func(tx *bolt.Tx) error { @@ -88,7 +88,8 @@ func (db *Database) Activity(start string, count int) ([]Activity, string) { } } } - for i := 0; i < count && k != nil; i++ { + for len(ret) < count && k != nil { + var activity Activity switch k[5] { case ForumPostType: post := &ForumPost{} @@ -96,23 +97,29 @@ func (db *Database) Activity(start string, count int) ([]Activity, string) { if err != nil { return err } - ret = append(ret, post) + if post.Host == "" { + post.Host = "www.pathofexile.com" + } + activity = post case RedditCommentType: comment := &RedditComment{} err := json.Unmarshal(v, comment) if err != nil { return err } - ret = append(ret, comment) + activity = comment case RedditPostType: post := &RedditPost{} err := json.Unmarshal(v, post) if err != nil { return err } - ret = append(ret, post) + activity = post + } + if filter == nil || filter(activity) { + ret = append(ret, activity) + next = base64.RawURLEncoding.EncodeToString(k) } - next = base64.RawURLEncoding.EncodeToString(k) k, v = c.Prev() } return nil diff --git a/server/database_test.go b/server/database_test.go index 55e7847..96153b6 100644 --- a/server/database_test.go +++ b/server/database_test.go @@ -34,18 +34,18 @@ func TestDatabase_ForumPosts(t *testing.T) { db.AddActivity([]Activity{post1, post2}) - posts, next := db.Activity("", 1) + posts, next := db.Activity("", 1, nil) require.Equal(t, 1, len(posts)) assert.Equal(t, post1.Id, posts[0].(*ForumPost).Id) assert.Equal(t, post1.Poster, posts[0].(*ForumPost).Poster) assert.Equal(t, post1.Time.Unix(), posts[0].(*ForumPost).Time.Unix()) - posts, next = db.Activity(next, 1) + posts, next = db.Activity(next, 1, nil) require.Equal(t, 1, len(posts)) assert.Equal(t, post2.Id, posts[0].(*ForumPost).Id) assert.Equal(t, post2.Poster, posts[0].(*ForumPost).Poster) assert.Equal(t, post2.Time.Unix(), posts[0].(*ForumPost).Time.Unix()) - posts, _ = db.Activity(next, 1) + posts, _ = db.Activity(next, 1, nil) require.Equal(t, 0, len(posts)) } diff --git a/server/forum_indexer.go b/server/forum_indexer.go index 1405c6a..de4c4bc 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -9,16 +9,17 @@ import ( "regexp" "strconv" "strings" + "sync" "time" "github.com/PuerkitoBio/goquery" - log "github.com/Sirupsen/logrus" + log "github.com/sirupsen/logrus" ) type ForumIndexer struct { configuration ForumIndexerConfiguration - closeSignal chan bool + closeSignal chan struct{} } type ForumIndexerConfiguration struct { @@ -29,20 +30,30 @@ type ForumIndexerConfiguration struct { func NewForumIndexer(configuration ForumIndexerConfiguration) (*ForumIndexer, error) { ret := &ForumIndexer{ configuration: configuration, - closeSignal: make(chan bool), + closeSignal: make(chan struct{}), } go ret.run() return ret, nil } func (indexer *ForumIndexer) Close() { - indexer.closeSignal <- true + close(indexer.closeSignal) } func (indexer *ForumIndexer) run() { log.Info("starting forum indexer") - posters := []string{ + hosts := []string{ + "www.pathofexile.com", + "br.pathofexile.com", + "ru.pathofexile.com", + "th.pathofexile.com", + "de.pathofexile.com", + "fr.pathofexile.com", + "es.pathofexile.com", + } + + accounts := []string{ "Chris", "Jonathan", "Erik", "Mark_GGG", "Samantha", "Rory", "Rhys", "Qarl", "Andrew_GGG", "Damien_GGG", "Joel_GGG", "Ari", "Thomas", "BrianWeissman", "Edwin_GGG", "Support", "Dylan", "MaxS", "Ammon_GGG", "Jess_GGG", "Robbie_GGG", "GGG_Neon", "Jason_GGG", "Henry_GGG", @@ -53,39 +64,48 @@ func (indexer *ForumIndexer) run() { "Rachel_GGG", "Rob_GGG", "Roman_GGG", "Sarah_GGG", "SarahB_GGG", "Tom_GGG", "Natalia_GGG", "Jeff_GGG", } - next := 0 timezone := (*time.Location)(nil) - for { + for timezone == nil { select { case <-indexer.closeSignal: return default: - if timezone == nil { - tz, err := indexer.sessionTimezone() - if err != nil { - log.WithError(err).Error("error getting forum timezone") - } else { - timezone = tz - log.WithFields(log.Fields{ - "timezone": timezone, - }).Info("forum timezone obtained") - } + if tz, err := indexer.sessionTimezone(); err != nil { + log.WithError(err).Error("error getting forum timezone") } else { - indexer.index(posters[next], timezone) - next += 1 - if next >= len(posters) { - next = 0 - } + timezone = tz + log.WithField("timezone", timezone).Info("forum timezone obtained") + break } time.Sleep(time.Second) } } + + var wg sync.WaitGroup + wg.Add(len(hosts)) + + for _, host := range hosts { + host := host + go func() { + for _, account := range accounts { + select { + case <-indexer.closeSignal: + return + default: + indexer.index(host, account, timezone) + time.Sleep(time.Second) + } + } + }() + } + + wg.Wait() } -func (indexer *ForumIndexer) requestDocument(resource string) (*goquery.Document, error) { - urlString := fmt.Sprintf("https://www.pathofexile.com/%v", strings.TrimPrefix(resource, "/")) +func (indexer *ForumIndexer) requestDocument(host, resource string) (*goquery.Document, error) { + urlString := fmt.Sprintf("https://%v/%v", host, strings.TrimPrefix(resource, "/")) jar, _ := cookiejar.New(nil) u, _ := url.Parse(urlString) jar.SetCookies(u, []*http.Cookie{ @@ -113,6 +133,33 @@ var postURLExpression = regexp.MustCompile("^/forum/view-thread/([0-9]+)/page/([ var threadURLExpression = regexp.MustCompile("^/forum/view-thread/([0-9]+)") var forumURLExpression = regexp.MustCompile("^/forum/view-forum/([0-9]+)") +var monthReplacer = strings.NewReplacer( + "ม.ค.", "Jan", + "ก.พ.", "Feb", + "มี.ค.", "Mar", + "เม.ย.", "Apr", + "พ.ค.", "May", + "มิ.ย.", "Jun", + "ก.ค.", "Jul", + "ส.ค.", "Aug", + "ก.ย.", "Sep", + "ต.ค.", "Oct", + "พ.ย.", "Nov", + "ธ.ค.", "Dec", + "janv.", "Jan", + "févr.", "Feb", + "mars", "Mar", + "avril", "Apr", + "mai", "May", + "juin", "Jun", + "juil.", "Jul", + "août", "Aug", + "sept.", "Sep", + "oct.", "Oct", + "nov.", "Nov", + "déc.", "Dec", +) + func ScrapeForumPosts(doc *goquery.Document, timezone *time.Location) ([]*ForumPost, error) { posts := []*ForumPost(nil) @@ -129,12 +176,24 @@ func ScrapeForumPosts(doc *goquery.Document, timezone *time.Location) ([]*ForumP } post.BodyHTML = body - text := sel.Find(".post_date").Text() - t, err := time.ParseInLocation("Jan _2, 2006 3:04:05 PM", text, timezone) - if err != nil { + timeText := monthReplacer.Replace(sel.Find(".post_date").Text()) + + for _, format := range []string{ + "Jan _2, 2006 3:04:05 PM", + "2/1/2006 15:04:05", + "2.1.2006 15:04:05", + "_2 Jan 2006, 15:04:05", + "_2 Jan 2006 15:04:05", + } { + if t, err := time.ParseInLocation(format, timeText, timezone); err == nil { + post.Time = t + break + } + } + if post.Time.IsZero() { + log.WithField("text", timeText).Error("unable to parse time") return false } - post.Time = t sel.Find("a").Each(func(i int, sel *goquery.Selection) { href := sel.AttrOr("href", "") @@ -165,16 +224,24 @@ func ScrapeForumPosts(doc *goquery.Document, timezone *time.Location) ([]*ForumP return posts, nil } -func (indexer *ForumIndexer) forumPosts(poster string, page int, timezone *time.Location) ([]*ForumPost, error) { - doc, err := indexer.requestDocument(fmt.Sprintf("/account/view-posts/%v/page/%v", poster, page)) +func (indexer *ForumIndexer) forumPosts(host, poster string, page int, timezone *time.Location) ([]*ForumPost, error) { + doc, err := indexer.requestDocument(host, fmt.Sprintf("/account/view-posts/%v/page/%v", poster, page)) if err != nil { return nil, err } - return ScrapeForumPosts(doc, timezone) + posts, err := ScrapeForumPosts(doc, timezone) + if err != nil { + return nil, err + } + for _, post := range posts { + post.Host = host + } + return posts, nil } -func (indexer *ForumIndexer) index(poster string, timezone *time.Location) { +func (indexer *ForumIndexer) index(host, poster string, timezone *time.Location) { logger := log.WithFields(log.Fields{ + "host": host, "poster": poster, }) @@ -182,7 +249,7 @@ func (indexer *ForumIndexer) index(poster string, timezone *time.Location) { activity := []Activity(nil) for page := 1; ; page++ { - posts, err := indexer.forumPosts(poster, page, timezone) + posts, err := indexer.forumPosts(host, poster, page, timezone) if err != nil { logger.WithError(err).Error("error requesting forum posts") } @@ -195,9 +262,7 @@ func (indexer *ForumIndexer) index(poster string, timezone *time.Location) { activity = append(activity, post) } - logger.WithFields(log.Fields{ - "count": len(posts), - }).Info("received forum posts") + logger.WithField("count", len(posts)).Info("received forum posts") if done { break @@ -219,7 +284,7 @@ func ScrapeForumTimezone(doc *goquery.Document) (*time.Location, error) { } func (indexer *ForumIndexer) sessionTimezone() (*time.Location, error) { - doc, err := indexer.requestDocument("/my-account/preferences") + doc, err := indexer.requestDocument("www.pathofexile.com", "/my-account/preferences") if err != nil { return nil, err } diff --git a/server/forum_post.go b/server/forum_post.go index 09afa7e..872b521 100644 --- a/server/forum_post.go +++ b/server/forum_post.go @@ -7,6 +7,7 @@ import ( type ForumPost struct { Id int `json:"id"` + Host string `json:"host"` BodyHTML string `json:"body_html"` Time time.Time `json:"time"` Poster string `json:"poster"` @@ -26,5 +27,5 @@ func (p *ForumPost) ActivityKey() uint32 { } func (p *ForumPost) PostURL() string { - return fmt.Sprintf("https://www.pathofexile.com/forum/view-post/%v", p.Id) + return fmt.Sprintf("https://%v/forum/view-post/%v", p.Host, p.Id) } diff --git a/server/index_handler.go b/server/index_handler.go index 8716dc7..e21be6e 100644 --- a/server/index_handler.go +++ b/server/index_handler.go @@ -63,7 +63,7 @@ var index = ` - + ` diff --git a/server/reddit_indexer.go b/server/reddit_indexer.go index eab7688..93f5358 100644 --- a/server/reddit_indexer.go +++ b/server/reddit_indexer.go @@ -8,7 +8,7 @@ import ( "strings" "time" - log "github.com/Sirupsen/logrus" + log "github.com/sirupsen/logrus" ) type RedditIndexerConfiguration struct { diff --git a/server/rss_handler.go b/server/rss_handler.go index 03540be..62dee5d 100644 --- a/server/rss_handler.go +++ b/server/rss_handler.go @@ -43,7 +43,7 @@ type rssResponse struct { func RSSHandler(db *Database) echo.HandlerFunc { return func(c echo.Context) error { - activity, _ := db.Activity(c.QueryParam("next"), 50) + activity, _ := db.Activity(c.QueryParam("next"), 50, ActivityFilterForRequest(c.Request())) response := rssResponse{ Version: "2.0", Atom: "http://www.w3.org/2005/Atom", diff --git a/server/static/index.js b/server/static/index.js index 734075f..3677f7a 100644 --- a/server/static/index.js +++ b/server/static/index.js @@ -37,7 +37,7 @@ function loadActivity() { if (type == 'forum_post') { $tr.append($('
').append($('') - .attr('href', 'https://www.pathofexile.com/forum/view-thread/' + activity.thread_id + '/filter-account-type/staff') + .attr('href', 'https://' + activity.host + '/forum/view-thread/' + activity.thread_id + '/filter-account-type/staff') .append($('')) )); } else if (type == 'reddit_comment') { @@ -54,7 +54,7 @@ function loadActivity() { if (type == 'forum_post') { $tr.append($('').append($('') - .attr('href', 'https://www.pathofexile.com/forum/view-post/' + activity.id) + .attr('href', 'https://' + activity.host + '/forum/view-post/' + activity.id) .text(activity.thread_title) )); } else if (type == "reddit_post") { @@ -71,7 +71,7 @@ function loadActivity() { if (type == 'forum_post') { $tr.append($('').append($('') - .attr('href', 'https://www.pathofexile.com/account/view-profile/' + encodeURIComponent(activity.poster)) + .attr('href', 'https://' + activity.host + '/account/view-profile/' + encodeURIComponent(activity.poster)) .text(activity.poster) )); } else { @@ -85,7 +85,7 @@ function loadActivity() { if (type == 'forum_post') { $tr.append($('').append($('') - .attr('href', 'https://www.pathofexile.com/forum/view-forum/' + encodeURIComponent(activity.forum_id)) + .attr('href', 'https://' + activity.host + '/forum/view-forum/' + encodeURIComponent(activity.forum_id)) .text(activity.forum_name) )); } else { @@ -107,7 +107,7 @@ function loadActivity() { $body.find('a').each(function() { var r = $(this).attr('href'); if (r && (r.indexOf(':') < 0 || r.indexOf('/') <= r.indexOf(':'))) { - var root = type == 'forum_post' ? 'https://www.pathofexile.com' : 'https://www.reddit.com'; + var root = type == 'forum_post' ? 'https://' + activity.host : 'https://www.reddit.com'; $(this).attr('href', root + (r[0] == '/' ? '' : '/') + r); } }); From cd7b687813ae0b47edf0c2805791a7bb59409e6f Mon Sep 17 00:00:00 2001 From: Christopher Brown Date: Mon, 5 Mar 2018 05:38:43 -0600 Subject: [PATCH 07/90] dockerfile fix --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3c7f0b7..37ab1cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,4 +10,4 @@ RUN dep ensure RUN go vet . && go test -v ./... RUN go build . -ENTRYPOINT ["gggtracker"] +ENTRYPOINT ["./gggtracker"] From 37f834e95a0dd16acf52938ad33bcc7cf5328468 Mon Sep 17 00:00:00 2001 From: Christopher Brown Date: Mon, 5 Mar 2018 15:30:21 -0600 Subject: [PATCH 08/90] forum indexing loop fix --- server/forum_indexer.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index de4c4bc..ff4daf3 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -89,13 +89,15 @@ func (indexer *ForumIndexer) run() { for _, host := range hosts { host := host go func() { - for _, account := range accounts { - select { - case <-indexer.closeSignal: - return - default: - indexer.index(host, account, timezone) - time.Sleep(time.Second) + for { + for _, account := range accounts { + select { + case <-indexer.closeSignal: + return + default: + indexer.index(host, account, timezone) + time.Sleep(time.Second) + } } } }() From 3232ef9d50950503b92b2a6777476fafa751d213 Mon Sep 17 00:00:00 2001 From: Christopher Brown Date: Mon, 5 Mar 2018 15:39:22 -0600 Subject: [PATCH 09/90] add several reddit users (thanks multiplicity!) --- server/reddit_indexer.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/reddit_indexer.go b/server/reddit_indexer.go index 93f5358..0cda96a 100644 --- a/server/reddit_indexer.go +++ b/server/reddit_indexer.go @@ -39,7 +39,8 @@ func (indexer *RedditIndexer) run() { users := []string{ "chris_wilson", "Bex_GGG", "Negitivefrags", "Omnitect", "qarldev", "BrianWeissman_GGG", "Mark_GGG", "RhysGGG", "Dan_GGG", "Rory_Rackham", "Blake_GGG", "Fitzy_GGG", "Hartlin_GGG", - "Hrishi_GGG", "Baltic_GGG", "KamilOrmanJanowski", "Daniel_GGG", + "Hrishi_GGG", "Baltic_GGG", "KamilOrmanJanowski", "Daniel_GGG", "Jeff_GGG", "NapfelGGG", + "Baltic_GGG", "Novynn", "Felipe_GGG", "Mel_GGG", "Sarah_GGG", "riandrake", "Kieren_GGG", } next := 0 From d3c266cf5c05acef81c11a0e81eb56bdc39b1c2e Mon Sep 17 00:00:00 2001 From: Christopher Brown Date: Mon, 5 Mar 2018 22:44:24 -0600 Subject: [PATCH 10/90] add localizations --- server/activity.go | 42 --------- server/activity_handler.go | 2 +- server/common.go | 15 +++ server/forum_indexer.go | 79 +++------------- server/forum_indexer_test.go | 2 +- server/index_handler.go | 38 ++++++-- server/localization.go | 141 ++++++++++++++++++++++++++++ server/rss_handler.go | 2 +- server/static/images/locales/br.png | Bin 0 -> 773 bytes server/static/images/locales/de.png | Bin 0 -> 141 bytes server/static/images/locales/es.png | Bin 0 -> 343 bytes server/static/images/locales/fr.png | Bin 0 -> 146 bytes server/static/images/locales/gb.png | Bin 0 -> 946 bytes server/static/images/locales/ru.png | Bin 0 -> 188 bytes server/static/images/locales/th.png | Bin 0 -> 182 bytes server/static/style.css | 15 +++ 16 files changed, 219 insertions(+), 117 deletions(-) create mode 100644 server/localization.go create mode 100644 server/static/images/locales/br.png create mode 100644 server/static/images/locales/de.png create mode 100644 server/static/images/locales/es.png create mode 100644 server/static/images/locales/fr.png create mode 100644 server/static/images/locales/gb.png create mode 100644 server/static/images/locales/ru.png create mode 100644 server/static/images/locales/th.png diff --git a/server/activity.go b/server/activity.go index 427020b..0c8fbe4 100644 --- a/server/activity.go +++ b/server/activity.go @@ -1,8 +1,6 @@ package server import ( - "net/http" - "strings" "time" ) @@ -10,43 +8,3 @@ type Activity interface { ActivityTime() time.Time ActivityKey() uint32 } - -func ActivityFilterForRequest(r *http.Request) func(Activity) bool { - subdomain := "" - if r.Host != "" { - subdomain = strings.Split(r.Host, ".")[0] - } - - includeReddit := false - includeForumHost := map[string]bool{} - - switch subdomain { - case "br": - includeForumHost["br.pathofexile.com"] = true - case "ru": - includeForumHost["ru.pathofexile.com"] = true - case "th": - includeForumHost["th.pathofexile.com"] = true - case "de": - includeForumHost["de.pathofexile.com"] = true - case "fr": - includeForumHost["fr.pathofexile.com"] = true - case "es": - includeForumHost["es.pathofexile.com"] = true - default: - includeReddit = true - includeForumHost["www.pathofexile.com"] = true - } - - return func(a Activity) bool { - switch a := a.(type) { - case *ForumPost: - return includeForumHost[a.Host] - case *RedditComment: - return includeReddit - case *RedditPost: - return includeReddit - } - return false - } -} diff --git a/server/activity_handler.go b/server/activity_handler.go index f75ebb9..8bfd3e6 100644 --- a/server/activity_handler.go +++ b/server/activity_handler.go @@ -16,7 +16,7 @@ type jsonResponse struct { func ActivityHandler(db *Database) echo.HandlerFunc { return func(c echo.Context) error { - activity, next := db.Activity(c.QueryParam("next"), 50, ActivityFilterForRequest(c.Request())) + activity, next := db.Activity(c.QueryParam("next"), 50, LocaleForRequest(c.Request()).ActivityFilter) response := jsonResponse{ Next: next, } diff --git a/server/common.go b/server/common.go index 459666c..f7d0297 100644 --- a/server/common.go +++ b/server/common.go @@ -1,6 +1,8 @@ package server import ( + "strings" + "github.com/labstack/echo" ) @@ -14,3 +16,16 @@ func AbsoluteURL(c echo.Context, resource string) string { } return scheme + "://" + c.Request().Host + resource } + +func SubdomainURL(c echo.Context, subdomain string) string { + scheme := c.Scheme() + if c.Request().Header.Get(echo.HeaderXForwardedProto) == "https" { + scheme = "https" + } + locale := LocaleForRequest(c.Request()) + host := strings.TrimPrefix(c.Request().Host, locale.Subdomain+".") + if subdomain != "" { + host = subdomain + "." + host + } + return scheme + "://" + host +} diff --git a/server/forum_indexer.go b/server/forum_indexer.go index ff4daf3..9db4c76 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -43,16 +43,6 @@ func (indexer *ForumIndexer) Close() { func (indexer *ForumIndexer) run() { log.Info("starting forum indexer") - hosts := []string{ - "www.pathofexile.com", - "br.pathofexile.com", - "ru.pathofexile.com", - "th.pathofexile.com", - "de.pathofexile.com", - "fr.pathofexile.com", - "es.pathofexile.com", - } - accounts := []string{ "Chris", "Jonathan", "Erik", "Mark_GGG", "Samantha", "Rory", "Rhys", "Qarl", "Andrew_GGG", "Damien_GGG", "Joel_GGG", "Ari", "Thomas", "BrianWeissman", "Edwin_GGG", "Support", "Dylan", @@ -84,10 +74,10 @@ func (indexer *ForumIndexer) run() { } var wg sync.WaitGroup - wg.Add(len(hosts)) + wg.Add(len(Locales)) - for _, host := range hosts { - host := host + for _, l := range Locales { + l := l go func() { for { for _, account := range accounts { @@ -95,7 +85,7 @@ func (indexer *ForumIndexer) run() { case <-indexer.closeSignal: return default: - indexer.index(host, account, timezone) + indexer.index(l, account, timezone) time.Sleep(time.Second) } } @@ -135,34 +125,7 @@ var postURLExpression = regexp.MustCompile("^/forum/view-thread/([0-9]+)/page/([ var threadURLExpression = regexp.MustCompile("^/forum/view-thread/([0-9]+)") var forumURLExpression = regexp.MustCompile("^/forum/view-forum/([0-9]+)") -var monthReplacer = strings.NewReplacer( - "ม.ค.", "Jan", - "ก.พ.", "Feb", - "มี.ค.", "Mar", - "เม.ย.", "Apr", - "พ.ค.", "May", - "มิ.ย.", "Jun", - "ก.ค.", "Jul", - "ส.ค.", "Aug", - "ก.ย.", "Sep", - "ต.ค.", "Oct", - "พ.ย.", "Nov", - "ธ.ค.", "Dec", - "janv.", "Jan", - "févr.", "Feb", - "mars", "Mar", - "avril", "Apr", - "mai", "May", - "juin", "Jun", - "juil.", "Jul", - "août", "Aug", - "sept.", "Sep", - "oct.", "Oct", - "nov.", "Nov", - "déc.", "Dec", -) - -func ScrapeForumPosts(doc *goquery.Document, timezone *time.Location) ([]*ForumPost, error) { +func ScrapeForumPosts(doc *goquery.Document, locale *Locale, timezone *time.Location) ([]*ForumPost, error) { posts := []*ForumPost(nil) err := error(nil) @@ -178,21 +141,9 @@ func ScrapeForumPosts(doc *goquery.Document, timezone *time.Location) ([]*ForumP } post.BodyHTML = body - timeText := monthReplacer.Replace(sel.Find(".post_date").Text()) - - for _, format := range []string{ - "Jan _2, 2006 3:04:05 PM", - "2/1/2006 15:04:05", - "2.1.2006 15:04:05", - "_2 Jan 2006, 15:04:05", - "_2 Jan 2006 15:04:05", - } { - if t, err := time.ParseInLocation(format, timeText, timezone); err == nil { - post.Time = t - break - } - } - if post.Time.IsZero() { + timeText := sel.Find(".post_date").Text() + + if post.Time, err = locale.ParseTime(timeText, timezone); err != nil { log.WithField("text", timeText).Error("unable to parse time") return false } @@ -226,24 +177,24 @@ func ScrapeForumPosts(doc *goquery.Document, timezone *time.Location) ([]*ForumP return posts, nil } -func (indexer *ForumIndexer) forumPosts(host, poster string, page int, timezone *time.Location) ([]*ForumPost, error) { - doc, err := indexer.requestDocument(host, fmt.Sprintf("/account/view-posts/%v/page/%v", poster, page)) +func (indexer *ForumIndexer) forumPosts(locale *Locale, poster string, page int, timezone *time.Location) ([]*ForumPost, error) { + doc, err := indexer.requestDocument(locale.ForumHost(), fmt.Sprintf("/account/view-posts/%v/page/%v", poster, page)) if err != nil { return nil, err } - posts, err := ScrapeForumPosts(doc, timezone) + posts, err := ScrapeForumPosts(doc, locale, timezone) if err != nil { return nil, err } for _, post := range posts { - post.Host = host + post.Host = locale.ForumHost() } return posts, nil } -func (indexer *ForumIndexer) index(host, poster string, timezone *time.Location) { +func (indexer *ForumIndexer) index(locale *Locale, poster string, timezone *time.Location) { logger := log.WithFields(log.Fields{ - "host": host, + "host": locale.ForumHost(), "poster": poster, }) @@ -251,7 +202,7 @@ func (indexer *ForumIndexer) index(host, poster string, timezone *time.Location) activity := []Activity(nil) for page := 1; ; page++ { - posts, err := indexer.forumPosts(host, poster, page, timezone) + posts, err := indexer.forumPosts(locale, poster, page, timezone) if err != nil { logger.WithError(err).Error("error requesting forum posts") } diff --git a/server/forum_indexer_test.go b/server/forum_indexer_test.go index f1600e5..6c85017 100644 --- a/server/forum_indexer_test.go +++ b/server/forum_indexer_test.go @@ -21,7 +21,7 @@ func TestScrapeForumPosts(t *testing.T) { tz, err := time.LoadLocation("America/Los_Angeles") require.NoError(t, err) - posts, err := ScrapeForumPosts(doc, tz) + posts, err := ScrapeForumPosts(doc, Locales[0], tz) require.NoError(t, err) assert.Equal(t, 10, len(posts)) diff --git a/server/index_handler.go b/server/index_handler.go index e21be6e..3c420d6 100644 --- a/server/index_handler.go +++ b/server/index_handler.go @@ -11,7 +11,7 @@ var index = ` GGG Tracker - + @@ -34,19 +34,24 @@ var index = `
+
    + {{range .Locales}} + + {{end}} +
-

Activity

+

{{call $.Translate "Activity"}}

- - - - + + + + @@ -71,16 +76,33 @@ type IndexConfiguration struct { GoogleAnalytics string } -func IndexHandler(configuration IndexConfiguration) echo.HandlerFunc { +var indexTemplate *template.Template + +func init() { t, err := template.New("index").Parse(index) if err != nil { panic(err) } + indexTemplate = t +} + +func IndexHandler(configuration IndexConfiguration) echo.HandlerFunc { return func(c echo.Context) error { - return t.Execute(c.Response(), struct { + locale := LocaleForRequest(c.Request()) + return indexTemplate.Execute(c.Response(), struct { Configuration IndexConfiguration + Locales []*Locale + Locale *Locale + Translate func(string) string + SubdomainURL func(string) string }{ Configuration: configuration, + Locales: Locales, + Locale: locale, + Translate: locale.Translate, + SubdomainURL: func(subdomain string) string { + return SubdomainURL(c, subdomain) + }, }) } } diff --git a/server/localization.go b/server/localization.go new file mode 100644 index 0000000..3b8cf5c --- /dev/null +++ b/server/localization.go @@ -0,0 +1,141 @@ +package server + +import ( + "net/http" + "strings" + "time" +) + +type Locale struct { + Subdomain string + Image string + IncludeReddit bool + Translations map[string]string + ParseTime func(s string, tz *time.Location) (time.Time, error) +} + +func (l *Locale) Translate(s string) string { + if translated, ok := l.Translations[s]; ok { + return translated + } + return s +} + +func (l *Locale) ActivityFilter(a Activity) bool { + switch a := a.(type) { + case *ForumPost: + return a.Host == l.ForumHost() + case *RedditComment: + return l.IncludeReddit + case *RedditPost: + return l.IncludeReddit + } + return false +} + +func (l *Locale) ForumHost() string { + if l.Subdomain != "" { + return l.Subdomain + ".pathofexile.com" + } + return "www.pathofexile.com" +} + +var thMonthReplacer = strings.NewReplacer( + "ม.ค.", "Jan", + "ก.พ.", "Feb", + "มี.ค.", "Mar", + "เม.ย.", "Apr", + "พ.ค.", "May", + "มิ.ย.", "Jun", + "ก.ค.", "Jul", + "ส.ค.", "Aug", + "ก.ย.", "Sep", + "ต.ค.", "Oct", + "พ.ย.", "Nov", + "ธ.ค.", "Dec", +) + +var frMonthReplacer = strings.NewReplacer( + "janv.", "Jan", + "févr.", "Feb", + "mars", "Mar", + "avril", "Apr", + "mai", "May", + "juin", "Jun", + "juil.", "Jul", + "août", "Aug", + "sept.", "Sep", + "oct.", "Oct", + "nov.", "Nov", + "déc.", "Dec", +) + +// TODO: add translations +var Locales = []*Locale{ + { + IncludeReddit: true, + Image: "static/images/locales/gb.png", + ParseTime: func(s string, tz *time.Location) (time.Time, error) { + return time.ParseInLocation("Jan _2, 2006 3:04:05 PM", s, tz) + }, + }, + { + Subdomain: "br", + Image: "static/images/locales/br.png", + ParseTime: func(s string, tz *time.Location) (time.Time, error) { + return time.ParseInLocation("2/1/2006 15:04:05", s, tz) + }, + }, + { + Subdomain: "ru", + Image: "static/images/locales/ru.png", + ParseTime: func(s string, tz *time.Location) (time.Time, error) { + return time.ParseInLocation("2.1.2006 15:04:05", s, tz) + }, + }, + { + Subdomain: "th", + Image: "static/images/locales/th.png", + ParseTime: func(s string, tz *time.Location) (time.Time, error) { + return time.ParseInLocation("_2 Jan 2006, 15:04:05", thMonthReplacer.Replace(s), tz) + }, + }, + { + Subdomain: "de", + Image: "static/images/locales/de.png", + ParseTime: func(s string, tz *time.Location) (time.Time, error) { + return time.ParseInLocation("2.1.2006 15:04:05", s, tz) + }, + }, + { + Subdomain: "fr", + Image: "static/images/locales/fr.png", + ParseTime: func(s string, tz *time.Location) (time.Time, error) { + return time.ParseInLocation("_2 Jan 2006 15:04:05", frMonthReplacer.Replace(s), tz) + }, + }, + { + Subdomain: "es", + Image: "static/images/locales/es.png", + ParseTime: func(s string, tz *time.Location) (time.Time, error) { + return time.ParseInLocation("2/1/2006 15:04:05", s, tz) + }, + }, +} + +func LocaleForRequest(r *http.Request) *Locale { + subdomain := "" + if r.Host != "" { + subdomain = strings.Split(r.Host, ".")[0] + } + + if subdomain != "" { + for _, l := range Locales { + if l.Subdomain == subdomain { + return l + } + } + } + + return Locales[0] +} diff --git a/server/rss_handler.go b/server/rss_handler.go index 62dee5d..38bc934 100644 --- a/server/rss_handler.go +++ b/server/rss_handler.go @@ -43,7 +43,7 @@ type rssResponse struct { func RSSHandler(db *Database) echo.HandlerFunc { return func(c echo.Context) error { - activity, _ := db.Activity(c.QueryParam("next"), 50, ActivityFilterForRequest(c.Request())) + activity, _ := db.Activity(c.QueryParam("next"), 50, LocaleForRequest(c.Request()).ActivityFilter) response := rssResponse{ Version: "2.0", Atom: "http://www.w3.org/2005/Atom", diff --git a/server/static/images/locales/br.png b/server/static/images/locales/br.png new file mode 100644 index 0000000000000000000000000000000000000000..ab9c25c7dea9b15a919604a49eb5c7ca43c6669a GIT binary patch literal 773 zcmV+g1N!`lP)^{F`xX9P`eo77ii*9V&oQM}aF1sQ?arp3AN8=|B3On^R%BViDpw zC&(pROHyg6z6yX8_%xfny20&@pE*%8L+A5--09y#SulWKm*n#bFJISTJ1ws5A7W2w znvq2pUrh)3g8Bn2=xUa#kTm)iXpN-ksEOfrNum>VbPlu=O;r=|dN8tCQmHHxlWE3A z4W@f<;P;voY`8VHmY!Ic=z<4RilTjoB57Zfrf>?Y3>^$zEm#s|| zG;Yz+^-_{X5Wi;{O@WJbae}HvN3<5RpcSEcOQB~nj7PONTrop9V6vtxOQhUna@LI^ z1ga`{J1_yN!m*?EtP3rIfNyz?zNrv*V`1hCDhOGG+Q6XUxjwZ5!e8psG&ll|G!VIbJ18xEGJ`FcHGER8Z{S4kbWJg`u>MH%0)d+3agb z5O!usq}}99!G?7i&K(-0t>qovi3+ZNsp3Ua$3}u8fBNs`Vrnb6G>ewXX^xi|R2oi< zjEhjoBIBhw?$78vn<}SZDZjw}3|)FLBsBQ5oUBf8Br-$qWQYgx3O;3gKddgF#FYxg z7Wf<%>peM!7u@761y<+)ZRK{5%v%Z{Gu|R>6{YeWvK;}a?}El`00000NkvXXu0mjf D``AO>_%)r1c48n{Iv*t(u1=&kHeO=ifaxsfqaa4po0ttzDx;Tb#Tu)9o!0qJ7 fpuUrfrC=gMKD)#VIknB-fbtBUu6{1-oD!MO;yj*T?Z^q`(G#>d=IMux3$Gmwwx^pqM z9fcicC)n(tl|MoN+5{$fSXDp;Us6j8%o>-m_!Z1}eRGfFc6^YZTr6!Q%}&(nn$F#H zB+vKJ%jEPk>lQ(85ZL?I`E+QR2HnB+EWh-@tqlu`iDQS9v-#*IPkgnEnr)jJCx8(8 z>R2zzB<=X8)=^I^K>_kNTz=G#8C97giF?&v$2f1;&XDE7tA@~Me05m67DPHQw4zE# pfx%1+YA{Z>y1#wHl$HPK_BU4CTN*`zHL3sr002ovPDHLkV1l??k(K}e literal 0 HcmV?d00001 diff --git a/server/static/images/locales/fr.png b/server/static/images/locales/fr.png new file mode 100644 index 0000000000000000000000000000000000000000..6caf5bce4f90f0873ce7eb1389422784eca2b891 GIT binary patch literal 146 zcmeAS@N?(olHy`uVBq!ia0vp^B0$W~!VDzEwtY$iaRPioTp5(7hGg&f|NsBzg6g-L zmfIaWe*vX93p^r=85p>QL70(Y)*K0-AbW|YuPggQE@n|no)EFpLm-npT^vI=t|un| kK~PXoVPRpS#1FVdQ&MBb@06p<1jQ{`u literal 0 HcmV?d00001 diff --git a/server/static/images/locales/gb.png b/server/static/images/locales/gb.png new file mode 100644 index 0000000000000000000000000000000000000000..2aad20bc391738e7bbbb72e4b5e1d776078098a6 GIT binary patch literal 946 zcmV;j15NyiP)EKPUtKh=ue+vyrP;7-#24O zNuu;%3r*h+U^eOjP_A9-jYd!)7>baZk%Wwlq8LpW65`MSJ&vsy65?@OY#}go4U;_` zRF(NTKaGyzpbM52Eb2>x;%^g2xIA3#x+rJe$)>PpH=_nM9W zx6fOzlx2u_kzQ|$x#J>`C<0JGgvi`Hs4BWx>&hN-#}z08KMX8+ghCM<_WLlHjfBVl zK%2dZ#gR$(y3H68C{5~P46Mv!xqJgS1ZL(ri!4_j>>ewkX%LUG7@1VM-4V%?;%QB-}NP9~@Y3h=o zCNfq3nG9}vT-rTt68oTAn!o9nF2g$MOWQ2Ikr}b&RY|XJLL?lLvHfo8+fpcXo(>`9 zppbD$NGNzue(wKGBoL6X18(WbFO-_b4k5iq}Z?qbyJZi=x`@9AVmNeyS_%%nEgU=0WO8=9SR UW&^<3kpKVy07*qoM6N<$f@mDXz5oCK literal 0 HcmV?d00001 diff --git a/server/static/images/locales/ru.png b/server/static/images/locales/ru.png new file mode 100644 index 0000000000000000000000000000000000000000..edd28c1b8f378e0e2c7c15991381dae8e34713c0 GIT binary patch literal 188 zcmeAS@N?(olHy`uVBq!ia0vp^B0$W~!2~3;e^r1e#^NA%Cx&(BWL^R}Ea{HEjtmSN z`?>!lvI6;>1s;*b3=DjSL74G){)!Z!pk#?_L`iUdT1k0gQ7S`0VrE{6US4X6f{C7i zo}s~7F0KPWMLM1?jv*HQ$v^xVB___2)R{i9|H4ld#S%MDj=BCqPR}Q|tjjt*vDa;0 cTCOF-)bfuzw$$xh57fut>FVdQ&MBb@00Nyij{pDw literal 0 HcmV?d00001 diff --git a/server/static/images/locales/th.png b/server/static/images/locales/th.png new file mode 100644 index 0000000000000000000000000000000000000000..7d632fda27e92989dc9bd46d729f4cff19348f22 GIT binary patch literal 182 zcmeAS@N?(olHy`uVBq!ia0vp^B0$W~!2~3;e^r1e&H|6fVg?3oVGw3ym^DWND9B#o z>Fdh=kV#mC(R$W=sd%7JsHcl#h{pNk8tui09P|=?{(r3gbJosNTqm!c`SJgM{U%dm z!vF?}(3C$K90qJ|pG=J>t)FgWzxmJ4&(r@f{QmXz_kknEr_Zx`pV Date: Wed, 7 Mar 2018 01:41:17 -0600 Subject: [PATCH 11/90] add translations --- server/localization.go | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/server/localization.go b/server/localization.go index 3b8cf5c..52557e6 100644 --- a/server/localization.go +++ b/server/localization.go @@ -85,6 +85,13 @@ var Locales = []*Locale{ ParseTime: func(s string, tz *time.Location) (time.Time, error) { return time.ParseInLocation("2/1/2006 15:04:05", s, tz) }, + Translations: map[string]string{ + "Activity": "Atividade", + "Thread": "Discussão", + "Poster": "Autor", + "Time": "Hora", + "Forum": "Fórum", + }, }, { Subdomain: "ru", @@ -92,6 +99,13 @@ var Locales = []*Locale{ ParseTime: func(s string, tz *time.Location) (time.Time, error) { return time.ParseInLocation("2.1.2006 15:04:05", s, tz) }, + Translations: map[string]string{ + "Activity": "Активность", + "Thread": "Тема", + "Poster": "Автор", + "Time": "Время", + "Forum": "Форум", + }, }, { Subdomain: "th", @@ -106,6 +120,13 @@ var Locales = []*Locale{ ParseTime: func(s string, tz *time.Location) (time.Time, error) { return time.ParseInLocation("2.1.2006 15:04:05", s, tz) }, + Translations: map[string]string{ + "Activity": "Aktivität", + "Thread": "Beitrag", + "Poster": "Verfasser", + "Time": "Datum", + "Forum": "Forum", + }, }, { Subdomain: "fr", @@ -113,6 +134,13 @@ var Locales = []*Locale{ ParseTime: func(s string, tz *time.Location) (time.Time, error) { return time.ParseInLocation("_2 Jan 2006 15:04:05", frMonthReplacer.Replace(s), tz) }, + Translations: map[string]string{ + "Activity": "Activité", + "Thread": "Fil de discussion", + "Poster": "Posteur", + "Time": "Date", + "Forum": "Forum", + }, }, { Subdomain: "es", @@ -120,6 +148,13 @@ var Locales = []*Locale{ ParseTime: func(s string, tz *time.Location) (time.Time, error) { return time.ParseInLocation("2/1/2006 15:04:05", s, tz) }, + Translations: map[string]string{ + "Activity": "Actividad", + "Thread": "Tema", + "Poster": "Autor", + "Time": "Fecha", + "Forum": "Foro", + }, }, } From e8b8251c851e22b83a0c54af9fe0f52078f20cb5 Mon Sep 17 00:00:00 2001 From: Christopher Brown Date: Wed, 7 Mar 2018 21:35:08 -0600 Subject: [PATCH 12/90] add a few additional forum posters --- server/forum_indexer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index 9db4c76..cccbd68 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -52,7 +52,7 @@ func (indexer *ForumIndexer) run() { "Alex_GGG", "Alexcc_GGG", "Andy", "CJ_GGG", "Eben_GGG", "Emma_GGG", "Ethan_GGG", "Fitzy_GGG", "Hartlin_GGG", "Jake_GGG", "Lionel_GGG", "Melissa_GGG", "MikeP_GGG", "Novynn", "Rachel_GGG", "Rob_GGG", "Roman_GGG", "Sarah_GGG", "SarahB_GGG", "Tom_GGG", "Natalia_GGG", - "Jeff_GGG", + "Jeff_GGG", "Lu_GGG", "JuliaS_GGG", "Alexander_GGG", } timezone := (*time.Location)(nil) From 2e6c2656adbdc4c12895a5c4a8664899583a2623 Mon Sep 17 00:00:00 2001 From: tgavankar <638772+tgavankar@users.noreply.github.com> Date: Fri, 29 Jun 2018 22:23:17 -0700 Subject: [PATCH 13/90] Add SamC_GGG to forum posters (#20) --- server/forum_indexer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index cccbd68..fdccd05 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -52,7 +52,7 @@ func (indexer *ForumIndexer) run() { "Alex_GGG", "Alexcc_GGG", "Andy", "CJ_GGG", "Eben_GGG", "Emma_GGG", "Ethan_GGG", "Fitzy_GGG", "Hartlin_GGG", "Jake_GGG", "Lionel_GGG", "Melissa_GGG", "MikeP_GGG", "Novynn", "Rachel_GGG", "Rob_GGG", "Roman_GGG", "Sarah_GGG", "SarahB_GGG", "Tom_GGG", "Natalia_GGG", - "Jeff_GGG", "Lu_GGG", "JuliaS_GGG", "Alexander_GGG", + "Jeff_GGG", "Lu_GGG", "JuliaS_GGG", "Alexander_GGG", "SamC_GGG", } timezone := (*time.Location)(nil) From c38faf6799ea78924da71cf8318d329595ace0c3 Mon Sep 17 00:00:00 2001 From: Christopher Brown Date: Sat, 30 Jun 2018 00:44:11 -0500 Subject: [PATCH 14/90] fix france date parsing for april --- server/localization.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/localization.go b/server/localization.go index 52557e6..1791f83 100644 --- a/server/localization.go +++ b/server/localization.go @@ -59,7 +59,7 @@ var frMonthReplacer = strings.NewReplacer( "janv.", "Jan", "févr.", "Feb", "mars", "Mar", - "avril", "Apr", + "avr.", "Apr", "mai", "May", "juin", "Jun", "juil.", "Jul", From 7ce15ae97fbd5ddc5c09cfd53b7a5bdb4d96df98 Mon Sep 17 00:00:00 2001 From: Chris Date: Sat, 1 Sep 2018 11:53:41 -0400 Subject: [PATCH 15/90] update time parsing (#22) * update time parsing * update testdata --- server/localization.go | 57 ++++++++++++++++++++++++++++---- server/localization_test.go | 32 ++++++++++++++++++ server/testdata/forum-posts.html | 20 +++++------ 3 files changed, 93 insertions(+), 16 deletions(-) create mode 100644 server/localization_test.go diff --git a/server/localization.go b/server/localization.go index 1791f83..ec0352e 100644 --- a/server/localization.go +++ b/server/localization.go @@ -40,6 +40,36 @@ func (l *Locale) ForumHost() string { return "www.pathofexile.com" } +var esMonthReplacer = strings.NewReplacer( + "ene", "Jan", + "feb", "Feb", + "mar", "Mar", + "abr", "Apr", + "may", "May", + "jun", "Jun", + "jul", "Jul", + "ago", "Aug", + "sep", "Sep", + "oct", "Oct", + "nov", "Nov", + "dic", "Dec", +) + +var brMonthReplacer = strings.NewReplacer( + "de jan de", "Jan", + "de fev de", "Feb", + "de mar de", "Mar", + "de abr de", "Apr", + "de mai de", "May", + "de jun de", "Jun", + "de jul de", "Jul", + "de ago de", "Aug", + "de set de", "Sep", + "de out de", "Oct", + "de nov de", "Nov", + "de dez de", "Dec", +) + var thMonthReplacer = strings.NewReplacer( "ม.ค.", "Jan", "ก.พ.", "Feb", @@ -70,20 +100,35 @@ var frMonthReplacer = strings.NewReplacer( "déc.", "Dec", ) +var ruMonthReplacer = strings.NewReplacer( + "янв.", "Jan", + "февр.", "Feb", + "марта", "Mar", + "апр.", "Apr", + "мая", "May", + "июня", "Jun", + "июля", "Jul", + "авг.", "Aug", + "сент.", "Sep", + "окт.", "Oct", + "нояб.", "Nov", + "дек.", "Dec", +) + // TODO: add translations var Locales = []*Locale{ { IncludeReddit: true, Image: "static/images/locales/gb.png", ParseTime: func(s string, tz *time.Location) (time.Time, error) { - return time.ParseInLocation("Jan _2, 2006 3:04:05 PM", s, tz) + return time.ParseInLocation("Jan _2, 2006, 3:04:05 PM", s, tz) }, }, { Subdomain: "br", Image: "static/images/locales/br.png", ParseTime: func(s string, tz *time.Location) (time.Time, error) { - return time.ParseInLocation("2/1/2006 15:04:05", s, tz) + return time.ParseInLocation("2 Jan 2006 15:04:05", brMonthReplacer.Replace(s), tz) }, Translations: map[string]string{ "Activity": "Atividade", @@ -97,7 +142,7 @@ var Locales = []*Locale{ Subdomain: "ru", Image: "static/images/locales/ru.png", ParseTime: func(s string, tz *time.Location) (time.Time, error) { - return time.ParseInLocation("2.1.2006 15:04:05", s, tz) + return time.ParseInLocation("2 Jan 2006 г., 15:04:05", ruMonthReplacer.Replace(s), tz) }, Translations: map[string]string{ "Activity": "Активность", @@ -111,14 +156,14 @@ var Locales = []*Locale{ Subdomain: "th", Image: "static/images/locales/th.png", ParseTime: func(s string, tz *time.Location) (time.Time, error) { - return time.ParseInLocation("_2 Jan 2006, 15:04:05", thMonthReplacer.Replace(s), tz) + return time.ParseInLocation("_2 Jan 2006 15:04:05", thMonthReplacer.Replace(s), tz) }, }, { Subdomain: "de", Image: "static/images/locales/de.png", ParseTime: func(s string, tz *time.Location) (time.Time, error) { - return time.ParseInLocation("2.1.2006 15:04:05", s, tz) + return time.ParseInLocation("2.1.2006, 15:04:05", s, tz) }, Translations: map[string]string{ "Activity": "Aktivität", @@ -146,7 +191,7 @@ var Locales = []*Locale{ Subdomain: "es", Image: "static/images/locales/es.png", ParseTime: func(s string, tz *time.Location) (time.Time, error) { - return time.ParseInLocation("2/1/2006 15:04:05", s, tz) + return time.ParseInLocation("2 Jan. 2006 15:04:05", esMonthReplacer.Replace(s), tz) }, Translations: map[string]string{ "Activity": "Actividad", diff --git a/server/localization_test.go b/server/localization_test.go new file mode 100644 index 0000000..12b53e1 --- /dev/null +++ b/server/localization_test.go @@ -0,0 +1,32 @@ +package server + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestLocale_ParseTime(t *testing.T) { + for subdomain, s := range map[string]string{ + "": "Aug 29, 2018, 5:51:19 PM", + "br": "31 de ago de 2018 00:50:19", + "ru": "1 сент. 2018 г., 2:09:52", + "th": "31 ส.ค. 2018 00:50:25", + "de": "31.08.2018, 00:50:20", + "fr": "31 août 2018 00:50:22", + "es": "31 ago. 2018 0:50:23", + } { + t.Run(subdomain, func(t *testing.T) { + var locale *Locale + for _, l := range Locales { + if l.Subdomain == subdomain { + locale = l + break + } + } + _, err := locale.ParseTime(s, time.FixedZone("UTC-5", -5*60*60)) + assert.NoError(t, err) + }) + } +} diff --git a/server/testdata/forum-posts.html b/server/testdata/forum-posts.html index 34b7a1f..b6fe2da 100644 --- a/server/testdata/forum-posts.html +++ b/server/testdata/forum-posts.html @@ -71,36 +71,36 @@
ThreadPosterTimeForum{{call $.Translate "Thread"}}{{call $.Translate "Poster"}}{{call $.Translate "Time"}}{{call $.Translate "Forum"}}
we had a great time too!
Lead Developer. Follow us on: Twitter | YouTube | Facebook | Contact Support if you need help!
maps3
View post
Avatar -
Posted by
on Feb 5, 2017 2:06:05 PM
Grinding Gear Games
Yesterday, we attended the San Francisco fan meetup and had a great time! It was our busiest one yet, and we were able to stay for three hours answering questions and meeting fans. Today's news post has some photos of the event! Read More. +
Posted by
on Feb 5, 2017, 2:06:05 PM
Grinding Gear Games
Yesterday, we attended the San Francisco fan meetup and had a great time! It was our busiest one yet, and we were able to stay for three hours answering questions and meeting fans. Today's news post has some photos of the event! Read More.
Lead Developer. Follow us on: Twitter | YouTube | Facebook | Contact Support if you need help!
maps3
Last edited by Chris on Feb 6, 2017 8:31:55 AM
View post
Avatar -
Posted by
on Feb 5, 2017 1:48:29 PM
Grinding Gear Games
Chris, Jonathan and Nick are going to be in San Francisco next week to hold a press tour for various upcoming announcements. If you're interested in meeting with them, they'll be hosting a fan meet-up on Saturday February 4th at 4pm (PST). If you're in or around the area, you should come on by! Read More. +
Posted by
on Feb 5, 2017, 1:48:29 PM
Grinding Gear Games
Chris, Jonathan and Nick are going to be in San Francisco next week to hold a press tour for various upcoming announcements. If you're interested in meeting with them, they'll be hosting a fan meet-up on Saturday February 4th at 4pm (PST). If you're in or around the area, you should come on by! Read More.
Lead Developer. Follow us on: Twitter | YouTube | Facebook | Contact Support if you need help!
maps3
View post
Avatar -
Posted by
on Jan 29, 2017 2:17:19 PM
Grinding Gear Games
Last year at the 2016 New Zealand Game Developers Conference, two of our designers gave a presentation called "The Labyrinth: Building Pathways for Players". It is a really detailed look at how Path of Exile's Labyrinth was designed and some of the challenges they faced when creating it. We have made the presentation (complete with their speaker notes to explain it) available here as a google document. +
Posted by
on Jan 29, 2017, 2:17:19 PM
Grinding Gear Games
Last year at the 2016 New Zealand Game Developers Conference, two of our designers gave a presentation called "The Labyrinth: Building Pathways for Players". It is a really detailed look at how Path of Exile's Labyrinth was designed and some of the challenges they faced when creating it. We have made the presentation (complete with their speaker notes to explain it) available here as a google document. Read More.
Lead Developer. Follow us on: Twitter | YouTube | Facebook | Contact Support if you need help!
maps3
View post
Avatar -
Posted by
on Jan 24, 2017 4:30:34 PM
Grinding Gear Games
"
Completed 9 ChallengesArcis wrote:
afaik, the main reason people want a SSF league is so the game doesn't have to be balanced around the fact that trading will always be much more efficient than soloing. if there are no additional benefits to playing SSF, i think this has missed the point.

+
Posted by
on Jan 24, 2017, 4:30:34 PM
Grinding Gear Games
"
Completed 9 ChallengesArcis wrote:
afaik, the main reason people want a SSF league is so the game doesn't have to be balanced around the fact that trading will always be much more efficient than soloing. if there are no additional benefits to playing SSF, i think this has missed the point.


Getting a fair playing field is really important to people who don't want to engage in trade.

I'm sure the two leagues will play out at massively different paces, and that's totally fine with us.
Lead Developer. Follow us on: Twitter | YouTube | Facebook | Contact Support if you need help!
maps3
View post
Avatar -
Posted by
on Jan 22, 2017 7:37:48 PM
Grinding Gear Games
Self-enforced Solo Self-Found play is an increasingly popular choice for players who want to have a more challenging experience in Path of Exile. In March's Content Update 2.6.0, we plan to add formal support for this mode so that players who enjoy SSF can prove that they reached certain levels without outside assistance. We have written a Development Manifesto post about our current plans. +
Posted by
on Jan 22, 2017, 7:37:48 PM
Grinding Gear Games
Self-enforced Solo Self-Found play is an increasingly popular choice for players who want to have a more challenging experience in Path of Exile. In March's Content Update 2.6.0, we plan to add formal support for this mode so that players who enjoy SSF can prove that they reached certain levels without outside assistance. We have written a Development Manifesto post about our current plans. Read More.
Lead Developer. Follow us on: Twitter | YouTube | Facebook | Contact Support if you need help!
maps3
View post
Avatar -
Posted by
on Jan 22, 2017 7:25:56 PM
Grinding Gear Games
We plan to add a system to officially record the progress of Solo Self-Found players in Content Update 2.6.0. This manifesto post describes our current plan. We welcome your feedback.
+
Posted by
on Jan 22, 2017, 7:25:56 PM
Grinding Gear Games
We plan to add a system to officially record the progress of Solo Self-Found players in Content Update 2.6.0. This manifesto post describes our current plan. We welcome your feedback.

What is Solo Self-Found?

@@ -136,7 +136,7 @@ Please let us know if you have any feedback on this system!
Lead Developer. Follow us on: Twitter | YouTube | Facebook | Contact Support if you need help!
maps3
View post
Avatar -
Posted by
on Jan 22, 2017 7:25:15 PM
Grinding Gear Games
When we started our Xbox One project in 2015, we looked at how other online game communities reacted to the announcement of a console version of those games. While many players were excited, some were disappointed and had a few fears:
+
Posted by
on Jan 22, 2017, 7:25:15 PM
Grinding Gear Games
When we started our Xbox One project in 2015, we looked at how other online game communities reacted to the announcement of a console version of those games. While many players were excited, some were disappointed and had a few fears:
  • The fear that development resources would be spent on a version of the game that didn't directly benefit that player.
  • The fear that the PC version would be modified to become more similar to the new console version.

After we saw other communities reacting like this, we internally pledged to make sure that our development of the console version of Path of Exile took these concerns into account.

@@ -149,17 +149,17 @@ I hope you enjoyed the new trailer and spotted a few of the easter eggs that hint at the massive amount of content we'll be announcing next month!
Lead Developer. Follow us on: Twitter | YouTube | Facebook | Contact Support if you need help!
maps3
Last edited by Chris on Jan 18, 2017 4:54:03 PM
View post
Avatar -
Posted by
on Jan 18, 2017 4:50:53 PM
Grinding Gear Games
Our programmers have completed another wave of game client performance improvements. We plan to release them next week in a minor update called 2.5.2. Today's news post explains how these changes improve Path of Exile's performance. Read More. +
Posted by
on Jan 18, 2017, 4:50:53 PM
Grinding Gear Games
Our programmers have completed another wave of game client performance improvements. We plan to release them next week in a minor update called 2.5.2. Today's news post explains how these changes improve Path of Exile's performance. Read More.
Lead Developer. Follow us on: Twitter | YouTube | Facebook | Contact Support if you need help!
maps3
View post
Avatar -
Posted by
on Jan 10, 2017 2:12:19 PM
Grinding Gear Games
Most of our team are now back at work after the New Zealand holiday period. We have a busy year ahead! The purpose of today's news post is to highlight the rough plans of what happens when the Breach leagues end in seven weeks. Read More. +
Posted by
on Jan 10, 2017, 2:12:19 PM
Grinding Gear Games
Most of our team are now back at work after the New Zealand holiday period. We have a busy year ahead! The purpose of today's news post is to highlight the rough plans of what happens when the Breach leagues end in seven weeks. Read More.
Lead Developer. Follow us on: Twitter | YouTube | Facebook | Contact Support if you need help!
maps3
View post
Avatar -
Posted by
on Jan 8, 2017 7:11:07 PM
Grinding Gear Games
+
Posted by
on Jan 8, 2017, 7:11:07 PM
Grinding Gear Games
From 76d92987143059844402d45248bfe627c0912f7f Mon Sep 17 00:00:00 2001 From: Chris Date: Mon, 3 Sep 2018 13:50:49 -0400 Subject: [PATCH 16/90] update for post url change (#24) --- server/forum_indexer.go | 8 +++----- server/forum_indexer_test.go | 1 - server/forum_post.go | 1 - server/testdata/forum-posts.html | 4 ++-- 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index fdccd05..6ee2ca4 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -121,7 +121,7 @@ func (indexer *ForumIndexer) requestDocument(host, resource string) (*goquery.Do return goquery.NewDocumentFromReader(resp.Body) } -var postURLExpression = regexp.MustCompile("^/forum/view-thread/([0-9]+)/page/([0-9]+)#p([0-9]+)") +var postURLExpression = regexp.MustCompile("^/forum/view-post/([0-9]+)") var threadURLExpression = regexp.MustCompile("^/forum/view-thread/([0-9]+)") var forumURLExpression = regexp.MustCompile("^/forum/view-forum/([0-9]+)") @@ -152,12 +152,10 @@ func ScrapeForumPosts(doc *goquery.Document, locale *Locale, timezone *time.Loca href := sel.AttrOr("href", "") if match := postURLExpression.FindStringSubmatch(href); match != nil { n, _ := strconv.Atoi(match[1]) - post.ThreadId = n - n, _ = strconv.Atoi(match[2]) - post.PageNumber = n - n, _ = strconv.Atoi(match[3]) post.Id = n } else if match := threadURLExpression.FindStringSubmatch(href); match != nil { + n, _ := strconv.Atoi(match[1]) + post.ThreadId = n post.ThreadTitle = sel.Text() } else if match := forumURLExpression.FindStringSubmatch(href); match != nil { n, _ := strconv.Atoi(match[1]) diff --git a/server/forum_indexer_test.go b/server/forum_indexer_test.go index 6c85017..442b950 100644 --- a/server/forum_indexer_test.go +++ b/server/forum_indexer_test.go @@ -33,7 +33,6 @@ func TestScrapeForumPosts(t *testing.T) { assert.Equal(t, "Chris", p.Poster) assert.Equal(t, "Photos of the Fan Meetup", p.ThreadTitle) assert.Equal(t, "Announcements", p.ForumName) - assert.Equal(t, 1, p.PageNumber) assert.Equal(t, "we had a great time too!", p.BodyHTML) assert.Equal(t, int64(1486332365), p.Time.Unix()) } diff --git a/server/forum_post.go b/server/forum_post.go index 872b521..3d4f0e7 100644 --- a/server/forum_post.go +++ b/server/forum_post.go @@ -13,7 +13,6 @@ type ForumPost struct { Poster string `json:"poster"` ThreadId int `json:"thread_id"` ThreadTitle string `json:"thread_title"` - PageNumber int `json:"page_number"` ForumId int `json:"forum_id"` ForumName string `json:"forum_name"` } diff --git a/server/testdata/forum-posts.html b/server/testdata/forum-posts.html index b6fe2da..9d23698 100644 --- a/server/testdata/forum-posts.html +++ b/server/testdata/forum-posts.html @@ -69,12 +69,12 @@
View Profile»Recent posts by Chris
').addClass(type == 'forum_post' ? 'forum' : 'reddit').addClass(type.replace('/_/', '-')); var $toggleTD = $('
we had a great time too!
Lead Developer. Follow us on: Twitter | YouTube | Facebook | Contact Support if you need help!
-maps3
View post
Avatar
Posted by
on Feb 5, 2017, 2:06:05 PM
Grinding Gear Games
Yesterday, we attended the San Francisco fan meetup and had a great time! It was our busiest one yet, and we were able to stay for three hours answering questions and meeting fans. Today's news post has some photos of the event! Read More.
Lead Developer. Follow us on: Twitter | YouTube | Facebook | Contact Support if you need help!
-maps3
Last edited by Chris on Feb 6, 2017 8:31:55 AM
View post
+maps3
Last edited by Chris on Feb 6, 2017 8:31:55 AM
View post
Avatar
Posted by
on Feb 5, 2017, 1:48:29 PM
Grinding Gear Games
Chris, Jonathan and Nick are going to be in San Francisco next week to hold a press tour for various upcoming announcements. If you're interested in meeting with them, they'll be hosting a fan meet-up on Saturday February 4th at 4pm (PST). If you're in or around the area, you should come on by! Read More. From 0eb880bb12232e0ef80061db7f4dea2a61af77a8 Mon Sep 17 00:00:00 2001 From: Chris Date: Mon, 3 Sep 2018 14:13:00 -0400 Subject: [PATCH 17/90] ignore forum posts stored with zero id (#25) --- server/database.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/database.go b/server/database.go index a79679b..7f7cbd7 100644 --- a/server/database.go +++ b/server/database.go @@ -100,7 +100,9 @@ func (db *Database) Activity(start string, count int, filter func(Activity) bool if post.Host == "" { post.Host = "www.pathofexile.com" } - activity = post + if post.Id != 0 { + activity = post + } case RedditCommentType: comment := &RedditComment{} err := json.Unmarshal(v, comment) @@ -116,7 +118,7 @@ func (db *Database) Activity(start string, count int, filter func(Activity) bool } activity = post } - if filter == nil || filter(activity) { + if activity != nil && (filter == nil || filter(activity)) { ret = append(ret, activity) next = base64.RawURLEncoding.EncodeToString(k) } From 0bdc213cbf977843c0ecc1d7f115d42ff3650bd0 Mon Sep 17 00:00:00 2001 From: Chris Date: Fri, 14 Sep 2018 00:04:50 -0400 Subject: [PATCH 18/90] fix es september parsing (#26) --- server/localization.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/localization.go b/server/localization.go index ec0352e..f47aad1 100644 --- a/server/localization.go +++ b/server/localization.go @@ -49,7 +49,7 @@ var esMonthReplacer = strings.NewReplacer( "jun", "Jun", "jul", "Jul", "ago", "Aug", - "sep", "Sep", + "sept", "Sep", "oct", "Oct", "nov", "Nov", "dic", "Dec", From c927a0d085009307c91001ab1a1c5e278dca0d62 Mon Sep 17 00:00:00 2001 From: Chris Date: Sun, 11 Nov 2018 23:43:37 -0500 Subject: [PATCH 19/90] add openarl to reddit (#27) --- server/reddit_indexer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/reddit_indexer.go b/server/reddit_indexer.go index 0cda96a..a65d150 100644 --- a/server/reddit_indexer.go +++ b/server/reddit_indexer.go @@ -41,6 +41,7 @@ func (indexer *RedditIndexer) run() { "Mark_GGG", "RhysGGG", "Dan_GGG", "Rory_Rackham", "Blake_GGG", "Fitzy_GGG", "Hartlin_GGG", "Hrishi_GGG", "Baltic_GGG", "KamilOrmanJanowski", "Daniel_GGG", "Jeff_GGG", "NapfelGGG", "Baltic_GGG", "Novynn", "Felipe_GGG", "Mel_GGG", "Sarah_GGG", "riandrake", "Kieren_GGG", + "Openarl", } next := 0 From 5ed72daaf0302c986b82e164c0d1ab6e51c13556 Mon Sep 17 00:00:00 2001 From: Chris Date: Sun, 25 Nov 2018 23:23:33 -0500 Subject: [PATCH 20/90] add dynamodb support (#28) * add dynamodb support * don't export dynamodb table creation function * compatibility fix for older db entries --- aws-sam/README.md | 3 + aws-sam/gggtracker.cfn.yaml | 19 ++++ main.go | 19 +++- server/activity_handler.go | 7 +- server/bolt_database.go | 84 ++++++++++++++++ server/bolt_database_test.go | 22 +++++ server/database.go | 161 ++++++++++--------------------- server/database_test.go | 28 +++--- server/dynamodb_database.go | 141 +++++++++++++++++++++++++++ server/dynamodb_database_test.go | 110 +++++++++++++++++++++ server/forum_indexer.go | 14 ++- server/localization.go | 1 - server/reddit_indexer.go | 14 ++- server/rss_handler.go | 7 +- 14 files changed, 488 insertions(+), 142 deletions(-) create mode 100644 aws-sam/README.md create mode 100644 aws-sam/gggtracker.cfn.yaml create mode 100644 server/bolt_database.go create mode 100644 server/bolt_database_test.go create mode 100644 server/dynamodb_database.go create mode 100644 server/dynamodb_database_test.go diff --git a/aws-sam/README.md b/aws-sam/README.md new file mode 100644 index 0000000..437266d --- /dev/null +++ b/aws-sam/README.md @@ -0,0 +1,3 @@ +# aws-sam + +This directory contains resources for deploying the tracker using the AWS Serverless Application Model (SAM). By deploying via serverless technologies, you can create a cost-effective (free or nearly free) scalable deployment. diff --git a/aws-sam/gggtracker.cfn.yaml b/aws-sam/gggtracker.cfn.yaml new file mode 100644 index 0000000..814247e --- /dev/null +++ b/aws-sam/gggtracker.cfn.yaml @@ -0,0 +1,19 @@ +AWSTemplateFormatVersion: '2010-09-09' +Description: github.com/ccbrown/gggtracker +Resources: + DynamoDBTable: + Type: AWS::DynamoDB::Table + Properties: + AttributeDefinitions: + - AttributeName: hk + AttributeType: B + - AttributeName: rk + AttributeType: B + KeySchema: + - AttributeName: hk + KeyType: HASH + - AttributeName: rk + KeyType: RANGE + ProvisionedThroughput: + ReadCapacityUnits: 25 + WriteCapacityUnits: 25 diff --git a/main.go b/main.go index 56cd657..11b2da1 100644 --- a/main.go +++ b/main.go @@ -4,7 +4,10 @@ import ( "fmt" "net/http" "path" + "strings" + "github.com/aws/aws-sdk-go-v2/aws/external" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" "github.com/labstack/echo" "github.com/labstack/echo/middleware" log "github.com/sirupsen/logrus" @@ -18,17 +21,29 @@ func main() { pflag.IntP("port", "p", 8080, "the port to listen on") pflag.String("staticdir", "./server/static", "the static files to serve") pflag.String("ga", "", "a google analytics account") - pflag.String("db", "./gggtracker.db", "the database path") + pflag.String("db", "./gggtracker.db", "the database file path") + pflag.String("dynamodb-table", "", "if given, DynamoDB will be used instead of a database file") pflag.String("forumsession", "", "the POESESSID cookie for a forum session") viper.BindPFlags(pflag.CommandLine) pflag.Parse() viper.SetEnvPrefix("gggtracker") + viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) viper.AutomaticEnv() e := echo.New() - db, err := server.OpenDatabase(viper.GetString("db")) + var db server.Database + var err error + if tableName := viper.GetString("dynamodb-table"); tableName != "" { + config, err := external.LoadDefaultAWSConfig() + if err != nil { + log.Fatal(err) + } + db, err = server.NewDynamoDBDatabase(dynamodb.New(config), tableName) + } else { + db, err = server.NewBoltDatabase(viper.GetString("db")) + } if err != nil { log.Fatal(err) } diff --git a/server/activity_handler.go b/server/activity_handler.go index 8bfd3e6..7f883f1 100644 --- a/server/activity_handler.go +++ b/server/activity_handler.go @@ -14,9 +14,12 @@ type jsonResponse struct { Next string `json:"next"` } -func ActivityHandler(db *Database) echo.HandlerFunc { +func ActivityHandler(db Database) echo.HandlerFunc { return func(c echo.Context) error { - activity, next := db.Activity(c.QueryParam("next"), 50, LocaleForRequest(c.Request()).ActivityFilter) + activity, next, err := db.Activity(LocaleForRequest(c.Request()), c.QueryParam("next"), 50) + if err != nil { + return err + } response := jsonResponse{ Next: next, } diff --git a/server/bolt_database.go b/server/bolt_database.go new file mode 100644 index 0000000..5a83aec --- /dev/null +++ b/server/bolt_database.go @@ -0,0 +1,84 @@ +package server + +import ( + "encoding/base64" + + "github.com/boltdb/bolt" +) + +type BoltDatabase struct { + db *bolt.DB +} + +func NewBoltDatabase(path string) (*BoltDatabase, error) { + db, err := bolt.Open(path, 0600, nil) + if err != nil { + return nil, err + } + + db.Update(func(tx *bolt.Tx) error { + _, err := tx.CreateBucketIfNotExists([]byte("activity")) + if err != nil { + return err + } + return nil + }) + + return &BoltDatabase{ + db: db, + }, nil +} + +func (db *BoltDatabase) AddActivity(activity []Activity) error { + return db.db.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("activity")) + for _, a := range activity { + k, v, err := marshalActivity(a) + if err != nil { + return err + } + b.Put(k, v) + } + return nil + }) +} + +func (db *BoltDatabase) Activity(locale *Locale, start string, count int) ([]Activity, string, error) { + ret := []Activity(nil) + next := "" + if err := db.db.View(func(tx *bolt.Tx) error { + c := tx.Bucket([]byte("activity")).Cursor() + var k, v []byte + if start == "" { + k, v = c.Last() + } else { + s, err := base64.RawURLEncoding.DecodeString(start) + if err != nil { + k, v = c.Last() + } else { + k, v = c.Seek(s) + if k != nil { + k, v = c.Prev() + } + } + } + for len(ret) < count && k != nil { + activity, err := unmarshalActivity(k, v) + if err != nil { + return err + } else if activity != nil && locale.ActivityFilter(activity) { + ret = append(ret, activity) + next = base64.RawURLEncoding.EncodeToString(k) + } + k, v = c.Prev() + } + return nil + }); err != nil { + return nil, "", err + } + return ret, next, nil +} + +func (db *BoltDatabase) Close() error { + return db.db.Close() +} diff --git a/server/bolt_database_test.go b/server/bolt_database_test.go new file mode 100644 index 0000000..9e1c361 --- /dev/null +++ b/server/bolt_database_test.go @@ -0,0 +1,22 @@ +package server + +import ( + "io/ioutil" + "os" + "path" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBoltDatabase(t *testing.T) { + dir, err := ioutil.TempDir("testdata", "db") + require.NoError(t, err) + defer os.RemoveAll(dir) + + db, err := NewBoltDatabase(path.Join(dir, "test.db")) + require.NoError(t, err) + defer db.Close() + + testDatabase(t, db) +} diff --git a/server/database.go b/server/database.go index 7f7cbd7..653103d 100644 --- a/server/database.go +++ b/server/database.go @@ -1,133 +1,70 @@ package server import ( - "encoding/base64" "encoding/binary" - "encoding/json" - "github.com/boltdb/bolt" + json "github.com/json-iterator/go" ) -type Database struct { - db *bolt.DB -} - -func OpenDatabase(path string) (*Database, error) { - db, err := bolt.Open(path, 0600, nil) - if err != nil { - return nil, err - } - - db.Update(func(tx *bolt.Tx) error { - _, err := tx.CreateBucketIfNotExists([]byte("activity")) - if err != nil { - return err - } - return nil - }) - - return &Database{ - db: db, - }, nil -} - -func (db *Database) Close() { - db.db.Close() -} - const ( ForumPostType = iota RedditCommentType RedditPostType ) -func (db *Database) AddActivity(activity []Activity) { - err := db.db.Update(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte("activity")) - for _, a := range activity { - buf, err := json.Marshal(a) - if err != nil { - return err - } - k := make([]byte, 10) - binary.BigEndian.PutUint64(k, uint64(a.ActivityTime().Unix())<<24) - switch a.(type) { - case *ForumPost: - k[5] = ForumPostType - case *RedditComment: - k[5] = RedditCommentType - case *RedditPost: - k[5] = RedditPostType - } - binary.BigEndian.PutUint32(k[6:], a.ActivityKey()) - b.Put(k, buf) - } - return nil - }) +type Database interface { + AddActivity(activity []Activity) error + Activity(locale *Locale, start string, count int) ([]Activity, string, error) + Close() error +} + +func marshalActivity(a Activity) (key, value []byte, err error) { + buf, err := json.Marshal(a) if err != nil { - panic(err) + return nil, nil, err } + k := make([]byte, 10) + binary.BigEndian.PutUint64(k, uint64(a.ActivityTime().Unix())<<24) + switch a.(type) { + case *ForumPost: + k[5] = ForumPostType + case *RedditComment: + k[5] = RedditCommentType + case *RedditPost: + k[5] = RedditPostType + } + binary.BigEndian.PutUint32(k[6:], a.ActivityKey()) + return k, buf, nil } -func (db *Database) Activity(start string, count int, filter func(Activity) bool) ([]Activity, string) { - ret := []Activity(nil) - next := "" - err := db.db.View(func(tx *bolt.Tx) error { - c := tx.Bucket([]byte("activity")).Cursor() - var k, v []byte - if start == "" { - k, v = c.Last() - } else { - s, err := base64.RawURLEncoding.DecodeString(start) - if err != nil { - k, v = c.Last() - } else { - k, v = c.Seek(s) - if k != nil { - k, v = c.Prev() - } - } +func unmarshalActivity(key, value []byte) (Activity, error) { + switch key[5] { + case ForumPostType: + post := &ForumPost{} + err := json.Unmarshal(value, post) + if err != nil { + return nil, err } - for len(ret) < count && k != nil { - var activity Activity - switch k[5] { - case ForumPostType: - post := &ForumPost{} - err := json.Unmarshal(v, post) - if err != nil { - return err - } - if post.Host == "" { - post.Host = "www.pathofexile.com" - } - if post.Id != 0 { - activity = post - } - case RedditCommentType: - comment := &RedditComment{} - err := json.Unmarshal(v, comment) - if err != nil { - return err - } - activity = comment - case RedditPostType: - post := &RedditPost{} - err := json.Unmarshal(v, post) - if err != nil { - return err - } - activity = post - } - if activity != nil && (filter == nil || filter(activity)) { - ret = append(ret, activity) - next = base64.RawURLEncoding.EncodeToString(k) - } - k, v = c.Prev() + if post.Host == "" { + post.Host = "www.pathofexile.com" } - return nil - }) - if err != nil { - panic(err) + if post.Id != 0 { + return post, nil + } + case RedditCommentType: + comment := &RedditComment{} + err := json.Unmarshal(value, comment) + if err != nil { + return nil, err + } + return comment, nil + case RedditPostType: + post := &RedditPost{} + err := json.Unmarshal(value, post) + if err != nil { + return nil, err + } + return post, nil } - return ret, next + return nil, nil } diff --git a/server/database_test.go b/server/database_test.go index 96153b6..d816b79 100644 --- a/server/database_test.go +++ b/server/database_test.go @@ -1,9 +1,6 @@ package server import ( - "io/ioutil" - "os" - "path" "testing" "time" @@ -11,41 +8,46 @@ import ( "github.com/stretchr/testify/require" ) -func TestDatabase_ForumPosts(t *testing.T) { - dir, err := ioutil.TempDir("testdata", "db") - require.NoError(t, err) - defer os.RemoveAll(dir) +func testDatabase(t *testing.T, db Database) { + t.Run("ForumPosts", func(t *testing.T) { + testDatabase_ForumPosts(t, db) + }) +} - db, err := OpenDatabase(path.Join(dir, "test.db")) - require.NoError(t, err) - defer db.Close() +func testDatabase_ForumPosts(t *testing.T, db Database) { + locale := Locales[0] post1 := &ForumPost{ Id: 9000, Poster: "Chris", Time: time.Unix(1486332365, 0), + Host: locale.ForumHost(), } post2 := &ForumPost{ Id: 9001, Poster: "Chris", Time: time.Unix(1486332364, 0), + Host: locale.ForumHost(), } db.AddActivity([]Activity{post1, post2}) - posts, next := db.Activity("", 1, nil) + posts, next, err := db.Activity(locale, "", 1) + require.NoError(t, err) require.Equal(t, 1, len(posts)) assert.Equal(t, post1.Id, posts[0].(*ForumPost).Id) assert.Equal(t, post1.Poster, posts[0].(*ForumPost).Poster) assert.Equal(t, post1.Time.Unix(), posts[0].(*ForumPost).Time.Unix()) - posts, next = db.Activity(next, 1, nil) + posts, next, err = db.Activity(locale, next, 1) + require.NoError(t, err) require.Equal(t, 1, len(posts)) assert.Equal(t, post2.Id, posts[0].(*ForumPost).Id) assert.Equal(t, post2.Poster, posts[0].(*ForumPost).Poster) assert.Equal(t, post2.Time.Unix(), posts[0].(*ForumPost).Time.Unix()) - posts, _ = db.Activity(next, 1, nil) + posts, _, err = db.Activity(locale, next, 1) + require.NoError(t, err) require.Equal(t, 0, len(posts)) } diff --git a/server/dynamodb_database.go b/server/dynamodb_database.go new file mode 100644 index 0000000..1411b32 --- /dev/null +++ b/server/dynamodb_database.go @@ -0,0 +1,141 @@ +package server + +import ( + "encoding/base64" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" +) + +type DynamoDBDatabase struct { + client *dynamodb.DynamoDB + tableName string +} + +func NewDynamoDBDatabase(client *dynamodb.DynamoDB, tableName string) (*DynamoDBDatabase, error) { + return &DynamoDBDatabase{ + client: client, + tableName: tableName, + }, nil +} + +func dynamoDBActivityHashKey(locale *Locale) []byte { + return []byte("activity_by_locale:" + locale.Subdomain) +} + +func (db *DynamoDBDatabase) AddActivity(activity []Activity) error { + for _, locale := range Locales { + var remaining []Activity + for _, a := range activity { + if locale.ActivityFilter(a) { + remaining = append(remaining, a) + } + } + + for len(remaining) > 0 { + batch := remaining + const maxBatchSize = 25 + if len(batch) > maxBatchSize { + batch = batch[:maxBatchSize] + } + + writeRequests := make([]dynamodb.WriteRequest, len(batch)) + for i, a := range batch { + k, v, err := marshalActivity(a) + if err != nil { + return err + } + writeRequests[i] = dynamodb.WriteRequest{ + PutRequest: &dynamodb.PutRequest{ + Item: map[string]dynamodb.AttributeValue{ + "hk": dynamodb.AttributeValue{ + B: dynamoDBActivityHashKey(locale), + }, + "rk": dynamodb.AttributeValue{ + B: []byte(k), + }, + "v": dynamodb.AttributeValue{ + B: []byte(v), + }, + }, + }, + } + } + unprocessed := map[string][]dynamodb.WriteRequest{ + db.tableName: writeRequests, + } + + for len(unprocessed) > 0 { + result, err := db.client.BatchWriteItemRequest(&dynamodb.BatchWriteItemInput{ + RequestItems: unprocessed, + }).Send() + if err != nil { + return err + } + unprocessed = result.UnprocessedItems + } + + remaining = remaining[len(batch):] + } + } + return nil +} + +func (db *DynamoDBDatabase) Activity(locale *Locale, start string, count int) ([]Activity, string, error) { + var activity []Activity + + var startKey map[string]dynamodb.AttributeValue + if start != "" { + rk, _ := base64.RawURLEncoding.DecodeString(start) + startKey = map[string]dynamodb.AttributeValue{ + "hk": dynamodb.AttributeValue{ + B: dynamoDBActivityHashKey(locale), + }, + "rk": dynamodb.AttributeValue{ + B: rk, + }, + } + } + + condition := "hk = :hash" + attributeValues := map[string]dynamodb.AttributeValue{ + ":hash": dynamodb.AttributeValue{ + B: dynamoDBActivityHashKey(locale), + }, + } + + for len(activity) < count { + result, err := db.client.QueryRequest(&dynamodb.QueryInput{ + TableName: aws.String(db.tableName), + KeyConditionExpression: aws.String(condition), + ExpressionAttributeValues: attributeValues, + ExclusiveStartKey: startKey, + Limit: aws.Int64(int64(count - len(activity))), + ScanIndexForward: aws.Bool(false), + }).Send() + if err != nil { + return nil, "", err + } + for _, item := range result.Items { + if a, err := unmarshalActivity(item["rk"].B, item["v"].B); err != nil { + return nil, "", err + } else if a != nil { + activity = append(activity, a) + } + } + if result.LastEvaluatedKey == nil { + break + } + startKey = result.LastEvaluatedKey + } + + var next string + if startKey != nil { + next = base64.RawURLEncoding.EncodeToString(startKey["rk"].B) + } + return activity, next, nil +} + +func (db *DynamoDBDatabase) Close() error { + return nil +} diff --git a/server/dynamodb_database_test.go b/server/dynamodb_database_test.go new file mode 100644 index 0000000..a8a3f12 --- /dev/null +++ b/server/dynamodb_database_test.go @@ -0,0 +1,110 @@ +package server + +import ( + "crypto/rand" + "encoding/base64" + "os" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/awserr" + "github.com/aws/aws-sdk-go-v2/aws/defaults" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" + "github.com/stretchr/testify/require" +) + +func newDynamoDBTestClient() (*dynamodb.DynamoDB, error) { + endpoint := os.Getenv("DYNAMODB_ENDPOINT") + + config := defaults.Config() + config.Region = "us-east-1" + config.EndpointResolver = aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) { + if endpoint != "" { + return aws.Endpoint{ + URL: endpoint, + }, nil + } + return aws.Endpoint{ + URL: "http://localhost:8000", + }, nil + }) + + credentialsBuf := make([]byte, 20) + if _, err := rand.Read(credentialsBuf); err != nil { + return nil, err + } + credentials := base64.RawURLEncoding.EncodeToString(credentialsBuf) + config.Credentials = aws.NewStaticCredentialsProvider(credentials, credentials, "") + config.Retryer = aws.DefaultRetryer{ + NumMaxRetries: 0, + } + + client := dynamodb.New(config) + if endpoint == "" { + if _, err := client.ListTablesRequest(&dynamodb.ListTablesInput{}).Send(); err != nil { + if err, ok := err.(awserr.Error); ok && err.Code() == "RequestError" { + return nil, nil + } + } + } + return client, nil +} + +func createDynamoDBTable(client *dynamodb.DynamoDB, tableName string) error { + if _, err := client.CreateTableRequest(&dynamodb.CreateTableInput{ + AttributeDefinitions: []dynamodb.AttributeDefinition{ + { + AttributeName: aws.String("hk"), + AttributeType: dynamodb.ScalarAttributeTypeB, + }, { + AttributeName: aws.String("rk"), + AttributeType: dynamodb.ScalarAttributeTypeB, + }, + }, + KeySchema: []dynamodb.KeySchemaElement{ + { + AttributeName: aws.String("hk"), + KeyType: dynamodb.KeyTypeHash, + }, { + AttributeName: aws.String("rk"), + KeyType: dynamodb.KeyTypeRange, + }, + }, + ProvisionedThroughput: &dynamodb.ProvisionedThroughput{ + ReadCapacityUnits: aws.Int64(25), + WriteCapacityUnits: aws.Int64(25), + }, + TableName: &tableName, + }).Send(); err != nil { + return err + } + return client.WaitUntilTableExists(&dynamodb.DescribeTableInput{ + TableName: aws.String(tableName), + }) +} + +func TestDynamoDBDatabase(t *testing.T) { + client, err := newDynamoDBTestClient() + require.NoError(t, err) + if client == nil { + t.Skip("launch a local dynamodb container to run this test: docker run --rm -it -p 8000:8000 dwmkerr/dynamodb -inMemory") + } + + const tableName = "TestDynamoDBDatabase" + + if _, err := client.DeleteTableRequest(&dynamodb.DeleteTableInput{ + TableName: aws.String(tableName), + }).Send(); err == nil { + require.NoError(t, client.WaitUntilTableNotExists(&dynamodb.DescribeTableInput{ + TableName: aws.String(tableName), + })) + } + + require.NoError(t, createDynamoDBTable(client, tableName)) + + db, err := NewDynamoDBDatabase(client, tableName) + require.NoError(t, err) + defer db.Close() + + testDatabase(t, db) +} diff --git a/server/forum_indexer.go b/server/forum_indexer.go index 6ee2ca4..e55cf73 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -23,7 +23,7 @@ type ForumIndexer struct { } type ForumIndexerConfiguration struct { - Database *Database + Database Database Session string } @@ -85,7 +85,9 @@ func (indexer *ForumIndexer) run() { case <-indexer.closeSignal: return default: - indexer.index(l, account, timezone) + if err := indexer.index(l, account, timezone); err != nil { + log.WithError(err).Error("error indexing forum account: " + account) + } time.Sleep(time.Second) } } @@ -190,7 +192,7 @@ func (indexer *ForumIndexer) forumPosts(locale *Locale, poster string, page int, return posts, nil } -func (indexer *ForumIndexer) index(locale *Locale, poster string, timezone *time.Location) { +func (indexer *ForumIndexer) index(locale *Locale, poster string, timezone *time.Location) error { logger := log.WithFields(log.Fields{ "host": locale.ForumHost(), "poster": poster, @@ -221,9 +223,11 @@ func (indexer *ForumIndexer) index(locale *Locale, poster string, timezone *time time.Sleep(time.Second) } - if len(activity) > 0 { - indexer.configuration.Database.AddActivity(activity) + if len(activity) == 0 { + return nil } + + return indexer.configuration.Database.AddActivity(activity) } func ScrapeForumTimezone(doc *goquery.Document) (*time.Location, error) { diff --git a/server/localization.go b/server/localization.go index f47aad1..83fd1f5 100644 --- a/server/localization.go +++ b/server/localization.go @@ -115,7 +115,6 @@ var ruMonthReplacer = strings.NewReplacer( "дек.", "Dec", ) -// TODO: add translations var Locales = []*Locale{ { IncludeReddit: true, diff --git a/server/reddit_indexer.go b/server/reddit_indexer.go index a65d150..5c7189a 100644 --- a/server/reddit_indexer.go +++ b/server/reddit_indexer.go @@ -12,7 +12,7 @@ import ( ) type RedditIndexerConfiguration struct { - Database *Database + Database Database } type RedditIndexer struct { @@ -50,7 +50,9 @@ func (indexer *RedditIndexer) run() { case <-indexer.closeSignal: return default: - indexer.index(users[next]) + if err := indexer.index(users[next]); err != nil { + log.WithError(err).Error("error indexing reddit user: " + users[next]) + } next += 1 if next >= len(users) { next = 0 @@ -144,7 +146,7 @@ func (indexer *RedditIndexer) redditActivity(user string, page string) ([]Activi return ParseRedditActivity(body) } -func (indexer *RedditIndexer) index(user string) { +func (indexer *RedditIndexer) index(user string) error { logger := log.WithFields(log.Fields{ "user": user, }) @@ -177,7 +179,9 @@ func (indexer *RedditIndexer) index(user string) { time.Sleep(time.Second * 2) } - if len(activity) > 0 { - indexer.configuration.Database.AddActivity(activity) + if len(activity) == 0 { + return nil } + + return indexer.configuration.Database.AddActivity(activity) } diff --git a/server/rss_handler.go b/server/rss_handler.go index 38bc934..8982272 100644 --- a/server/rss_handler.go +++ b/server/rss_handler.go @@ -41,9 +41,12 @@ type rssResponse struct { Channel rssChannel `xml:"channel"` } -func RSSHandler(db *Database) echo.HandlerFunc { +func RSSHandler(db Database) echo.HandlerFunc { return func(c echo.Context) error { - activity, _ := db.Activity(c.QueryParam("next"), 50, LocaleForRequest(c.Request()).ActivityFilter) + activity, _, err := db.Activity(LocaleForRequest(c.Request()), c.QueryParam("next"), 50) + if err != nil { + return err + } response := rssResponse{ Version: "2.0", Atom: "http://www.w3.org/2005/Atom", From 8aa1cffa0776f83fd2b229ab337c5370fec6d6e6 Mon Sep 17 00:00:00 2001 From: Chris Date: Mon, 26 Nov 2018 03:12:42 -0500 Subject: [PATCH 21/90] finish aws-sam template (#29) --- .gitignore | 1 + Dockerfile | 1 + Gopkg.lock | 71 ++++++++++- README.md | 2 +- aws-sam/.gitignore | 2 + aws-sam/Makefile | 18 +++ aws-sam/docker-compose.yaml | 15 +++ aws-sam/gggtracker.cfn.yaml | 243 +++++++++++++++++++++++++++++++++++- aws-sam/main.go | 160 ++++++++++++++++++++++++ main.go | 22 +--- server/index_handler.go | 1 + server/server.go | 50 ++++++++ 12 files changed, 563 insertions(+), 23 deletions(-) create mode 100644 aws-sam/.gitignore create mode 100644 aws-sam/Makefile create mode 100644 aws-sam/docker-compose.yaml create mode 100644 aws-sam/main.go create mode 100644 server/server.go diff --git a/.gitignore b/.gitignore index f744723..4edd60f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.db gggtracker +/server/bindata.go /vendor diff --git a/Dockerfile b/Dockerfile index 37ab1cc..4ab23ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,7 @@ RUN wget -O - https://raw.githubusercontent.com/golang/dep/master/install.sh | s ADD . . RUN dep ensure +RUN go generate ./... RUN go vet . && go test -v ./... RUN go build . diff --git a/Gopkg.lock b/Gopkg.lock index 2c713f1..a190ce6 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -13,6 +13,46 @@ revision = "901648c87902174f774fac311d7f176f8647bdaa" version = "v1.0.0" +[[projects]] + name = "github.com/aws/aws-lambda-go" + packages = [ + "events", + "lambda", + "lambda/messages", + "lambdacontext" + ] + revision = "fb8f88d824489a878aee1e0badde7d8e129f8767" + version = "v1.7.0" + +[[projects]] + name = "github.com/aws/aws-sdk-go-v2" + packages = [ + "aws", + "aws/awserr", + "aws/defaults", + "aws/ec2metadata", + "aws/ec2rolecreds", + "aws/endpointcreds", + "aws/endpoints", + "aws/external", + "aws/signer/v4", + "aws/stscreds", + "internal/awsutil", + "internal/ini", + "internal/sdk", + "private/protocol", + "private/protocol/json/jsonutil", + "private/protocol/jsonrpc", + "private/protocol/query", + "private/protocol/query/queryutil", + "private/protocol/rest", + "private/protocol/xml/xmlutil", + "service/dynamodb", + "service/sts" + ] + revision = "d52522b5f4b95591ff6528d7c54923951aadf099" + version = "v2.0.0-preview.5" + [[projects]] name = "github.com/boltdb/bolt" packages = ["."] @@ -53,6 +93,17 @@ ] revision = "23c074d0eceb2b8a5bfdbb271ab780cde70f05a8" +[[projects]] + name = "github.com/jmespath/go-jmespath" + packages = ["."] + revision = "0b12d6b5" + +[[projects]] + name = "github.com/json-iterator/go" + packages = ["."] + revision = "1624edc4454b8682399def8740d46db5e4362ba4" + version = "v1.1.5" + [[projects]] name = "github.com/labstack/echo" packages = [ @@ -97,12 +148,30 @@ packages = ["."] revision = "00c29f56e2386353d58c599509e8dc3801b0d716" +[[projects]] + name = "github.com/modern-go/concurrent" + packages = ["."] + revision = "bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94" + version = "1.0.3" + +[[projects]] + name = "github.com/modern-go/reflect2" + packages = ["."] + revision = "4b7aa43c6742a2c18fdef89dd197aaae7dac7ccd" + version = "1.0.1" + [[projects]] name = "github.com/pelletier/go-toml" packages = ["."] revision = "acdc4509485b587f5e675510c4f2c63e90ff68a8" version = "v1.1.0" +[[projects]] + name = "github.com/pkg/errors" + packages = ["."] + revision = "645ef00459ed84a119197bfb8d8205042c6df63d" + version = "v0.8.0" + [[projects]] name = "github.com/pmezard/go-difflib" packages = ["difflib"] @@ -219,6 +288,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "fb3f71f8b31a4ef69f8c12df7c17800632ad541fb57db8b99989ec33570b63fd" + inputs-digest = "6ff140fe2ee3cc2d088f6f049eac879676eaa9f3ddc7a5b4ca6c58e0b1f5e79f" solver-name = "gps-cdcl" solver-version = 1 diff --git a/README.md b/README.md index 992b553..6f92932 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This is the repository for [gggtracker.com](https://gggtracker.com). If there's ### Development -If you have Go installed, you can develop and run the server in the usual way: `go get ./... && go run main.go` +If you have Go installed, you can develop and run the server in the usual way: `go get ./... && go generate ./... && go run main.go` ![Development](development.gif) diff --git a/aws-sam/.gitignore b/aws-sam/.gitignore new file mode 100644 index 0000000..f6f05fb --- /dev/null +++ b/aws-sam/.gitignore @@ -0,0 +1,2 @@ +/build +/dist.zip diff --git a/aws-sam/Makefile b/aws-sam/Makefile new file mode 100644 index 0000000..6ee2e6c --- /dev/null +++ b/aws-sam/Makefile @@ -0,0 +1,18 @@ +.PHONY: build build-environment-build lambda-environment-build + +dist.zip: build + zip -j dist.zip build/main + +build: + dep ensure + go generate ../... + docker-compose run --rm build-environment make build-environment-build + docker-compose run --rm lambda-environment make lambda-environment-build + +build-environment-build: + rm -rf ./build + go build -o ./build/main . + ./build/main --help + +lambda-environment-build: + ./build/main --help || (ldd ./build/main && exit 1) diff --git a/aws-sam/docker-compose.yaml b/aws-sam/docker-compose.yaml new file mode 100644 index 0000000..bf9424b --- /dev/null +++ b/aws-sam/docker-compose.yaml @@ -0,0 +1,15 @@ +version: '3' +services: + build-environment: + image: golang:1.11 + volumes: + - ../:/go/src/github.com/ccbrown/gggtracker + working_dir: /go/src/github.com/ccbrown/gggtracker/aws-sam + lambda-environment: + entrypoint: [] + environment: + LD_LIBRARY_PATH: '' + image: lambci/lambda:go1.x + volumes: + - ../:/go/src/github.com/ccbrown/gggtracker + working_dir: /go/src/github.com/ccbrown/gggtracker/aws-sam diff --git a/aws-sam/gggtracker.cfn.yaml b/aws-sam/gggtracker.cfn.yaml index 814247e..a5807e7 100644 --- a/aws-sam/gggtracker.cfn.yaml +++ b/aws-sam/gggtracker.cfn.yaml @@ -1,6 +1,203 @@ AWSTemplateFormatVersion: '2010-09-09' Description: github.com/ccbrown/gggtracker +Parameters: + BinaryMediaType: + Type: String + AllowedValues: + - '*~1*' + - '*/*' + Description: See https://forums.aws.amazon.com/thread.jspa?messageID=797934 + CodeS3Bucket: + Type: String + Description: The bucket that contains the code to deploy. + CodeS3Key: + Type: String + Description: The key for the code to deploy. + CertificateARN: + Type: String + Description: A certificate corresponding to DomainName. + DomainName: + Type: String + Description: The desired domain name. It'll be up to you to actually change your DNS after the stack is deployed. + GoogleAnalytics: + Type: String + Default: '' Resources: + API: + Type: AWS::ApiGateway::RestApi + Properties: + BinaryMediaTypes: + - !Ref BinaryMediaType + Name: !Ref AWS::StackName + APIAccount: + Type: AWS::ApiGateway::Account + Properties: + CloudWatchRoleArn: !GetAtt APICloudWatchRole.Arn + APIBasePathMapping: + Type: AWS::ApiGateway::BasePathMapping + DependsOn: APIDomainName + Properties: + DomainName: !Ref DomainName + RestApiId: !Ref API + Stage: !Ref APIStage + APIBasePathMappingSubdomain1: + Type: AWS::ApiGateway::BasePathMapping + DependsOn: APISubdomainName1 + Properties: + DomainName: !Sub br.${DomainName} + RestApiId: !Ref API + Stage: !Ref APIStage + APIBasePathMappingSubdomain2: + Type: AWS::ApiGateway::BasePathMapping + DependsOn: APISubdomainName2 + Properties: + DomainName: !Sub ru.${DomainName} + RestApiId: !Ref API + Stage: !Ref APIStage + APIBasePathMappingSubdomain3: + Type: AWS::ApiGateway::BasePathMapping + DependsOn: APISubdomainName3 + Properties: + DomainName: !Sub th.${DomainName} + RestApiId: !Ref API + Stage: !Ref APIStage + APIBasePathMappingSubdomain4: + Type: AWS::ApiGateway::BasePathMapping + DependsOn: APISubdomainName4 + Properties: + DomainName: !Sub de.${DomainName} + RestApiId: !Ref API + Stage: !Ref APIStage + APIBasePathMappingSubdomain5: + Type: AWS::ApiGateway::BasePathMapping + DependsOn: APISubdomainName5 + Properties: + DomainName: !Sub fr.${DomainName} + RestApiId: !Ref API + Stage: !Ref APIStage + APIBasePathMappingSubdomain6: + Type: AWS::ApiGateway::BasePathMapping + DependsOn: APISubdomainName6 + Properties: + DomainName: !Sub es.${DomainName} + RestApiId: !Ref API + Stage: !Ref APIStage + APICloudWatchRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: apigateway.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AmazonAPIGatewayPushToCloudWatchLogs + APIDomainName: + Type: AWS::ApiGateway::DomainName + Properties: + CertificateArn: !Ref CertificateARN + DomainName: !Ref DomainName + EndpointConfiguration: + Types: + - EDGE + APISubdomainName1: + Type: AWS::ApiGateway::DomainName + Properties: + CertificateArn: !Ref CertificateARN + DomainName: !Sub br.${DomainName} + EndpointConfiguration: + Types: + - EDGE + APISubdomainName2: + Type: AWS::ApiGateway::DomainName + Properties: + CertificateArn: !Ref CertificateARN + DomainName: !Sub ru.${DomainName} + EndpointConfiguration: + Types: + - EDGE + APISubdomainName3: + Type: AWS::ApiGateway::DomainName + Properties: + CertificateArn: !Ref CertificateARN + DomainName: !Sub th.${DomainName} + EndpointConfiguration: + Types: + - EDGE + APISubdomainName4: + Type: AWS::ApiGateway::DomainName + Properties: + CertificateArn: !Ref CertificateARN + DomainName: !Sub de.${DomainName} + EndpointConfiguration: + Types: + - EDGE + APISubdomainName5: + Type: AWS::ApiGateway::DomainName + Properties: + CertificateArn: !Ref CertificateARN + DomainName: !Sub fr.${DomainName} + EndpointConfiguration: + Types: + - EDGE + APISubdomainName6: + Type: AWS::ApiGateway::DomainName + Properties: + CertificateArn: !Ref CertificateARN + DomainName: !Sub es.${DomainName} + EndpointConfiguration: + Types: + - EDGE + APIProxyResource: + Type: AWS::ApiGateway::Resource + Properties: + ParentId: !GetAtt API.RootResourceId + PathPart: '{proxy+}' + RestApiId: !Ref API + APIRootMethod: + Type: AWS::ApiGateway::Method + Properties: + AuthorizationType: NONE + HttpMethod: ANY + Integration: + Type: AWS_PROXY + IntegrationHttpMethod: POST + Uri: !Sub arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${APIFunction.Arn}/invocations + ResourceId: !GetAtt API.RootResourceId + RestApiId: !Ref API + APIProxyMethod: + Type: AWS::ApiGateway::Method + Properties: + AuthorizationType: NONE + HttpMethod: ANY + Integration: + Type: AWS_PROXY + IntegrationHttpMethod: POST + Uri: !Sub arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${APIFunction.Arn}/invocations + ResourceId: !Ref APIProxyResource + RestApiId: !Ref API + APIDeployment: + Type: AWS::ApiGateway::Deployment + DependsOn: + - APIRootMethod + - APIProxyMethod + Properties: + RestApiId: !Ref API + APIStage: + Type: AWS::ApiGateway::Stage + DependsOn: + - APIAccount + Properties: + DeploymentId: !Ref APIDeployment + MethodSettings: + - HttpMethod: '*' + LoggingLevel: ERROR + MetricsEnabled: true + ResourcePath: /* + RestApiId: !Ref API + StageName: stage DynamoDBTable: Type: AWS::DynamoDB::Table Properties: @@ -16,4 +213,48 @@ Resources: KeyType: RANGE ProvisionedThroughput: ReadCapacityUnits: 25 - WriteCapacityUnits: 25 + WriteCapacityUnits: 50 + APIFunction: + Type: AWS::Lambda::Function + Properties: + Code: + S3Bucket: !Ref CodeS3Bucket + S3Key: !Ref CodeS3Key + Environment: + Variables: + GGGTRACKER_DYNAMODB_TABLE: !Ref DynamoDBTable + GGGTRACKER_GA: !Ref GoogleAnalytics + Handler: main + MemorySize: 128 + Role: !GetAtt AppFunctionRole.Arn + Runtime: go1.x + Timeout: 60 + APIFunctionAPIPermission: + Type: AWS::Lambda::Permission + Properties: + Action: lambda:InvokeFunction + FunctionName: !GetAtt APIFunction.Arn + Principal: apigateway.amazonaws.com + SourceArn: !Sub arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}:${API}/*/*/* + AppFunctionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: lambda.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole + Policies: + - PolicyName: gggtracker + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: dynamodb:* + Resource: + - !Sub arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${DynamoDBTable} + - !Sub arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${DynamoDBTable}/* diff --git a/aws-sam/main.go b/aws-sam/main.go new file mode 100644 index 0000000..f9ca730 --- /dev/null +++ b/aws-sam/main.go @@ -0,0 +1,160 @@ +package main + +import ( + "bufio" + "bytes" + "encoding/base64" + "flag" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "os" + "strings" + + "github.com/aws/aws-lambda-go/events" + "github.com/aws/aws-lambda-go/lambda" + "github.com/aws/aws-sdk-go-v2/aws/external" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/crypto/ssh/terminal" + "golang.org/x/sys/unix" + + "github.com/ccbrown/gggtracker/server" +) + +type ResponseWriter struct { + header http.Header + statusCode int + body *bytes.Buffer +} + +func (w *ResponseWriter) Header() http.Header { + if w.header == nil { + w.header = http.Header{} + } + return w.header +} + +func (w *ResponseWriter) Write(b []byte) (int, error) { + if w.body == nil { + w.body = &bytes.Buffer{} + } + return w.body.Write(b) +} + +func (w *ResponseWriter) WriteHeader(statusCode int) { + w.statusCode = statusCode +} + +func (w *ResponseWriter) APIGatewayProxyResponse() (*events.APIGatewayProxyResponse, error) { + ret := &events.APIGatewayProxyResponse{} + if w.statusCode != 0 { + ret.StatusCode = w.statusCode + } else { + ret.StatusCode = http.StatusOK + } + for key, values := range w.header { + if len(values) > 1 { + return nil, fmt.Errorf("header has multiple values: " + key) + } else if len(values) == 1 { + if ret.Headers == nil { + ret.Headers = map[string]string{} + } + ret.Headers[key] = values[0] + } + } + if w.body != nil { + ret.Body = base64.StdEncoding.EncodeToString(w.body.Bytes()) + ret.IsBase64Encoded = true + } + return ret, nil +} + +func NewRequest(request *events.APIGatewayProxyRequest) (*http.Request, error) { + resource, err := url.ParseRequestURI(request.Path) + if err != nil { + return nil, errors.Wrap(err, "unable to parse request URI") + } + if len(request.QueryStringParameters) > 0 { + query := url.Values{} + for key, value := range request.QueryStringParameters { + query.Set(key, value) + } + resource.RawQuery = query.Encode() + } + req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(request.HTTPMethod + " " + resource.RequestURI() + " HTTP/1.0\r\n\r\n"))) + if err != nil { + return nil, errors.Wrap(err, "unable to create request") + } + + req.Proto = "HTTP/1.1" + req.ProtoMinor = 1 + + if request.Body != "" { + var body []byte + if request.IsBase64Encoded { + body, err = base64.StdEncoding.DecodeString(request.Body) + if err != nil { + return nil, errors.Wrap(err, "unable to decode base64 body") + } + } else { + body = []byte(request.Body) + } + req.ContentLength = int64(len(body)) + req.Body = ioutil.NopCloser(bytes.NewReader(body)) + } + + for key, value := range request.Headers { + req.Header.Set(key, value) + } + req.Host = req.Header.Get("Host") + req.RemoteAddr = request.RequestContext.Identity.SourceIP + + return req, nil +} + +func Handler(handler http.Handler) func(*events.APIGatewayProxyRequest) (*events.APIGatewayProxyResponse, error) { + return func(request *events.APIGatewayProxyRequest) (*events.APIGatewayProxyResponse, error) { + req, err := NewRequest(request) + if err != nil { + return nil, err + } + resp := &ResponseWriter{} + handler.ServeHTTP(resp, req) + return resp.APIGatewayProxyResponse() + } +} + +func StartHTTPHandler() { + config, err := external.LoadDefaultAWSConfig() + if err != nil { + logrus.Fatal(err) + } + db, err := server.NewDynamoDBDatabase(dynamodb.New(config), os.Getenv("GGGTRACKER_DYNAMODB_TABLE")) + if err != nil { + logrus.Fatal(err) + } + defer db.Close() + + e := server.New(db, os.Getenv("GGGTRACKER_GA")) + lambda.Start(Handler(e)) +} + +func main() { + if !terminal.IsTerminal(unix.Stdout) { + logrus.SetFormatter(&logrus.JSONFormatter{}) + } + + flags := flag.NewFlagSet(os.Args[0], flag.ContinueOnError) + if err := flags.Parse(os.Args[1:]); err != nil { + if err == flag.ErrHelp { + // Exit with no error if --help was given. This is used to test the build. + os.Exit(0) + } + logrus.Fatal(err) + } + + StartHTTPHandler() +} diff --git a/main.go b/main.go index 11b2da1..1df23f8 100644 --- a/main.go +++ b/main.go @@ -2,14 +2,10 @@ package main import ( "fmt" - "net/http" - "path" "strings" "github.com/aws/aws-sdk-go-v2/aws/external" "github.com/aws/aws-sdk-go-v2/service/dynamodb" - "github.com/labstack/echo" - "github.com/labstack/echo/middleware" log "github.com/sirupsen/logrus" "github.com/spf13/pflag" "github.com/spf13/viper" @@ -19,7 +15,7 @@ import ( func main() { pflag.IntP("port", "p", 8080, "the port to listen on") - pflag.String("staticdir", "./server/static", "the static files to serve") + pflag.String("staticdir", "", "this argument is ignored and will be removed") pflag.String("ga", "", "a google analytics account") pflag.String("db", "./gggtracker.db", "the database file path") pflag.String("dynamodb-table", "", "if given, DynamoDB will be used instead of a database file") @@ -31,8 +27,6 @@ func main() { viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) viper.AutomaticEnv() - e := echo.New() - var db server.Database var err error if tableName := viper.GetString("dynamodb-table"); tableName != "" { @@ -68,18 +62,6 @@ func main() { defer forumIndexer.Close() } - e.Use(middleware.Recover()) - - e.GET("/", server.IndexHandler(server.IndexConfiguration{ - GoogleAnalytics: viper.GetString("ga"), - })) - e.GET("/activity.json", server.ActivityHandler(db)) - e.GET("/rss", server.RSSHandler(db)) - e.GET("/rss.php", func(c echo.Context) error { - return c.Redirect(http.StatusMovedPermanently, server.AbsoluteURL(c, "/rss")) - }) - e.File("/favicon.ico", path.Join(viper.GetString("staticdir"), "favicon.ico")) - e.Static("/static", viper.GetString("staticdir")) - + e := server.New(db, viper.GetString("ga")) log.Fatal(e.Start(fmt.Sprintf(":%v", viper.GetInt("port")))) } diff --git a/server/index_handler.go b/server/index_handler.go index 3c420d6..ee7bc72 100644 --- a/server/index_handler.go +++ b/server/index_handler.go @@ -88,6 +88,7 @@ func init() { func IndexHandler(configuration IndexConfiguration) echo.HandlerFunc { return func(c echo.Context) error { + c.Response().Header().Set("Content-Type", "text/html; charset=utf-8") locale := LocaleForRequest(c.Request()) return indexTemplate.Execute(c.Response(), struct { Configuration IndexConfiguration diff --git a/server/server.go b/server/server.go new file mode 100644 index 0000000..d2ae0d3 --- /dev/null +++ b/server/server.go @@ -0,0 +1,50 @@ +//go:generate sh -c "go get -u github.com/kevinburke/go-bindata/... && `go env GOPATH`/bin/go-bindata -pkg server -ignore '(^|/)\\..*' static/..." +package server + +import ( + "bytes" + "net/http" + "net/url" + "path" + "path/filepath" + "time" + + "github.com/labstack/echo" + "github.com/labstack/echo/middleware" +) + +func serveAsset(c echo.Context, path string) error { + b, err := Asset(path) + if err != nil { + http.NotFound(c.Response(), c.Request()) + return nil + } + http.ServeContent(c.Response(), c.Request(), path, time.Time{}, bytes.NewReader(b)) + return nil +} + +func New(db Database, ga string) *echo.Echo { + e := echo.New() + e.Use(middleware.Recover()) + + e.GET("/", IndexHandler(IndexConfiguration{ + GoogleAnalytics: ga, + })) + e.GET("/activity.json", ActivityHandler(db)) + e.GET("/rss", RSSHandler(db)) + e.GET("/rss.php", func(c echo.Context) error { + return c.Redirect(http.StatusMovedPermanently, AbsoluteURL(c, "/rss")) + }) + e.GET("/favicon.ico", func(c echo.Context) error { + return serveAsset(c, "static/favicon.ico") + }) + e.GET("/static/*", func(c echo.Context) error { + p, err := url.PathUnescape(c.Param("*")) + if err != nil { + return err + } + return serveAsset(c, filepath.Join("static", path.Clean("/"+p))) + }) + + return e +} From bd2def44decc80eeeaa77eab2182fa3ed8930728 Mon Sep 17 00:00:00 2001 From: Chris Date: Mon, 26 Nov 2018 11:04:29 -0500 Subject: [PATCH 22/90] fewer writes (#30) --- server/forum_indexer.go | 10 +++++++--- server/reddit_indexer.go | 14 +++++++++----- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index e55cf73..7c859b2 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -198,7 +198,8 @@ func (indexer *ForumIndexer) index(locale *Locale, poster string, timezone *time "poster": poster, }) - cutoff := time.Now().Add(time.Hour * -12) + pageCutoff := time.Now().Add(-12 * time.Hour) + cutoff := time.Now().Add(-14 * 24 * time.Hour) activity := []Activity(nil) for page := 1; ; page++ { @@ -209,13 +210,16 @@ func (indexer *ForumIndexer) index(locale *Locale, poster string, timezone *time done := len(posts) == 0 for _, post := range posts { - if post.Time.Before(cutoff) { + if post.Time.Before(pageCutoff) { done = true } + if post.Time.Before(cutoff) { + break + } activity = append(activity, post) } - logger.WithField("count", len(posts)).Info("received forum posts") + logger.WithField("count", len(activity)).Info("received forum posts") if done { break diff --git a/server/reddit_indexer.go b/server/reddit_indexer.go index 5c7189a..628186b 100644 --- a/server/reddit_indexer.go +++ b/server/reddit_indexer.go @@ -57,7 +57,7 @@ func (indexer *RedditIndexer) run() { if next >= len(users) { next = 0 } - time.Sleep(time.Second * 2) + time.Sleep(time.Second * 3) } } } @@ -151,7 +151,8 @@ func (indexer *RedditIndexer) index(user string) error { "user": user, }) - cutoff := time.Now().Add(time.Hour * -12) + pageCutoff := time.Now().Add(-12 * time.Hour) + cutoff := time.Now().Add(-14 * 24 * time.Hour) activity := []Activity(nil) for page := ""; ; { @@ -163,20 +164,23 @@ func (indexer *RedditIndexer) index(user string) error { done := len(things) == 0 for _, thing := range things { - if thing.ActivityTime().Before(cutoff) { + if thing.ActivityTime().Before(pageCutoff) { done = true } + if thing.ActivityTime().Before(cutoff) { + break + } activity = append(activity, thing) } logger.WithFields(log.Fields{ - "count": len(things), + "count": len(activity), }).Info("received reddit activity") if done { break } - time.Sleep(time.Second * 2) + time.Sleep(time.Second * 3) } if len(activity) == 0 { From 9e65f0acf9be7f5b8ebf3f6c62853684cef1f3d7 Mon Sep 17 00:00:00 2001 From: Chris Date: Tue, 27 Nov 2018 23:54:55 -0500 Subject: [PATCH 23/90] fix www subdomain (#32) --- aws-sam/gggtracker.cfn.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/aws-sam/gggtracker.cfn.yaml b/aws-sam/gggtracker.cfn.yaml index a5807e7..2001a87 100644 --- a/aws-sam/gggtracker.cfn.yaml +++ b/aws-sam/gggtracker.cfn.yaml @@ -82,6 +82,13 @@ Resources: DomainName: !Sub es.${DomainName} RestApiId: !Ref API Stage: !Ref APIStage + APIBasePathMappingSubdomain7: + Type: AWS::ApiGateway::BasePathMapping + DependsOn: APISubdomainName7 + Properties: + DomainName: !Sub www.${DomainName} + RestApiId: !Ref API + Stage: !Ref APIStage APICloudWatchRole: Type: AWS::IAM::Role Properties: @@ -150,6 +157,14 @@ Resources: EndpointConfiguration: Types: - EDGE + APISubdomainName7: + Type: AWS::ApiGateway::DomainName + Properties: + CertificateArn: !Ref CertificateARN + DomainName: !Sub www.${DomainName} + EndpointConfiguration: + Types: + - EDGE APIProxyResource: Type: AWS::ApiGateway::Resource Properties: From 130cb2c2f8bfd772d92da47e178d145f89dbcd29 Mon Sep 17 00:00:00 2001 From: tgavankar <638772+tgavankar@users.noreply.github.com> Date: Tue, 8 Jan 2019 00:52:31 -0800 Subject: [PATCH 24/90] Add AndrewE_GGG to forum posters (#34) e.g. https://www.pathofexile.com/forum/view-thread/2282576/page/3 --- server/forum_indexer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index 7c859b2..6c3cf14 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -52,7 +52,7 @@ func (indexer *ForumIndexer) run() { "Alex_GGG", "Alexcc_GGG", "Andy", "CJ_GGG", "Eben_GGG", "Emma_GGG", "Ethan_GGG", "Fitzy_GGG", "Hartlin_GGG", "Jake_GGG", "Lionel_GGG", "Melissa_GGG", "MikeP_GGG", "Novynn", "Rachel_GGG", "Rob_GGG", "Roman_GGG", "Sarah_GGG", "SarahB_GGG", "Tom_GGG", "Natalia_GGG", - "Jeff_GGG", "Lu_GGG", "JuliaS_GGG", "Alexander_GGG", "SamC_GGG", + "Jeff_GGG", "Lu_GGG", "JuliaS_GGG", "Alexander_GGG", "SamC_GGG", "AndrewE_GGG", } timezone := (*time.Location)(nil) From adeac9fb69b761426008e66a2581865c3bed4bb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ernst=20Sj=C3=B6strand?= Date: Thu, 21 Mar 2019 20:21:29 +0100 Subject: [PATCH 25/90] Add Kyle_GGG to forum posters (#35) e.g. https://www.pathofexile.com/forum/view-thread/2461183 --- server/forum_indexer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index 6c3cf14..989a717 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -52,7 +52,7 @@ func (indexer *ForumIndexer) run() { "Alex_GGG", "Alexcc_GGG", "Andy", "CJ_GGG", "Eben_GGG", "Emma_GGG", "Ethan_GGG", "Fitzy_GGG", "Hartlin_GGG", "Jake_GGG", "Lionel_GGG", "Melissa_GGG", "MikeP_GGG", "Novynn", "Rachel_GGG", "Rob_GGG", "Roman_GGG", "Sarah_GGG", "SarahB_GGG", "Tom_GGG", "Natalia_GGG", - "Jeff_GGG", "Lu_GGG", "JuliaS_GGG", "Alexander_GGG", "SamC_GGG", "AndrewE_GGG", + "Jeff_GGG", "Lu_GGG", "JuliaS_GGG", "Alexander_GGG", "SamC_GGG", "AndrewE_GGG", "Kyle_GGG", } timezone := (*time.Location)(nil) From 8064ba7b4db1ca1416036aa6ebb2631d22b2212c Mon Sep 17 00:00:00 2001 From: Chris Date: Thu, 21 Mar 2019 17:08:44 -0400 Subject: [PATCH 26/90] locale support update (#37) --- server/database.go | 3 - server/database_test.go | 2 - server/forum_indexer.go | 69 ++++++++++--------- server/forum_indexer_test.go | 2 +- server/forum_post.go | 39 ++++++++++- server/localization.go | 128 ++++++++++------------------------- server/localization_test.go | 26 ++----- 7 files changed, 113 insertions(+), 156 deletions(-) diff --git a/server/database.go b/server/database.go index 653103d..d58dbe8 100644 --- a/server/database.go +++ b/server/database.go @@ -45,9 +45,6 @@ func unmarshalActivity(key, value []byte) (Activity, error) { if err != nil { return nil, err } - if post.Host == "" { - post.Host = "www.pathofexile.com" - } if post.Id != 0 { return post, nil } diff --git a/server/database_test.go b/server/database_test.go index d816b79..6654c54 100644 --- a/server/database_test.go +++ b/server/database_test.go @@ -21,14 +21,12 @@ func testDatabase_ForumPosts(t *testing.T, db Database) { Id: 9000, Poster: "Chris", Time: time.Unix(1486332365, 0), - Host: locale.ForumHost(), } post2 := &ForumPost{ Id: 9001, Poster: "Chris", Time: time.Unix(1486332364, 0), - Host: locale.ForumHost(), } db.AddActivity([]Activity{post1, post2}) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index 989a717..aa4969b 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -9,7 +9,6 @@ import ( "regexp" "strconv" "strings" - "sync" "time" "github.com/PuerkitoBio/goquery" @@ -73,33 +72,37 @@ func (indexer *ForumIndexer) run() { } } - var wg sync.WaitGroup - wg.Add(len(Locales)) - - for _, l := range Locales { - l := l - go func() { - for { - for _, account := range accounts { - select { - case <-indexer.closeSignal: - return - default: - if err := indexer.index(l, account, timezone); err != nil { - log.WithError(err).Error("error indexing forum account: " + account) - } - time.Sleep(time.Second) - } + for { + for _, locale := range Locales { + select { + case <-indexer.closeSignal: + return + default: + logger := log.WithField("host", locale.ForumHost()) + if err := locale.RefreshForumIds(); err != nil { + logger.WithError(err).Error("error refreshing forum ids") + } else { + logger.Info("refreshed forum ids") } + time.Sleep(time.Second) } - }() + } + for _, account := range accounts { + select { + case <-indexer.closeSignal: + return + default: + if err := indexer.index(account, timezone); err != nil { + log.WithError(err).Error("error indexing forum account: " + account) + } + time.Sleep(time.Second) + } + } } - - wg.Wait() } -func (indexer *ForumIndexer) requestDocument(host, resource string) (*goquery.Document, error) { - urlString := fmt.Sprintf("https://%v/%v", host, strings.TrimPrefix(resource, "/")) +func (indexer *ForumIndexer) requestDocument(resource string) (*goquery.Document, error) { + urlString := fmt.Sprintf("https://www.pathofexile.com/%v", strings.TrimPrefix(resource, "/")) jar, _ := cookiejar.New(nil) u, _ := url.Parse(urlString) jar.SetCookies(u, []*http.Cookie{ @@ -127,7 +130,7 @@ var postURLExpression = regexp.MustCompile("^/forum/view-post/([0-9]+)") var threadURLExpression = regexp.MustCompile("^/forum/view-thread/([0-9]+)") var forumURLExpression = regexp.MustCompile("^/forum/view-forum/([0-9]+)") -func ScrapeForumPosts(doc *goquery.Document, locale *Locale, timezone *time.Location) ([]*ForumPost, error) { +func ScrapeForumPosts(doc *goquery.Document, timezone *time.Location) ([]*ForumPost, error) { posts := []*ForumPost(nil) err := error(nil) @@ -145,7 +148,7 @@ func ScrapeForumPosts(doc *goquery.Document, locale *Locale, timezone *time.Loca timeText := sel.Find(".post_date").Text() - if post.Time, err = locale.ParseTime(timeText, timezone); err != nil { + if post.Time, err = time.ParseInLocation("Jan _2, 2006, 3:04:05 PM", timeText, timezone); err != nil { log.WithField("text", timeText).Error("unable to parse time") return false } @@ -177,24 +180,20 @@ func ScrapeForumPosts(doc *goquery.Document, locale *Locale, timezone *time.Loca return posts, nil } -func (indexer *ForumIndexer) forumPosts(locale *Locale, poster string, page int, timezone *time.Location) ([]*ForumPost, error) { - doc, err := indexer.requestDocument(locale.ForumHost(), fmt.Sprintf("/account/view-posts/%v/page/%v", poster, page)) +func (indexer *ForumIndexer) forumPosts(poster string, page int, timezone *time.Location) ([]*ForumPost, error) { + doc, err := indexer.requestDocument(fmt.Sprintf("/account/view-posts/%v/page/%v", poster, page)) if err != nil { return nil, err } - posts, err := ScrapeForumPosts(doc, locale, timezone) + posts, err := ScrapeForumPosts(doc, timezone) if err != nil { return nil, err } - for _, post := range posts { - post.Host = locale.ForumHost() - } return posts, nil } -func (indexer *ForumIndexer) index(locale *Locale, poster string, timezone *time.Location) error { +func (indexer *ForumIndexer) index(poster string, timezone *time.Location) error { logger := log.WithFields(log.Fields{ - "host": locale.ForumHost(), "poster": poster, }) @@ -203,7 +202,7 @@ func (indexer *ForumIndexer) index(locale *Locale, poster string, timezone *time activity := []Activity(nil) for page := 1; ; page++ { - posts, err := indexer.forumPosts(locale, poster, page, timezone) + posts, err := indexer.forumPosts(poster, page, timezone) if err != nil { logger.WithError(err).Error("error requesting forum posts") } @@ -243,7 +242,7 @@ func ScrapeForumTimezone(doc *goquery.Document) (*time.Location, error) { } func (indexer *ForumIndexer) sessionTimezone() (*time.Location, error) { - doc, err := indexer.requestDocument("www.pathofexile.com", "/my-account/preferences") + doc, err := indexer.requestDocument("/my-account/preferences") if err != nil { return nil, err } diff --git a/server/forum_indexer_test.go b/server/forum_indexer_test.go index 442b950..d2b8700 100644 --- a/server/forum_indexer_test.go +++ b/server/forum_indexer_test.go @@ -21,7 +21,7 @@ func TestScrapeForumPosts(t *testing.T) { tz, err := time.LoadLocation("America/Los_Angeles") require.NoError(t, err) - posts, err := ScrapeForumPosts(doc, Locales[0], tz) + posts, err := ScrapeForumPosts(doc, tz) require.NoError(t, err) assert.Equal(t, 10, len(posts)) diff --git a/server/forum_post.go b/server/forum_post.go index 3d4f0e7..1cd6e41 100644 --- a/server/forum_post.go +++ b/server/forum_post.go @@ -1,13 +1,13 @@ package server import ( + "encoding/json" "fmt" "time" ) type ForumPost struct { Id int `json:"id"` - Host string `json:"host"` BodyHTML string `json:"body_html"` Time time.Time `json:"time"` Poster string `json:"poster"` @@ -17,6 +17,32 @@ type ForumPost struct { ForumName string `json:"forum_name"` } +type forumPostWithHost struct { + Id int `json:"id"` + BodyHTML string `json:"body_html"` + Time time.Time `json:"time"` + Poster string `json:"poster"` + ThreadId int `json:"thread_id"` + ThreadTitle string `json:"thread_title"` + ForumId int `json:"forum_id"` + ForumName string `json:"forum_name"` + Host string `json:"host"` +} + +func (p ForumPost) MarshalJSON() ([]byte, error) { + return json.Marshal(&forumPostWithHost{ + Id: p.Id, + BodyHTML: p.BodyHTML, + Time: p.Time, + Poster: p.Poster, + ThreadId: p.ThreadId, + ThreadTitle: p.ThreadTitle, + ForumId: p.ForumId, + ForumName: p.ForumName, + Host: p.Host(), + }) +} + func (p *ForumPost) ActivityTime() time.Time { return p.Time } @@ -25,6 +51,15 @@ func (p *ForumPost) ActivityKey() uint32 { return uint32(p.Id) } +func (p *ForumPost) Host() string { + for _, l := range Locales { + if l.ForumIds()[p.ForumId] { + return l.ForumHost() + } + } + return "www.pathofexile.com" +} + func (p *ForumPost) PostURL() string { - return fmt.Sprintf("https://%v/forum/view-post/%v", p.Host, p.Id) + return fmt.Sprintf("https://%v/forum/view-post/%v", p.Host(), p.Id) } diff --git a/server/localization.go b/server/localization.go index 83fd1f5..a420eaa 100644 --- a/server/localization.go +++ b/server/localization.go @@ -1,9 +1,14 @@ package server import ( + "fmt" "net/http" + "strconv" "strings" + "sync/atomic" "time" + + "github.com/PuerkitoBio/goquery" ) type Locale struct { @@ -12,6 +17,8 @@ type Locale struct { IncludeReddit bool Translations map[string]string ParseTime func(s string, tz *time.Location) (time.Time, error) + + forumIds atomic.Value } func (l *Locale) Translate(s string) string { @@ -24,7 +31,7 @@ func (l *Locale) Translate(s string) string { func (l *Locale) ActivityFilter(a Activity) bool { switch a := a.(type) { case *ForumPost: - return a.Host == l.ForumHost() + return a.Host() == l.ForumHost() case *RedditComment: return l.IncludeReddit case *RedditPost: @@ -40,95 +47,47 @@ func (l *Locale) ForumHost() string { return "www.pathofexile.com" } -var esMonthReplacer = strings.NewReplacer( - "ene", "Jan", - "feb", "Feb", - "mar", "Mar", - "abr", "Apr", - "may", "May", - "jun", "Jun", - "jul", "Jul", - "ago", "Aug", - "sept", "Sep", - "oct", "Oct", - "nov", "Nov", - "dic", "Dec", -) +func (l *Locale) ForumIds() map[int]bool { + ret, _ := l.forumIds.Load().(map[int]bool) + return ret +} -var brMonthReplacer = strings.NewReplacer( - "de jan de", "Jan", - "de fev de", "Feb", - "de mar de", "Mar", - "de abr de", "Apr", - "de mai de", "May", - "de jun de", "Jun", - "de jul de", "Jul", - "de ago de", "Aug", - "de set de", "Sep", - "de out de", "Oct", - "de nov de", "Nov", - "de dez de", "Dec", -) +func (l *Locale) RefreshForumIds() error { + client := http.Client{ + Timeout: time.Second * 10, + } + resp, err := client.Get(fmt.Sprintf("https://%v/forum", l.ForumHost())) + if err != nil { + return err + } + defer resp.Body.Close() -var thMonthReplacer = strings.NewReplacer( - "ม.ค.", "Jan", - "ก.พ.", "Feb", - "มี.ค.", "Mar", - "เม.ย.", "Apr", - "พ.ค.", "May", - "มิ.ย.", "Jun", - "ก.ค.", "Jul", - "ส.ค.", "Aug", - "ก.ย.", "Sep", - "ต.ค.", "Oct", - "พ.ย.", "Nov", - "ธ.ค.", "Dec", -) + doc, err := goquery.NewDocumentFromReader(resp.Body) + if err != nil { + return err + } -var frMonthReplacer = strings.NewReplacer( - "janv.", "Jan", - "févr.", "Feb", - "mars", "Mar", - "avr.", "Apr", - "mai", "May", - "juin", "Jun", - "juil.", "Jul", - "août", "Aug", - "sept.", "Sep", - "oct.", "Oct", - "nov.", "Nov", - "déc.", "Dec", -) + forumIds := map[int]bool{} + doc.Find(".forumTable tbody tr").Each(func(i int, sel *goquery.Selection) { + if idStr := sel.AttrOr("data-id", ""); idStr != "" { + if id, err := strconv.Atoi(idStr); err == nil { + forumIds[id] = true + } + } + }) + l.forumIds.Store(forumIds) -var ruMonthReplacer = strings.NewReplacer( - "янв.", "Jan", - "февр.", "Feb", - "марта", "Mar", - "апр.", "Apr", - "мая", "May", - "июня", "Jun", - "июля", "Jul", - "авг.", "Aug", - "сент.", "Sep", - "окт.", "Oct", - "нояб.", "Nov", - "дек.", "Dec", -) + return nil +} var Locales = []*Locale{ { IncludeReddit: true, Image: "static/images/locales/gb.png", - ParseTime: func(s string, tz *time.Location) (time.Time, error) { - return time.ParseInLocation("Jan _2, 2006, 3:04:05 PM", s, tz) - }, }, { Subdomain: "br", Image: "static/images/locales/br.png", - ParseTime: func(s string, tz *time.Location) (time.Time, error) { - return time.ParseInLocation("2 Jan 2006 15:04:05", brMonthReplacer.Replace(s), tz) - }, Translations: map[string]string{ "Activity": "Atividade", "Thread": "Discussão", @@ -140,9 +99,6 @@ var Locales = []*Locale{ { Subdomain: "ru", Image: "static/images/locales/ru.png", - ParseTime: func(s string, tz *time.Location) (time.Time, error) { - return time.ParseInLocation("2 Jan 2006 г., 15:04:05", ruMonthReplacer.Replace(s), tz) - }, Translations: map[string]string{ "Activity": "Активность", "Thread": "Тема", @@ -154,16 +110,10 @@ var Locales = []*Locale{ { Subdomain: "th", Image: "static/images/locales/th.png", - ParseTime: func(s string, tz *time.Location) (time.Time, error) { - return time.ParseInLocation("_2 Jan 2006 15:04:05", thMonthReplacer.Replace(s), tz) - }, }, { Subdomain: "de", Image: "static/images/locales/de.png", - ParseTime: func(s string, tz *time.Location) (time.Time, error) { - return time.ParseInLocation("2.1.2006, 15:04:05", s, tz) - }, Translations: map[string]string{ "Activity": "Aktivität", "Thread": "Beitrag", @@ -175,9 +125,6 @@ var Locales = []*Locale{ { Subdomain: "fr", Image: "static/images/locales/fr.png", - ParseTime: func(s string, tz *time.Location) (time.Time, error) { - return time.ParseInLocation("_2 Jan 2006 15:04:05", frMonthReplacer.Replace(s), tz) - }, Translations: map[string]string{ "Activity": "Activité", "Thread": "Fil de discussion", @@ -189,9 +136,6 @@ var Locales = []*Locale{ { Subdomain: "es", Image: "static/images/locales/es.png", - ParseTime: func(s string, tz *time.Location) (time.Time, error) { - return time.ParseInLocation("2 Jan. 2006 15:04:05", esMonthReplacer.Replace(s), tz) - }, Translations: map[string]string{ "Activity": "Actividad", "Thread": "Tema", diff --git a/server/localization_test.go b/server/localization_test.go index 12b53e1..a50ff22 100644 --- a/server/localization_test.go +++ b/server/localization_test.go @@ -2,31 +2,15 @@ package server import ( "testing" - "time" "github.com/stretchr/testify/assert" ) -func TestLocale_ParseTime(t *testing.T) { - for subdomain, s := range map[string]string{ - "": "Aug 29, 2018, 5:51:19 PM", - "br": "31 de ago de 2018 00:50:19", - "ru": "1 сент. 2018 г., 2:09:52", - "th": "31 ส.ค. 2018 00:50:25", - "de": "31.08.2018, 00:50:20", - "fr": "31 août 2018 00:50:22", - "es": "31 ago. 2018 0:50:23", - } { - t.Run(subdomain, func(t *testing.T) { - var locale *Locale - for _, l := range Locales { - if l.Subdomain == subdomain { - locale = l - break - } - } - _, err := locale.ParseTime(s, time.FixedZone("UTC-5", -5*60*60)) - assert.NoError(t, err) +func TestLocale_RefreshForumIds(t *testing.T) { + for _, l := range Locales { + t.Run(l.ForumHost(), func(t *testing.T) { + assert.NoError(t, l.RefreshForumIds()) + assert.NotEmpty(t, l.ForumIds()) }) } } From 952678d4b078fec52714b2ad281e4019e0131356 Mon Sep 17 00:00:00 2001 From: Chris Date: Thu, 23 May 2019 14:36:42 -0400 Subject: [PATCH 27/90] fix extra www (#40) --- server/common.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/common.go b/server/common.go index f7d0297..6c35a07 100644 --- a/server/common.go +++ b/server/common.go @@ -23,7 +23,8 @@ func SubdomainURL(c echo.Context, subdomain string) string { scheme = "https" } locale := LocaleForRequest(c.Request()) - host := strings.TrimPrefix(c.Request().Host, locale.Subdomain+".") + host := strings.TrimPrefix(c.Request().Host, "www.") + host = strings.TrimPrefix(c.Request().Host, locale.Subdomain+".") if subdomain != "" { host = subdomain + "." + host } From 9b933aaae35c938fdd1d10518c475a1422abb5b7 Mon Sep 17 00:00:00 2001 From: Chris Date: Thu, 23 May 2019 15:08:06 -0400 Subject: [PATCH 28/90] fix extra www fix (#41) --- server/common.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/common.go b/server/common.go index 6c35a07..048b4cc 100644 --- a/server/common.go +++ b/server/common.go @@ -24,7 +24,7 @@ func SubdomainURL(c echo.Context, subdomain string) string { } locale := LocaleForRequest(c.Request()) host := strings.TrimPrefix(c.Request().Host, "www.") - host = strings.TrimPrefix(c.Request().Host, locale.Subdomain+".") + host = strings.TrimPrefix(host, locale.Subdomain+".") if subdomain != "" { host = subdomain + "." + host } From 86d1533dcfe242ba71911f3b7b4918039086fbbf Mon Sep 17 00:00:00 2001 From: Zzphyr <49877032+Zzphyr@users.noreply.github.com> Date: Fri, 28 Jun 2019 21:38:34 +0100 Subject: [PATCH 29/90] Update forum_indexer.go (#42) * Update forum_indexer.go Add "Stacey_GGG" as forum poster (https://www.pathofexile.com/account/view-profile/Stacey_GGG) * Add new GGG staff to list He is quite active at the bug forum atm - https://www.pathofexile.com/account/view-profile/Jatin_GGG --- server/forum_indexer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index aa4969b..1b6d8cd 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -51,7 +51,7 @@ func (indexer *ForumIndexer) run() { "Alex_GGG", "Alexcc_GGG", "Andy", "CJ_GGG", "Eben_GGG", "Emma_GGG", "Ethan_GGG", "Fitzy_GGG", "Hartlin_GGG", "Jake_GGG", "Lionel_GGG", "Melissa_GGG", "MikeP_GGG", "Novynn", "Rachel_GGG", "Rob_GGG", "Roman_GGG", "Sarah_GGG", "SarahB_GGG", "Tom_GGG", "Natalia_GGG", - "Jeff_GGG", "Lu_GGG", "JuliaS_GGG", "Alexander_GGG", "SamC_GGG", "AndrewE_GGG", "Kyle_GGG", + "Jeff_GGG", "Lu_GGG", "JuliaS_GGG", "Alexander_GGG", "SamC_GGG", "AndrewE_GGG", "Kyle_GGG", "Stacey_GGG", "Jatin_GGG" } timezone := (*time.Location)(nil) From b608db4d186f3a533ec0c801198e616c0e3c089e Mon Sep 17 00:00:00 2001 From: Chris Date: Fri, 28 Jun 2019 16:51:44 -0400 Subject: [PATCH 30/90] fix trailing comma --- server/forum_indexer.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index 1b6d8cd..89d1a6a 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -51,7 +51,8 @@ func (indexer *ForumIndexer) run() { "Alex_GGG", "Alexcc_GGG", "Andy", "CJ_GGG", "Eben_GGG", "Emma_GGG", "Ethan_GGG", "Fitzy_GGG", "Hartlin_GGG", "Jake_GGG", "Lionel_GGG", "Melissa_GGG", "MikeP_GGG", "Novynn", "Rachel_GGG", "Rob_GGG", "Roman_GGG", "Sarah_GGG", "SarahB_GGG", "Tom_GGG", "Natalia_GGG", - "Jeff_GGG", "Lu_GGG", "JuliaS_GGG", "Alexander_GGG", "SamC_GGG", "AndrewE_GGG", "Kyle_GGG", "Stacey_GGG", "Jatin_GGG" + "Jeff_GGG", "Lu_GGG", "JuliaS_GGG", "Alexander_GGG", "SamC_GGG", "AndrewE_GGG", "Kyle_GGG", + "Stacey_GGG", "Jatin_GGG", } timezone := (*time.Location)(nil) From d353323fabf0c3f98b0ba4baea353d603320ac33 Mon Sep 17 00:00:00 2001 From: Daniel Grigsby <17211674+dbgrigsby@users.noreply.github.com> Date: Mon, 26 Aug 2019 17:16:49 -0700 Subject: [PATCH 31/90] Add some missing GGG reddit accounts (#43) * Add some missing GGG reddit accounts All of these accounts post with the GGG flair on the subreddit, but are not tracked on GGG tracker, so here they are. * Remove some duplicates Co-Authored-By: Chris --- server/reddit_indexer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/reddit_indexer.go b/server/reddit_indexer.go index 628186b..9af2203 100644 --- a/server/reddit_indexer.go +++ b/server/reddit_indexer.go @@ -41,7 +41,7 @@ func (indexer *RedditIndexer) run() { "Mark_GGG", "RhysGGG", "Dan_GGG", "Rory_Rackham", "Blake_GGG", "Fitzy_GGG", "Hartlin_GGG", "Hrishi_GGG", "Baltic_GGG", "KamilOrmanJanowski", "Daniel_GGG", "Jeff_GGG", "NapfelGGG", "Baltic_GGG", "Novynn", "Felipe_GGG", "Mel_GGG", "Sarah_GGG", "riandrake", "Kieren_GGG", - "Openarl", + "Openarl", "Natalia_GGG", "pantherNZ", "Stacey_GGG", } next := 0 From 230e4efdd1e7bfb9ec160eac5cd3447e323f6744 Mon Sep 17 00:00:00 2001 From: Chris Date: Mon, 18 Nov 2019 17:10:47 -0500 Subject: [PATCH 32/90] housekeeping: dep -> mod, dependency updates (#46) --- .dockerignore | 3 +- Dockerfile | 9 +- Gopkg.lock | 293 ------------------------------- Gopkg.toml | 58 ------ go.mod | 43 +++++ go.sum | 86 +++++++++ server/dynamodb_database.go | 9 +- server/dynamodb_database_test.go | 15 +- 8 files changed, 148 insertions(+), 368 deletions(-) delete mode 100644 Gopkg.lock delete mode 100644 Gopkg.toml create mode 100644 go.mod create mode 100644 go.sum diff --git a/.dockerignore b/.dockerignore index 88cef9a..40d6004 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,5 @@ * -!Gopkg.* !*.go !server/* +!go.mod +!go.lock diff --git a/Dockerfile b/Dockerfile index 4ab23ca..9d601df 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,13 @@ -FROM golang:1.10-alpine +FROM golang:1.13-alpine WORKDIR /go/src/github.com/ccbrown/gggtracker -RUN apk add --no-cache git -RUN wget -O - https://raw.githubusercontent.com/golang/dep/master/install.sh | sh +RUN apk add --no-cache git g++ ADD . . -RUN dep ensure RUN go generate ./... -RUN go vet . && go test -v ./... +RUN go vet . +RUN go test -v ./... RUN go build . ENTRYPOINT ["./gggtracker"] diff --git a/Gopkg.lock b/Gopkg.lock deleted file mode 100644 index a190ce6..0000000 --- a/Gopkg.lock +++ /dev/null @@ -1,293 +0,0 @@ -# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. - - -[[projects]] - name = "github.com/PuerkitoBio/goquery" - packages = ["."] - revision = "09540e565986583836b5fc792fe951ec5fd201dd" - version = "v1.3.0" - -[[projects]] - name = "github.com/andybalholm/cascadia" - packages = ["."] - revision = "901648c87902174f774fac311d7f176f8647bdaa" - version = "v1.0.0" - -[[projects]] - name = "github.com/aws/aws-lambda-go" - packages = [ - "events", - "lambda", - "lambda/messages", - "lambdacontext" - ] - revision = "fb8f88d824489a878aee1e0badde7d8e129f8767" - version = "v1.7.0" - -[[projects]] - name = "github.com/aws/aws-sdk-go-v2" - packages = [ - "aws", - "aws/awserr", - "aws/defaults", - "aws/ec2metadata", - "aws/ec2rolecreds", - "aws/endpointcreds", - "aws/endpoints", - "aws/external", - "aws/signer/v4", - "aws/stscreds", - "internal/awsutil", - "internal/ini", - "internal/sdk", - "private/protocol", - "private/protocol/json/jsonutil", - "private/protocol/jsonrpc", - "private/protocol/query", - "private/protocol/query/queryutil", - "private/protocol/rest", - "private/protocol/xml/xmlutil", - "service/dynamodb", - "service/sts" - ] - revision = "d52522b5f4b95591ff6528d7c54923951aadf099" - version = "v2.0.0-preview.5" - -[[projects]] - name = "github.com/boltdb/bolt" - packages = ["."] - revision = "2f1ce7a837dcb8da3ec595b1dac9d0632f0f99e8" - version = "v1.3.1" - -[[projects]] - name = "github.com/davecgh/go-spew" - packages = ["spew"] - revision = "346938d642f2ec3594ed81d874461961cd0faa76" - version = "v1.1.0" - -[[projects]] - name = "github.com/dgrijalva/jwt-go" - packages = ["."] - revision = "dbeaa9332f19a944acb5736b4456cfcc02140e29" - version = "v3.1.0" - -[[projects]] - name = "github.com/fsnotify/fsnotify" - packages = ["."] - revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" - version = "v1.4.7" - -[[projects]] - branch = "master" - name = "github.com/hashicorp/hcl" - packages = [ - ".", - "hcl/ast", - "hcl/parser", - "hcl/scanner", - "hcl/strconv", - "hcl/token", - "json/parser", - "json/scanner", - "json/token" - ] - revision = "23c074d0eceb2b8a5bfdbb271ab780cde70f05a8" - -[[projects]] - name = "github.com/jmespath/go-jmespath" - packages = ["."] - revision = "0b12d6b5" - -[[projects]] - name = "github.com/json-iterator/go" - packages = ["."] - revision = "1624edc4454b8682399def8740d46db5e4362ba4" - version = "v1.1.5" - -[[projects]] - name = "github.com/labstack/echo" - packages = [ - ".", - "middleware" - ] - revision = "b338075a0fc6e1a0683dbf03d09b4957a289e26f" - version = "3.2.6" - -[[projects]] - name = "github.com/labstack/gommon" - packages = [ - "bytes", - "color", - "log", - "random" - ] - revision = "57409ada9da0f2afad6664c49502f8c50fbd8476" - version = "0.2.3" - -[[projects]] - name = "github.com/magiconair/properties" - packages = ["."] - revision = "c3beff4c2358b44d0493c7dda585e7db7ff28ae6" - version = "v1.7.6" - -[[projects]] - name = "github.com/mattn/go-colorable" - packages = ["."] - revision = "167de6bfdfba052fa6b2d3664c8f5272e23c9072" - version = "v0.0.9" - -[[projects]] - name = "github.com/mattn/go-isatty" - packages = ["."] - revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39" - version = "v0.0.3" - -[[projects]] - branch = "master" - name = "github.com/mitchellh/mapstructure" - packages = ["."] - revision = "00c29f56e2386353d58c599509e8dc3801b0d716" - -[[projects]] - name = "github.com/modern-go/concurrent" - packages = ["."] - revision = "bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94" - version = "1.0.3" - -[[projects]] - name = "github.com/modern-go/reflect2" - packages = ["."] - revision = "4b7aa43c6742a2c18fdef89dd197aaae7dac7ccd" - version = "1.0.1" - -[[projects]] - name = "github.com/pelletier/go-toml" - packages = ["."] - revision = "acdc4509485b587f5e675510c4f2c63e90ff68a8" - version = "v1.1.0" - -[[projects]] - name = "github.com/pkg/errors" - packages = ["."] - revision = "645ef00459ed84a119197bfb8d8205042c6df63d" - version = "v0.8.0" - -[[projects]] - name = "github.com/pmezard/go-difflib" - packages = ["difflib"] - revision = "792786c7400a136282c1664665ae0a8db921c6c2" - version = "v1.0.0" - -[[projects]] - name = "github.com/sirupsen/logrus" - packages = ["."] - revision = "d682213848ed68c0a260ca37d6dd5ace8423f5ba" - version = "v1.0.4" - -[[projects]] - name = "github.com/spf13/afero" - packages = [ - ".", - "mem" - ] - revision = "bb8f1927f2a9d3ab41c9340aa034f6b803f4359c" - version = "v1.0.2" - -[[projects]] - name = "github.com/spf13/cast" - packages = ["."] - revision = "8965335b8c7107321228e3e3702cab9832751bac" - version = "v1.2.0" - -[[projects]] - branch = "master" - name = "github.com/spf13/jwalterweatherman" - packages = ["."] - revision = "7c0cea34c8ece3fbeb2b27ab9b59511d360fb394" - -[[projects]] - name = "github.com/spf13/pflag" - packages = ["."] - revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66" - version = "v1.0.0" - -[[projects]] - name = "github.com/spf13/viper" - packages = ["."] - revision = "25b30aa063fc18e48662b86996252eabdcf2f0c7" - version = "v1.0.0" - -[[projects]] - name = "github.com/stretchr/testify" - packages = [ - "assert", - "require" - ] - revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71" - version = "v1.2.1" - -[[projects]] - branch = "master" - name = "github.com/valyala/bytebufferpool" - packages = ["."] - revision = "e746df99fe4a3986f4d4f79e13c1e0117ce9c2f7" - -[[projects]] - branch = "master" - name = "github.com/valyala/fasttemplate" - packages = ["."] - revision = "dcecefd839c4193db0d35b88ec65b4c12d360ab0" - -[[projects]] - branch = "master" - name = "golang.org/x/crypto" - packages = [ - "acme", - "acme/autocert", - "ssh/terminal" - ] - revision = "91a49db82a88618983a78a06c1cbd4e00ab749ab" - -[[projects]] - branch = "master" - name = "golang.org/x/net" - packages = [ - "html", - "html/atom" - ] - revision = "22ae77b79946ea320088417e4d50825671d82d57" - -[[projects]] - branch = "master" - name = "golang.org/x/sys" - packages = [ - "unix", - "windows" - ] - revision = "dd2ff4accc098aceecb86b36eaa7829b2a17b1c9" - -[[projects]] - name = "golang.org/x/text" - packages = [ - "internal/gen", - "internal/triegen", - "internal/ucd", - "transform", - "unicode/cldr", - "unicode/norm" - ] - revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" - version = "v0.3.0" - -[[projects]] - name = "gopkg.in/yaml.v2" - packages = ["."] - revision = "7f97868eec74b32b0982dd158a51a446d1da7eb5" - version = "v2.1.1" - -[solve-meta] - analyzer-name = "dep" - analyzer-version = 1 - inputs-digest = "6ff140fe2ee3cc2d088f6f049eac879676eaa9f3ddc7a5b4ca6c58e0b1f5e79f" - solver-name = "gps-cdcl" - solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml deleted file mode 100644 index e04d3c1..0000000 --- a/Gopkg.toml +++ /dev/null @@ -1,58 +0,0 @@ -# Gopkg.toml example -# -# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html -# for detailed Gopkg.toml documentation. -# -# required = ["github.com/user/thing/cmd/thing"] -# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] -# -# [[constraint]] -# name = "github.com/user/project" -# version = "1.0.0" -# -# [[constraint]] -# name = "github.com/user/project2" -# branch = "dev" -# source = "github.com/myfork/project2" -# -# [[override]] -# name = "github.com/x/y" -# version = "2.4.0" -# -# [prune] -# non-go = false -# go-tests = true -# unused-packages = true - - -[[constraint]] - name = "github.com/PuerkitoBio/goquery" - version = "1.3.0" - -[[constraint]] - name = "github.com/boltdb/bolt" - version = "1.3.1" - -[[constraint]] - name = "github.com/labstack/echo" - version = "3.2.6" - -[[constraint]] - name = "github.com/sirupsen/logrus" - version = "1.0.4" - -[[constraint]] - name = "github.com/spf13/pflag" - version = "1.0.0" - -[[constraint]] - name = "github.com/spf13/viper" - version = "1.0.0" - -[[constraint]] - name = "github.com/stretchr/testify" - version = "1.2.1" - -[prune] - go-tests = true - unused-packages = true diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..628aeff --- /dev/null +++ b/go.mod @@ -0,0 +1,43 @@ +module github.com/ccbrown/gggtracker + +go 1.13 + +require ( + github.com/PuerkitoBio/goquery v1.3.0 + github.com/andybalholm/cascadia v1.0.0 + github.com/aws/aws-lambda-go v1.7.0 + github.com/aws/aws-sdk-go-v2 v0.16.0 + github.com/boltdb/bolt v1.3.1 + github.com/davecgh/go-spew v1.1.1 + github.com/dgrijalva/jwt-go v3.1.0+incompatible + github.com/fsnotify/fsnotify v1.4.7 + github.com/hashicorp/hcl v0.0.0-20171017181929-23c074d0eceb + github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af + github.com/json-iterator/go v1.1.5 + github.com/kevinburke/go-bindata v3.16.0+incompatible // indirect + github.com/labstack/echo v3.2.6+incompatible + github.com/labstack/gommon v0.2.2-0.20170925052817-57409ada9da0 + github.com/magiconair/properties v1.7.6 + github.com/mattn/go-colorable v0.0.9 + github.com/mattn/go-isatty v0.0.3 + github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238 + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd + github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 + github.com/pelletier/go-toml v1.1.0 + github.com/pkg/errors v0.8.0 + github.com/pmezard/go-difflib v1.0.0 + github.com/sirupsen/logrus v1.0.4 + github.com/spf13/afero v1.0.2 + github.com/spf13/cast v1.2.0 + github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec + github.com/spf13/pflag v1.0.0 + github.com/spf13/viper v1.0.0 + github.com/stretchr/testify v1.2.2 + github.com/valyala/bytebufferpool v1.0.0 + github.com/valyala/fasttemplate v0.0.0-20170224212429-dcecefd839c4 + golang.org/x/crypto v0.0.0-20180228161326-91a49db82a88 + golang.org/x/net v0.0.0-20181201002055-351d144fa1fc + golang.org/x/sys v0.0.0-20180302081741-dd2ff4accc09 + golang.org/x/text v0.3.0 + gopkg.in/yaml.v2 v2.1.1 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..30194d0 --- /dev/null +++ b/go.sum @@ -0,0 +1,86 @@ +github.com/PuerkitoBio/goquery v1.3.0 h1:2LzdaeRwZjIMW7iKEei51jiCPB33mou4AI7QCzS4NgE= +github.com/PuerkitoBio/goquery v1.3.0/go.mod h1:T9ezsOHcCrDCgA8aF1Cqr3sSYbO/xgdy8/R/XiIMAhA= +github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o= +github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/aws/aws-lambda-go v1.7.0 h1:g3Ad7aw27B2lhQLIuK7Aha+cWSaHr7ZNlngveHkhZyo= +github.com/aws/aws-lambda-go v1.7.0/go.mod h1:zUsUQhAUjYzR8AuduJPCfhBuKWUaDbQiPOG+ouzmE1A= +github.com/aws/aws-sdk-go-v2 v0.16.0 h1:X5pkFnjRNdDEX18NwDGWMaWL5ocNQX0qIYEhEcsTy64= +github.com/aws/aws-sdk-go-v2 v0.16.0/go.mod h1:pFLIN9LDjOEwHfruGweAXEq0XaD6uRkY8FsRkxhuBIg= +github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.1.0+incompatible h1:FFziAwDQQ2dz1XClWMkwvukur3evtZx7x/wMHKM1i20= +github.com/dgrijalva/jwt-go v3.1.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/hashicorp/hcl v0.0.0-20171017181929-23c074d0eceb h1:1OvvPvZkn/yCQ3xBcM8y4020wdkMXPHLB4+NfoGWh4U= +github.com/hashicorp/hcl v0.0.0-20171017181929-23c074d0eceb/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= +github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/json-iterator/go v1.1.5 h1:gL2yXlmiIo4+t+y32d4WGwOjKGYcGOuyrg46vadswDE= +github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/kevinburke/go-bindata v3.16.0+incompatible h1:TFzFZop2KxGhqNwsyjgmIh5JOrpG940MZlm5gNbxr8g= +github.com/kevinburke/go-bindata v3.16.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= +github.com/labstack/echo v3.2.6+incompatible h1:28U/uXFFKBIP+VQIqq641LuYhMS7Br9ZlwEXERaohCc= +github.com/labstack/echo v3.2.6+incompatible/go.mod h1:0INS7j/VjnFxD4E2wkz67b8cVwCLbBmJyDaka6Cmk1s= +github.com/labstack/gommon v0.2.2-0.20170925052817-57409ada9da0 h1:7AIW1qc9sYYTZLamTsRKSmVvJDXkZZrIWXHDK4Gq4X0= +github.com/labstack/gommon v0.2.2-0.20170925052817-57409ada9da0/go.mod h1:/tj9csK2iPSBvn+3NLM9e52usepMtrd5ilFYA+wQNJ4= +github.com/magiconair/properties v1.7.6 h1:U+1DqNen04MdEPgFiIwdOUiqZ8qPa37xgogX/sd3+54= +github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238 h1:+MZW2uvHgN8kYvksEN3f7eFL2wpzk0GxmlFsMybWc7E= +github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/pelletier/go-toml v1.1.0 h1:cmiOvKzEunMsAxyhXSzpL5Q1CRKpVv0KQsnAIcSEVYM= +github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.0.4 h1:gzbtLsZC3Ic5PptoRG+kQj4L60qjK7H7XszrU163JNQ= +github.com/sirupsen/logrus v1.0.4/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/spf13/afero v1.0.2 h1:5bRmqmInNmNFkI9NG9O0Xc/Lgl9wOWWUUA/O8XZqTCo= +github.com/spf13/afero v1.0.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.2.0 h1:HHl1DSRbEQN2i8tJmtS6ViPyHx35+p51amrdsiTCrkg= +github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= +github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec h1:2ZXvIUGghLpdTVHR1UfvfrzoVlZaE/yOWC5LueIHZig= +github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.0 h1:oaPbdDe/x0UncahuwiPxW1GYJyilRAdsPnq3e1yaPcI= +github.com/spf13/pflag v1.0.0/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.0.0 h1:RUA/ghS2i64rlnn4ydTfblY8Og8QzcPtCcHvgMn+w/I= +github.com/spf13/viper v1.0.0/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= +github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v0.0.0-20170224212429-dcecefd839c4 h1:gKMu1Bf6QINDnvyZuTaACm9ofY+PRh+5vFz4oxBZeF8= +github.com/valyala/fasttemplate v0.0.0-20170224212429-dcecefd839c4/go.mod h1:50wTf68f99/Zt14pr046Tgt3Lp2vLyFZKzbFXTOabXw= +golang.org/x/crypto v0.0.0-20180228161326-91a49db82a88 h1:jLkAo/qlT9whgCLYC5GAJ9kcKrv3Wj8VCc4N+KJ4wpw= +golang.org/x/crypto v0.0.0-20180228161326-91a49db82a88/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180301190904-22ae77b79946 h1:Ovdd3aDjGOBx9xVuWVwMZWXbFQ3542Ve+TGnNVoE/Mc= +golang.org/x/net v0.0.0-20180301190904-22ae77b79946/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc h1:a3CU5tJYVj92DY2LaA1kUkrsqD5/3mLDhx2NcNqyW+0= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/sys v0.0.0-20180302081741-dd2ff4accc09 h1:wNPZbZUOH0tyqngVRXeF2iQm19+ssqyebJTCFBvxsow= +golang.org/x/sys v0.0.0-20180302081741-dd2ff4accc09/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.1.1 h1:fxK3tv8mQPVEgxu/S2LJ040LyqiajHt+syP0CdDS/Sc= +gopkg.in/yaml.v2 v2.1.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/server/dynamodb_database.go b/server/dynamodb_database.go index 1411b32..b339081 100644 --- a/server/dynamodb_database.go +++ b/server/dynamodb_database.go @@ -1,6 +1,7 @@ package server import ( + "context" "encoding/base64" "github.com/aws/aws-sdk-go-v2/aws" @@ -8,11 +9,11 @@ import ( ) type DynamoDBDatabase struct { - client *dynamodb.DynamoDB + client *dynamodb.Client tableName string } -func NewDynamoDBDatabase(client *dynamodb.DynamoDB, tableName string) (*DynamoDBDatabase, error) { +func NewDynamoDBDatabase(client *dynamodb.Client, tableName string) (*DynamoDBDatabase, error) { return &DynamoDBDatabase{ client: client, tableName: tableName, @@ -68,7 +69,7 @@ func (db *DynamoDBDatabase) AddActivity(activity []Activity) error { for len(unprocessed) > 0 { result, err := db.client.BatchWriteItemRequest(&dynamodb.BatchWriteItemInput{ RequestItems: unprocessed, - }).Send() + }).Send(context.Background()) if err != nil { return err } @@ -112,7 +113,7 @@ func (db *DynamoDBDatabase) Activity(locale *Locale, start string, count int) ([ ExclusiveStartKey: startKey, Limit: aws.Int64(int64(count - len(activity))), ScanIndexForward: aws.Bool(false), - }).Send() + }).Send(context.Background()) if err != nil { return nil, "", err } diff --git a/server/dynamodb_database_test.go b/server/dynamodb_database_test.go index a8a3f12..e448f0b 100644 --- a/server/dynamodb_database_test.go +++ b/server/dynamodb_database_test.go @@ -1,6 +1,7 @@ package server import ( + "context" "crypto/rand" "encoding/base64" "os" @@ -13,7 +14,7 @@ import ( "github.com/stretchr/testify/require" ) -func newDynamoDBTestClient() (*dynamodb.DynamoDB, error) { +func newDynamoDBTestClient() (*dynamodb.Client, error) { endpoint := os.Getenv("DYNAMODB_ENDPOINT") config := defaults.Config() @@ -41,7 +42,7 @@ func newDynamoDBTestClient() (*dynamodb.DynamoDB, error) { client := dynamodb.New(config) if endpoint == "" { - if _, err := client.ListTablesRequest(&dynamodb.ListTablesInput{}).Send(); err != nil { + if _, err := client.ListTablesRequest(&dynamodb.ListTablesInput{}).Send(context.Background()); err != nil { if err, ok := err.(awserr.Error); ok && err.Code() == "RequestError" { return nil, nil } @@ -50,7 +51,7 @@ func newDynamoDBTestClient() (*dynamodb.DynamoDB, error) { return client, nil } -func createDynamoDBTable(client *dynamodb.DynamoDB, tableName string) error { +func createDynamoDBTable(client *dynamodb.Client, tableName string) error { if _, err := client.CreateTableRequest(&dynamodb.CreateTableInput{ AttributeDefinitions: []dynamodb.AttributeDefinition{ { @@ -75,10 +76,10 @@ func createDynamoDBTable(client *dynamodb.DynamoDB, tableName string) error { WriteCapacityUnits: aws.Int64(25), }, TableName: &tableName, - }).Send(); err != nil { + }).Send(context.Background()); err != nil { return err } - return client.WaitUntilTableExists(&dynamodb.DescribeTableInput{ + return client.WaitUntilTableExists(context.Background(), &dynamodb.DescribeTableInput{ TableName: aws.String(tableName), }) } @@ -94,8 +95,8 @@ func TestDynamoDBDatabase(t *testing.T) { if _, err := client.DeleteTableRequest(&dynamodb.DeleteTableInput{ TableName: aws.String(tableName), - }).Send(); err == nil { - require.NoError(t, client.WaitUntilTableNotExists(&dynamodb.DescribeTableInput{ + }).Send(context.Background()); err == nil { + require.NoError(t, client.WaitUntilTableNotExists(context.Background(), &dynamodb.DescribeTableInput{ TableName: aws.String(tableName), })) } From 7f8c08a262b9af5ee6fe2a0b51e57e706d809c57 Mon Sep 17 00:00:00 2001 From: Christopher Brown Date: Tue, 4 Feb 2020 01:14:41 -0500 Subject: [PATCH 33/90] remove qarl, add yolandi --- server/forum_indexer.go | 4 ++-- server/reddit_indexer.go | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index 89d1a6a..42636d6 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -43,7 +43,7 @@ func (indexer *ForumIndexer) run() { log.Info("starting forum indexer") accounts := []string{ - "Chris", "Jonathan", "Erik", "Mark_GGG", "Samantha", "Rory", "Rhys", "Qarl", "Andrew_GGG", + "Chris", "Jonathan", "Erik", "Mark_GGG", "Samantha", "Rory", "Rhys", "Andrew_GGG", "Damien_GGG", "Joel_GGG", "Ari", "Thomas", "BrianWeissman", "Edwin_GGG", "Support", "Dylan", "MaxS", "Ammon_GGG", "Jess_GGG", "Robbie_GGG", "GGG_Neon", "Jason_GGG", "Henry_GGG", "Michael_GGG", "Bex_GGG", "Cagan_GGG", "Daniel_GGG", "Kieren_GGG", "Yeran_GGG", "Gary_GGG", @@ -52,7 +52,7 @@ func (indexer *ForumIndexer) run() { "Fitzy_GGG", "Hartlin_GGG", "Jake_GGG", "Lionel_GGG", "Melissa_GGG", "MikeP_GGG", "Novynn", "Rachel_GGG", "Rob_GGG", "Roman_GGG", "Sarah_GGG", "SarahB_GGG", "Tom_GGG", "Natalia_GGG", "Jeff_GGG", "Lu_GGG", "JuliaS_GGG", "Alexander_GGG", "SamC_GGG", "AndrewE_GGG", "Kyle_GGG", - "Stacey_GGG", "Jatin_GGG", + "Stacey_GGG", "Jatin_GGG", "Yolandi_GGG", } timezone := (*time.Location)(nil) diff --git a/server/reddit_indexer.go b/server/reddit_indexer.go index 9af2203..ed1592e 100644 --- a/server/reddit_indexer.go +++ b/server/reddit_indexer.go @@ -37,11 +37,11 @@ func (indexer *RedditIndexer) run() { log.Info("starting reddit indexer") users := []string{ - "chris_wilson", "Bex_GGG", "Negitivefrags", "Omnitect", "qarldev", "BrianWeissman_GGG", - "Mark_GGG", "RhysGGG", "Dan_GGG", "Rory_Rackham", "Blake_GGG", "Fitzy_GGG", "Hartlin_GGG", - "Hrishi_GGG", "Baltic_GGG", "KamilOrmanJanowski", "Daniel_GGG", "Jeff_GGG", "NapfelGGG", - "Baltic_GGG", "Novynn", "Felipe_GGG", "Mel_GGG", "Sarah_GGG", "riandrake", "Kieren_GGG", - "Openarl", "Natalia_GGG", "pantherNZ", "Stacey_GGG", + "chris_wilson", "Bex_GGG", "Negitivefrags", "Omnitect", "BrianWeissman_GGG", "Mark_GGG", + "RhysGGG", "Dan_GGG", "Rory_Rackham", "Blake_GGG", "Fitzy_GGG", "Hartlin_GGG", "Hrishi_GGG", + "Baltic_GGG", "KamilOrmanJanowski", "Daniel_GGG", "Jeff_GGG", "NapfelGGG", "Baltic_GGG", + "Novynn", "Felipe_GGG", "Mel_GGG", "Sarah_GGG", "riandrake", "Kieren_GGG", "Openarl", + "Natalia_GGG", "pantherNZ", "Stacey_GGG", } next := 0 From a748c90da37b8bb59b4fc46aa23aa88472f4b20b Mon Sep 17 00:00:00 2001 From: Alex Singleton Date: Fri, 14 Feb 2020 10:14:23 -0800 Subject: [PATCH 34/90] add new GGG reddit accounts, zaccie and viperesque (#52) announced in this post: https://www.pathofexile.com/forum/view-thread/2769628 their forum accounts don't have the GGG tag yet --- server/reddit_indexer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/reddit_indexer.go b/server/reddit_indexer.go index ed1592e..11d2dd3 100644 --- a/server/reddit_indexer.go +++ b/server/reddit_indexer.go @@ -41,7 +41,7 @@ func (indexer *RedditIndexer) run() { "RhysGGG", "Dan_GGG", "Rory_Rackham", "Blake_GGG", "Fitzy_GGG", "Hartlin_GGG", "Hrishi_GGG", "Baltic_GGG", "KamilOrmanJanowski", "Daniel_GGG", "Jeff_GGG", "NapfelGGG", "Baltic_GGG", "Novynn", "Felipe_GGG", "Mel_GGG", "Sarah_GGG", "riandrake", "Kieren_GGG", "Openarl", - "Natalia_GGG", "pantherNZ", "Stacey_GGG", + "Natalia_GGG", "pantherNZ", "Stacey_GGG", "ZaccieA", "viperesque", } next := 0 From 68e08204eab014f7a67396d3eb8a485726bf8e08 Mon Sep 17 00:00:00 2001 From: Alex Singleton Date: Thu, 9 Apr 2020 21:30:50 -0700 Subject: [PATCH 35/90] Add new GGG reddit account rach_ggg (#59) --- server/reddit_indexer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/reddit_indexer.go b/server/reddit_indexer.go index 11d2dd3..beade95 100644 --- a/server/reddit_indexer.go +++ b/server/reddit_indexer.go @@ -41,7 +41,7 @@ func (indexer *RedditIndexer) run() { "RhysGGG", "Dan_GGG", "Rory_Rackham", "Blake_GGG", "Fitzy_GGG", "Hartlin_GGG", "Hrishi_GGG", "Baltic_GGG", "KamilOrmanJanowski", "Daniel_GGG", "Jeff_GGG", "NapfelGGG", "Baltic_GGG", "Novynn", "Felipe_GGG", "Mel_GGG", "Sarah_GGG", "riandrake", "Kieren_GGG", "Openarl", - "Natalia_GGG", "pantherNZ", "Stacey_GGG", "ZaccieA", "viperesque", + "Natalia_GGG", "pantherNZ", "Stacey_GGG", "ZaccieA", "viperesque", "rach_ggg", } next := 0 From e1bfd9d6113cbe494ef9c844964868c2780a08ec Mon Sep 17 00:00:00 2001 From: Chris Date: Thu, 16 Apr 2020 15:21:06 -0400 Subject: [PATCH 36/90] use more specific forum post selector (#60) --- server/forum_indexer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index 42636d6..6e2577c 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -136,7 +136,7 @@ func ScrapeForumPosts(doc *goquery.Document, timezone *time.Location) ([]*ForumP err := error(nil) - doc.Find(".forumPostListTable tr").EachWithBreak(func(i int, sel *goquery.Selection) bool { + doc.Find(".forumPostListTable > tbody > tr").EachWithBreak(func(i int, sel *goquery.Selection) bool { post := &ForumPost{ Poster: sel.Find(".post_by_account").Text(), } From dcae1c5d72d8f3af214ced25e4290b1ef4d4ed88 Mon Sep 17 00:00:00 2001 From: Chris Date: Wed, 17 Jun 2020 22:40:14 -0400 Subject: [PATCH 37/90] gzip data (#62) --- server/database.go | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/server/database.go b/server/database.go index d58dbe8..e144320 100644 --- a/server/database.go +++ b/server/database.go @@ -1,7 +1,10 @@ package server import ( + "bytes" + "compress/gzip" "encoding/binary" + "io/ioutil" json "github.com/json-iterator/go" ) @@ -18,11 +21,16 @@ type Database interface { Close() error } +const gzipMarker = 0 + func marshalActivity(a Activity) (key, value []byte, err error) { - buf, err := json.Marshal(a) - if err != nil { + buf := &bytes.Buffer{} + buf.Write([]byte{gzipMarker}) + w := gzip.NewWriter(buf) + if err := json.NewEncoder(w).Encode(a); err != nil { return nil, nil, err } + w.Close() k := make([]byte, 10) binary.BigEndian.PutUint64(k, uint64(a.ActivityTime().Unix())<<24) switch a.(type) { @@ -34,10 +42,23 @@ func marshalActivity(a Activity) (key, value []byte, err error) { k[5] = RedditPostType } binary.BigEndian.PutUint32(k[6:], a.ActivityKey()) - return k, buf, nil + return k, buf.Bytes(), nil } func unmarshalActivity(key, value []byte) (Activity, error) { + if len(value) > 0 && value[0] == gzipMarker { + r, err := gzip.NewReader(bytes.NewReader(value[1:])) + if err != nil { + return nil, err + } + defer r.Close() + buf, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + value = buf + } + switch key[5] { case ForumPostType: post := &ForumPost{} From f506e6633ec66abd265d71d43d9c5153cdc40b98 Mon Sep 17 00:00:00 2001 From: Christopher Brown Date: Wed, 17 Jun 2020 23:35:46 -0400 Subject: [PATCH 38/90] update aws-sam build files --- aws-sam/Makefile | 1 - aws-sam/docker-compose.yaml | 2 +- go.mod | 2 +- go.sum | 2 ++ 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/aws-sam/Makefile b/aws-sam/Makefile index 6ee2e6c..55590af 100644 --- a/aws-sam/Makefile +++ b/aws-sam/Makefile @@ -4,7 +4,6 @@ dist.zip: build zip -j dist.zip build/main build: - dep ensure go generate ../... docker-compose run --rm build-environment make build-environment-build docker-compose run --rm lambda-environment make lambda-environment-build diff --git a/aws-sam/docker-compose.yaml b/aws-sam/docker-compose.yaml index bf9424b..1cce8ff 100644 --- a/aws-sam/docker-compose.yaml +++ b/aws-sam/docker-compose.yaml @@ -1,7 +1,7 @@ version: '3' services: build-environment: - image: golang:1.11 + image: golang:1.13 volumes: - ../:/go/src/github.com/ccbrown/gggtracker working_dir: /go/src/github.com/ccbrown/gggtracker/aws-sam diff --git a/go.mod b/go.mod index 628aeff..420293b 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/hashicorp/hcl v0.0.0-20171017181929-23c074d0eceb github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af github.com/json-iterator/go v1.1.5 - github.com/kevinburke/go-bindata v3.16.0+incompatible // indirect + github.com/kevinburke/go-bindata v3.21.0+incompatible // indirect github.com/labstack/echo v3.2.6+incompatible github.com/labstack/gommon v0.2.2-0.20170925052817-57409ada9da0 github.com/magiconair/properties v1.7.6 diff --git a/go.sum b/go.sum index 30194d0..46319ee 100644 --- a/go.sum +++ b/go.sum @@ -27,6 +27,8 @@ github.com/json-iterator/go v1.1.5 h1:gL2yXlmiIo4+t+y32d4WGwOjKGYcGOuyrg46vadswD github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/kevinburke/go-bindata v3.16.0+incompatible h1:TFzFZop2KxGhqNwsyjgmIh5JOrpG940MZlm5gNbxr8g= github.com/kevinburke/go-bindata v3.16.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= +github.com/kevinburke/go-bindata v3.21.0+incompatible h1:baK7hwFJDlAHrOqmE9U3u8tow1Uc5ihN9E/b7djcK2g= +github.com/kevinburke/go-bindata v3.21.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= github.com/labstack/echo v3.2.6+incompatible h1:28U/uXFFKBIP+VQIqq641LuYhMS7Br9ZlwEXERaohCc= github.com/labstack/echo v3.2.6+incompatible/go.mod h1:0INS7j/VjnFxD4E2wkz67b8cVwCLbBmJyDaka6Cmk1s= github.com/labstack/gommon v0.2.2-0.20170925052817-57409ada9da0 h1:7AIW1qc9sYYTZLamTsRKSmVvJDXkZZrIWXHDK4Gq4X0= From 5fe5e689fcb8c4cc315c4a0e7c89979c92aefed1 Mon Sep 17 00:00:00 2001 From: Alex Singleton Date: Sun, 29 Nov 2020 18:22:13 -0700 Subject: [PATCH 39/90] Add new Community_Team account to reddit indexer (#68) Announced here: https://old.reddit.com/r/pathofexile/comments/k3ipxx/the_community_team_account/ --- server/reddit_indexer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/reddit_indexer.go b/server/reddit_indexer.go index beade95..8a93da0 100644 --- a/server/reddit_indexer.go +++ b/server/reddit_indexer.go @@ -41,7 +41,7 @@ func (indexer *RedditIndexer) run() { "RhysGGG", "Dan_GGG", "Rory_Rackham", "Blake_GGG", "Fitzy_GGG", "Hartlin_GGG", "Hrishi_GGG", "Baltic_GGG", "KamilOrmanJanowski", "Daniel_GGG", "Jeff_GGG", "NapfelGGG", "Baltic_GGG", "Novynn", "Felipe_GGG", "Mel_GGG", "Sarah_GGG", "riandrake", "Kieren_GGG", "Openarl", - "Natalia_GGG", "pantherNZ", "Stacey_GGG", "ZaccieA", "viperesque", "rach_ggg", + "Natalia_GGG", "pantherNZ", "Stacey_GGG", "ZaccieA", "viperesque", "rach_ggg", "Community_Team", } next := 0 From dd4fbb8ba3f013fa3d0d00e42e618a14bb0b4a7b Mon Sep 17 00:00:00 2001 From: Christopher Brown Date: Sun, 29 Nov 2020 23:03:20 -0500 Subject: [PATCH 40/90] bug fix for fresh reddit users --- server/reddit_indexer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/reddit_indexer.go b/server/reddit_indexer.go index 8a93da0..7dd4334 100644 --- a/server/reddit_indexer.go +++ b/server/reddit_indexer.go @@ -162,7 +162,7 @@ func (indexer *RedditIndexer) index(user string) error { logger.WithError(err).Error("error requesting reddit activity") } - done := len(things) == 0 + done := len(things) == 0 || next == "" for _, thing := range things { if thing.ActivityTime().Before(pageCutoff) { done = true From fd812b611bfd9fda1d3937aa1b7dada3e083c20c Mon Sep 17 00:00:00 2001 From: DanNeely Date: Mon, 4 Jan 2021 23:21:23 -0500 Subject: [PATCH 41/90] Add Community_Team to forum posters (#71) untested fix for issue #70 --- server/forum_indexer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index 6e2577c..3647c05 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -52,7 +52,7 @@ func (indexer *ForumIndexer) run() { "Fitzy_GGG", "Hartlin_GGG", "Jake_GGG", "Lionel_GGG", "Melissa_GGG", "MikeP_GGG", "Novynn", "Rachel_GGG", "Rob_GGG", "Roman_GGG", "Sarah_GGG", "SarahB_GGG", "Tom_GGG", "Natalia_GGG", "Jeff_GGG", "Lu_GGG", "JuliaS_GGG", "Alexander_GGG", "SamC_GGG", "AndrewE_GGG", "Kyle_GGG", - "Stacey_GGG", "Jatin_GGG", "Yolandi_GGG", + "Stacey_GGG", "Jatin_GGG", "Yolandi_GGG", "Community_Team", } timezone := (*time.Location)(nil) From 26828f98b35f1afa13dfe1d1c76f40535d6e3951 Mon Sep 17 00:00:00 2001 From: Alex Denford Date: Fri, 16 Apr 2021 11:40:47 +1200 Subject: [PATCH 42/90] AlexDenfordGGG posts not tracked on Reddit #72 (#73) --- server/reddit_indexer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/reddit_indexer.go b/server/reddit_indexer.go index 7dd4334..482f5c5 100644 --- a/server/reddit_indexer.go +++ b/server/reddit_indexer.go @@ -41,7 +41,7 @@ func (indexer *RedditIndexer) run() { "RhysGGG", "Dan_GGG", "Rory_Rackham", "Blake_GGG", "Fitzy_GGG", "Hartlin_GGG", "Hrishi_GGG", "Baltic_GGG", "KamilOrmanJanowski", "Daniel_GGG", "Jeff_GGG", "NapfelGGG", "Baltic_GGG", "Novynn", "Felipe_GGG", "Mel_GGG", "Sarah_GGG", "riandrake", "Kieren_GGG", "Openarl", - "Natalia_GGG", "pantherNZ", "Stacey_GGG", "ZaccieA", "viperesque", "rach_ggg", "Community_Team", + "Natalia_GGG", "AlexDenfordGGG", "Stacey_GGG", "ZaccieA", "viperesque", "rach_ggg", "Community_Team", } next := 0 From 0dfd1430b84acaf8bb9701af63d32a4c42a75702 Mon Sep 17 00:00:00 2001 From: Alex Singleton Date: Mon, 19 Apr 2021 16:58:33 -0700 Subject: [PATCH 43/90] Add GGG account /u/M59Gar (#77) http://reddit.com/user/M59Gar Account belongs to Narrative Designer Matt Dymerski - https://www.pathofexile.com/forum/view-thread/2321645 --- server/reddit_indexer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/reddit_indexer.go b/server/reddit_indexer.go index 482f5c5..c6ed3b0 100644 --- a/server/reddit_indexer.go +++ b/server/reddit_indexer.go @@ -41,7 +41,7 @@ func (indexer *RedditIndexer) run() { "RhysGGG", "Dan_GGG", "Rory_Rackham", "Blake_GGG", "Fitzy_GGG", "Hartlin_GGG", "Hrishi_GGG", "Baltic_GGG", "KamilOrmanJanowski", "Daniel_GGG", "Jeff_GGG", "NapfelGGG", "Baltic_GGG", "Novynn", "Felipe_GGG", "Mel_GGG", "Sarah_GGG", "riandrake", "Kieren_GGG", "Openarl", - "Natalia_GGG", "AlexDenfordGGG", "Stacey_GGG", "ZaccieA", "viperesque", "rach_ggg", "Community_Team", + "Natalia_GGG", "AlexDenfordGGG", "Stacey_GGG", "ZaccieA", "viperesque", "rach_ggg", "Community_Team", "M59Gar", } next := 0 From b3e4e442d62f3c3bfa3f3d8e48a40b560edfe5a8 Mon Sep 17 00:00:00 2001 From: Alex Singleton Date: Mon, 19 Apr 2021 22:09:33 -0700 Subject: [PATCH 44/90] Add /u/Dominic_GGG (#79) * Add /u/Dominic_GGG https://www.reddit.com/user/Dominic_GGG Sorry for the double commit, looks like GGG is getting their developers on official accounts. * Update server/reddit_indexer.go Co-authored-by: Chris --- server/reddit_indexer.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/reddit_indexer.go b/server/reddit_indexer.go index c6ed3b0..c7139d3 100644 --- a/server/reddit_indexer.go +++ b/server/reddit_indexer.go @@ -41,7 +41,8 @@ func (indexer *RedditIndexer) run() { "RhysGGG", "Dan_GGG", "Rory_Rackham", "Blake_GGG", "Fitzy_GGG", "Hartlin_GGG", "Hrishi_GGG", "Baltic_GGG", "KamilOrmanJanowski", "Daniel_GGG", "Jeff_GGG", "NapfelGGG", "Baltic_GGG", "Novynn", "Felipe_GGG", "Mel_GGG", "Sarah_GGG", "riandrake", "Kieren_GGG", "Openarl", - "Natalia_GGG", "AlexDenfordGGG", "Stacey_GGG", "ZaccieA", "viperesque", "rach_ggg", "Community_Team", "M59Gar", + "Natalia_GGG", "AlexDenfordGGG", "Stacey_GGG", "ZaccieA", "viperesque", "rach_ggg", + "Community_Team", "M59Gar", "Dominic_GGG", } next := 0 From 6c29ce7d4f91f8397f4ddc161fd24b6011b1d9e7 Mon Sep 17 00:00:00 2001 From: Chris Date: Thu, 8 Jul 2021 22:16:13 -0400 Subject: [PATCH 45/90] set user agent when scraping forums (#82) * set user agent when scraping forums * travis -> github actions --- .github/workflows/push.yml | 12 ++++++++++++ .travis.yml | 6 ------ server/forum_indexer.go | 11 ++++++++++- 3 files changed, 22 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/push.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml new file mode 100644 index 0000000..f033944 --- /dev/null +++ b/.github/workflows/push.yml @@ -0,0 +1,12 @@ +name: Push +on: [push] +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Build + run: | + make docker-image diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index bf46dbe..0000000 --- a/.travis.yml +++ /dev/null @@ -1,6 +0,0 @@ -language: go -sudo: required -services: - - docker -script: - - make docker-image diff --git a/server/forum_indexer.go b/server/forum_indexer.go index 3647c05..1df85a7 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -16,6 +16,8 @@ import ( log "github.com/sirupsen/logrus" ) +const UserAgent = "gggtracker.com (github.com/ccbrown/gggtracker)" + type ForumIndexer struct { configuration ForumIndexerConfiguration closeSignal chan struct{} @@ -118,7 +120,14 @@ func (indexer *ForumIndexer) requestDocument(resource string) (*goquery.Document Jar: jar, Timeout: time.Second * 10, } - resp, err := client.Get(urlString) + + req, err := http.NewRequest("GET", urlString, nil) + req.Header.Set("User-Agent", UserAgent) + if err != nil { + return nil, err + } + + resp, err := client.Do(req) if err != nil { return nil, err } From 0f35328cd1d4ff7d843c0be27e6c2a35b428771d Mon Sep 17 00:00:00 2001 From: Oscar Oosterling <45697856+o1oo3@users.noreply.github.com> Date: Wed, 6 Jul 2022 00:56:10 +0200 Subject: [PATCH 46/90] Update reddit_indexer.go (#96) * Update reddit_indexer.go Added new community staff nick_GGG https://www.reddit.com/user/Nick_GGG/ https://www.reddit.com/r/pathofexile/comments/vqeque/comment/ier41zd/?context=10 * Update server/reddit_indexer.go Co-authored-by: Chris --- server/reddit_indexer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/reddit_indexer.go b/server/reddit_indexer.go index c7139d3..e1a966c 100644 --- a/server/reddit_indexer.go +++ b/server/reddit_indexer.go @@ -42,7 +42,7 @@ func (indexer *RedditIndexer) run() { "Baltic_GGG", "KamilOrmanJanowski", "Daniel_GGG", "Jeff_GGG", "NapfelGGG", "Baltic_GGG", "Novynn", "Felipe_GGG", "Mel_GGG", "Sarah_GGG", "riandrake", "Kieren_GGG", "Openarl", "Natalia_GGG", "AlexDenfordGGG", "Stacey_GGG", "ZaccieA", "viperesque", "rach_ggg", - "Community_Team", "M59Gar", "Dominic_GGG", + "Community_Team", "M59Gar", "Dominic_GGG", "nick_GGG", } next := 0 From fd1ca4be6f16c6ba2acce89a8692b9c668cc82a7 Mon Sep 17 00:00:00 2001 From: TwentyFiveEx <109299419+TwentyFiveEx@users.noreply.github.com> Date: Sun, 17 Jul 2022 18:07:06 -0700 Subject: [PATCH 47/90] Add ggg devs to forum indexer (#100) * Update forum_indexer.go Added missing ggg posters: Dominic, Nick, Guy, Ben, BenH, and Nav.. * Update forum_indexer.go trailing comma --- server/forum_indexer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index 1df85a7..4dbc5a6 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -55,6 +55,7 @@ func (indexer *ForumIndexer) run() { "Rachel_GGG", "Rob_GGG", "Roman_GGG", "Sarah_GGG", "SarahB_GGG", "Tom_GGG", "Natalia_GGG", "Jeff_GGG", "Lu_GGG", "JuliaS_GGG", "Alexander_GGG", "SamC_GGG", "AndrewE_GGG", "Kyle_GGG", "Stacey_GGG", "Jatin_GGG", "Yolandi_GGG", "Community_Team", + "Dominic_GGG", "Nick_GGG", "Guy_GGG", "Ben_GGG", "BenH_GGG", "Nav_GGG", } timezone := (*time.Location)(nil) From afe18dc0bdb8f2ad4508cd3e401d02fb4820b43e Mon Sep 17 00:00:00 2001 From: "Lothrik (MaXiMiUS)" Date: Sun, 17 Jul 2022 18:08:29 -0700 Subject: [PATCH 48/90] Add tracking of /r/PathOfExileBuilds (#89) Fixes #86 --- server/reddit_indexer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/reddit_indexer.go b/server/reddit_indexer.go index e1a966c..7af4bf8 100644 --- a/server/reddit_indexer.go +++ b/server/reddit_indexer.go @@ -93,7 +93,7 @@ func ParseRedditActivity(b []byte) ([]Activity, string, error) { } for _, thing := range root.Data.Children { - if thing.Data.SubredditId != "t5_2sf6m" { + if thing.Data.SubredditId != "t5_2sf6m" && thing.Data.SubredditId != "t5_2w3q8" { continue } switch thing.Kind { From 5c2da7c786a6c7fa4f6a714645ce0cfbede044f0 Mon Sep 17 00:00:00 2001 From: Oscar Oosterling <45697856+o1oo3@users.noreply.github.com> Date: Mon, 18 Jul 2022 03:09:58 +0200 Subject: [PATCH 49/90] nick_GGG to Nick_GGG (#99) * Update reddit_indexer.go Added new community staff nick_GGG https://www.reddit.com/user/Nick_GGG/ https://www.reddit.com/r/pathofexile/comments/vqeque/comment/ier41zd/?context=10 * Update server/reddit_indexer.go * nick_GGG changed to Nick_GGG for consistency with other _GGG names Co-authored-by: Chris --- server/reddit_indexer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/reddit_indexer.go b/server/reddit_indexer.go index 7af4bf8..4c6fe04 100644 --- a/server/reddit_indexer.go +++ b/server/reddit_indexer.go @@ -42,7 +42,7 @@ func (indexer *RedditIndexer) run() { "Baltic_GGG", "KamilOrmanJanowski", "Daniel_GGG", "Jeff_GGG", "NapfelGGG", "Baltic_GGG", "Novynn", "Felipe_GGG", "Mel_GGG", "Sarah_GGG", "riandrake", "Kieren_GGG", "Openarl", "Natalia_GGG", "AlexDenfordGGG", "Stacey_GGG", "ZaccieA", "viperesque", "rach_ggg", - "Community_Team", "M59Gar", "Dominic_GGG", "nick_GGG", + "Community_Team", "M59Gar", "Dominic_GGG", "Nick_GGG", } next := 0 From 2698ca927a1110e7fc0d23f33ac96f65feb71caf Mon Sep 17 00:00:00 2001 From: Chris Date: Mon, 8 Aug 2022 23:24:33 -0400 Subject: [PATCH 50/90] remove hardcoded subreddit (#104) --- go.mod | 2 +- go.sum | 2 ++ server/index_handler.go | 2 +- server/reddit_comment.go | 15 +++++++++++++-- server/reddit_indexer.go | 3 +++ server/reddit_indexer_test.go | 2 ++ server/reddit_post.go | 3 +++ server/static/index.js | 9 +++++---- 8 files changed, 30 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 420293b..14c1539 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/hashicorp/hcl v0.0.0-20171017181929-23c074d0eceb github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af github.com/json-iterator/go v1.1.5 - github.com/kevinburke/go-bindata v3.21.0+incompatible // indirect + github.com/kevinburke/go-bindata v3.23.0+incompatible // indirect github.com/labstack/echo v3.2.6+incompatible github.com/labstack/gommon v0.2.2-0.20170925052817-57409ada9da0 github.com/magiconair/properties v1.7.6 diff --git a/go.sum b/go.sum index 46319ee..1bb49eb 100644 --- a/go.sum +++ b/go.sum @@ -29,6 +29,8 @@ github.com/kevinburke/go-bindata v3.16.0+incompatible h1:TFzFZop2KxGhqNwsyjgmIh5 github.com/kevinburke/go-bindata v3.16.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= github.com/kevinburke/go-bindata v3.21.0+incompatible h1:baK7hwFJDlAHrOqmE9U3u8tow1Uc5ihN9E/b7djcK2g= github.com/kevinburke/go-bindata v3.21.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= +github.com/kevinburke/go-bindata v3.23.0+incompatible h1:rqNOXZlqrYhMVVAsQx8wuc+LaA73YcfbQ407wAykyS8= +github.com/kevinburke/go-bindata v3.23.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= github.com/labstack/echo v3.2.6+incompatible h1:28U/uXFFKBIP+VQIqq641LuYhMS7Br9ZlwEXERaohCc= github.com/labstack/echo v3.2.6+incompatible/go.mod h1:0INS7j/VjnFxD4E2wkz67b8cVwCLbBmJyDaka6Cmk1s= github.com/labstack/gommon v0.2.2-0.20170925052817-57409ada9da0 h1:7AIW1qc9sYYTZLamTsRKSmVvJDXkZZrIWXHDK4Gq4X0= diff --git a/server/index_handler.go b/server/index_handler.go index ee7bc72..b08c1f6 100644 --- a/server/index_handler.go +++ b/server/index_handler.go @@ -68,7 +68,7 @@ var index = `
- + ` diff --git a/server/reddit_comment.go b/server/reddit_comment.go index 8de747d..d5bae23 100644 --- a/server/reddit_comment.go +++ b/server/reddit_comment.go @@ -13,6 +13,9 @@ type RedditComment struct { PostId string `json:"post_id"` PostTitle string `json:"post_title"` Time time.Time `json:"time"` + + // Added in August 2022. Comments stored before then may not have this. + Subreddit string `json:"subreddit"` } func (c *RedditComment) ActivityTime() time.Time { @@ -25,9 +28,17 @@ func (c *RedditComment) ActivityKey() uint32 { } func (c *RedditComment) PostURL() string { - return fmt.Sprintf("https://www.reddit.com/r/pathofexile/comments/%v/", c.PostId) + subreddit := "pathofexile" + if c.Subreddit != "" { + subreddit = c.Subreddit + } + return fmt.Sprintf("https://www.reddit.com/r/%v/comments/%v/", subreddit, c.PostId) } func (c *RedditComment) CommentURL() string { - return fmt.Sprintf("https://www.reddit.com/r/pathofexile/comments/%v/-/%v/?context=3", c.PostId, c.Id) + subreddit := "pathofexile" + if c.Subreddit != "" { + subreddit = c.Subreddit + } + return fmt.Sprintf("https://www.reddit.com/r/%v/comments/%v/-/%v/?context=3", subreddit, c.PostId, c.Id) } diff --git a/server/reddit_indexer.go b/server/reddit_indexer.go index 4c6fe04..88f8e52 100644 --- a/server/reddit_indexer.go +++ b/server/reddit_indexer.go @@ -83,6 +83,7 @@ func ParseRedditActivity(b []byte) ([]Activity, string, error) { CreatedUTC float64 `json:"created_utc"` LinkId string `json:"link_id"` LinkTitle string `json:"link_title"` + Subreddit string `json:"subreddit"` } `json:"data"` } `json:"children"` } `json:"data"` @@ -105,6 +106,7 @@ func ParseRedditActivity(b []byte) ([]Activity, string, error) { PostId: strings.TrimPrefix(thing.Data.LinkId, "t3_"), PostTitle: thing.Data.LinkTitle, Time: time.Unix(int64(thing.Data.CreatedUTC), 0), + Subreddit: thing.Data.Subreddit, }) case "t3": activity = append(activity, &RedditPost{ @@ -115,6 +117,7 @@ func ParseRedditActivity(b []byte) ([]Activity, string, error) { Title: thing.Data.Title, URL: thing.Data.URL, Time: time.Unix(int64(thing.Data.CreatedUTC), 0), + Subreddit: thing.Data.Subreddit, }) } } diff --git a/server/reddit_indexer_test.go b/server/reddit_indexer_test.go index 13b1f94..6079a0d 100644 --- a/server/reddit_indexer_test.go +++ b/server/reddit_indexer_test.go @@ -30,6 +30,7 @@ func TestParseRedditComments(t *testing.T) { assert.Equal(t, "/r/pathofexile/comments/5q12qc/another_update_on_singapore_fibre_cuts/", post.Permalink) assert.Equal(t, "Another Update on Singapore Fibre Cuts", post.Title) assert.Equal(t, time.Unix(1485316926, 0), post.Time) + assert.Equal(t, "pathofexile", post.Subreddit) comment, ok := activity[1].(*RedditComment) require.True(t, ok) @@ -37,4 +38,5 @@ func TestParseRedditComments(t *testing.T) { assert.Equal(t, "5plxw0", comment.PostId) assert.Equal(t, "Development Manifesto: Solo Self-Found Support in 2.6.0", comment.PostTitle) assert.Equal(t, time.Unix(1485287813, 0), comment.Time) + assert.Equal(t, "pathofexile", comment.Subreddit) } diff --git a/server/reddit_post.go b/server/reddit_post.go index fa238a2..f910bd8 100644 --- a/server/reddit_post.go +++ b/server/reddit_post.go @@ -13,6 +13,9 @@ type RedditPost struct { BodyHTML string `json:"body_html,omitempty"` Permalink string `json:"permalink"` Time time.Time `json:"time"` + + // Added in August 2022. Comments stored before then may not have this. + Subreddit string `json:"subreddit"` } func (p *RedditPost) ActivityTime() time.Time { diff --git a/server/static/index.js b/server/static/index.js index 3677f7a..4b65dbe 100644 --- a/server/static/index.js +++ b/server/static/index.js @@ -30,6 +30,7 @@ function loadActivity() { for (var i = 0; i < data.activity.length; ++i) { var type = data.activity[i].type; var activity = data.activity[i].data; + var subreddit = activity.subreddit || 'pathofexile'; var $tr = $('
'); @@ -42,7 +43,7 @@ function loadActivity() { )); } else if (type == 'reddit_comment') { $tr.append($('').append($('') - .attr('href', 'https://www.reddit.com/r/pathofexile/comments/' + activity.post_id) + .attr('href', 'https://www.reddit.com/r/' + subreddit + '/comments/' + activity.post_id) .append($('')) )); } else { @@ -64,7 +65,7 @@ function loadActivity() { )); } else if (type == "reddit_comment") { $tr.append($('').append($('') - .attr('href', 'https://www.reddit.com/r/pathofexile/comments/' + activity.post_id + '/-/' + activity.id + '/?context=3') + .attr('href', 'https://www.reddit.com/r/' + subreddit + '/comments/' + activity.post_id + '/-/' + activity.id + '/?context=3') .text(activity.post_title) )); } @@ -90,8 +91,8 @@ function loadActivity() { )); } else { $tr.append($('').append($('') - .attr('href', 'https://www.reddit.com/r/pathofexile') - .text('pathofexile') + .attr('href', 'https://www.reddit.com/r/' + subreddit) + .text(subreddit) )); } From adf2a34fa5e09741e0207b307f3ee4ee03d7accd Mon Sep 17 00:00:00 2001 From: DanNeely Date: Sun, 30 Jul 2023 12:40:19 -0400 Subject: [PATCH 51/90] Update forum_indexer.go (#109) Added multiple GGG accounts to the forum tracker from this thread: "Will_GGG", "Scott_GGG", "JC_GGG", "Dylan_GGG", "Chulainn_GGG", https://www.pathofexile.com/forum/view-thread/3323720/filter-account-type/staff/page/2 --- server/forum_indexer.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index 4dbc5a6..e169b7f 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -55,7 +55,8 @@ func (indexer *ForumIndexer) run() { "Rachel_GGG", "Rob_GGG", "Roman_GGG", "Sarah_GGG", "SarahB_GGG", "Tom_GGG", "Natalia_GGG", "Jeff_GGG", "Lu_GGG", "JuliaS_GGG", "Alexander_GGG", "SamC_GGG", "AndrewE_GGG", "Kyle_GGG", "Stacey_GGG", "Jatin_GGG", "Yolandi_GGG", "Community_Team", - "Dominic_GGG", "Nick_GGG", "Guy_GGG", "Ben_GGG", "BenH_GGG", "Nav_GGG", + "Dominic_GGG", "Nick_GGG", "Guy_GGG", "Ben_GGG", "BenH_GGG", "Nav_GGG", "Will_GGG", + "Scott_GGG", "JC_GGG", "Dylan_GGG", "Chulainn_GGG", } timezone := (*time.Location)(nil) From 492770b659652a1e9381b1baab20e0d399cbeb4d Mon Sep 17 00:00:00 2001 From: Chris Brown Date: Sun, 30 Jul 2023 12:42:18 -0400 Subject: [PATCH 52/90] rm BrianWeissman_GGG --- server/forum_indexer.go | 6 +++--- server/reddit_indexer.go | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index e169b7f..ca84cf6 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -54,9 +54,9 @@ func (indexer *ForumIndexer) run() { "Fitzy_GGG", "Hartlin_GGG", "Jake_GGG", "Lionel_GGG", "Melissa_GGG", "MikeP_GGG", "Novynn", "Rachel_GGG", "Rob_GGG", "Roman_GGG", "Sarah_GGG", "SarahB_GGG", "Tom_GGG", "Natalia_GGG", "Jeff_GGG", "Lu_GGG", "JuliaS_GGG", "Alexander_GGG", "SamC_GGG", "AndrewE_GGG", "Kyle_GGG", - "Stacey_GGG", "Jatin_GGG", "Yolandi_GGG", "Community_Team", - "Dominic_GGG", "Nick_GGG", "Guy_GGG", "Ben_GGG", "BenH_GGG", "Nav_GGG", "Will_GGG", - "Scott_GGG", "JC_GGG", "Dylan_GGG", "Chulainn_GGG", + "Stacey_GGG", "Jatin_GGG", "Yolandi_GGG", "Community_Team", "Dominic_GGG", "Nick_GGG", + "Guy_GGG", "Ben_GGG", "BenH_GGG", "Nav_GGG", "Will_GGG", "Scott_GGG", "JC_GGG", "Dylan_GGG", + "Chulainn_GGG", } timezone := (*time.Location)(nil) diff --git a/server/reddit_indexer.go b/server/reddit_indexer.go index 88f8e52..b14f7b7 100644 --- a/server/reddit_indexer.go +++ b/server/reddit_indexer.go @@ -37,12 +37,12 @@ func (indexer *RedditIndexer) run() { log.Info("starting reddit indexer") users := []string{ - "chris_wilson", "Bex_GGG", "Negitivefrags", "Omnitect", "BrianWeissman_GGG", "Mark_GGG", - "RhysGGG", "Dan_GGG", "Rory_Rackham", "Blake_GGG", "Fitzy_GGG", "Hartlin_GGG", "Hrishi_GGG", - "Baltic_GGG", "KamilOrmanJanowski", "Daniel_GGG", "Jeff_GGG", "NapfelGGG", "Baltic_GGG", - "Novynn", "Felipe_GGG", "Mel_GGG", "Sarah_GGG", "riandrake", "Kieren_GGG", "Openarl", - "Natalia_GGG", "AlexDenfordGGG", "Stacey_GGG", "ZaccieA", "viperesque", "rach_ggg", - "Community_Team", "M59Gar", "Dominic_GGG", "Nick_GGG", + "chris_wilson", "Bex_GGG", "Negitivefrags", "Omnitect", "Mark_GGG", "RhysGGG", "Dan_GGG", + "Rory_Rackham", "Blake_GGG", "Fitzy_GGG", "Hartlin_GGG", "Hrishi_GGG", "Baltic_GGG", + "KamilOrmanJanowski", "Daniel_GGG", "Jeff_GGG", "NapfelGGG", "Baltic_GGG", "Novynn", + "Felipe_GGG", "Mel_GGG", "Sarah_GGG", "riandrake", "Kieren_GGG", "Openarl", "Natalia_GGG", + "AlexDenfordGGG", "Stacey_GGG", "ZaccieA", "viperesque", "rach_ggg", "Community_Team", + "M59Gar", "Dominic_GGG", "Nick_GGG", } next := 0 From 7e2d6317200c2d5a8452e59e2f27ed44e8de2cb9 Mon Sep 17 00:00:00 2001 From: Chris Brown Date: Sun, 30 Jul 2023 12:49:56 -0400 Subject: [PATCH 53/90] update go, use go:embed --- Dockerfile | 2 +- go.mod | 55 ++++++++++++++++++++++++++---------------------- go.sum | 18 ++++++---------- server/server.go | 12 +++++++---- 4 files changed, 46 insertions(+), 41 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9d601df..b41e4d1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.13-alpine +FROM golang:1.20-alpine WORKDIR /go/src/github.com/ccbrown/gggtracker diff --git a/go.mod b/go.mod index 14c1539..6e82471 100644 --- a/go.mod +++ b/go.mod @@ -1,43 +1,48 @@ module github.com/ccbrown/gggtracker -go 1.13 +go 1.20 require ( github.com/PuerkitoBio/goquery v1.3.0 - github.com/andybalholm/cascadia v1.0.0 github.com/aws/aws-lambda-go v1.7.0 github.com/aws/aws-sdk-go-v2 v0.16.0 github.com/boltdb/bolt v1.3.1 - github.com/davecgh/go-spew v1.1.1 - github.com/dgrijalva/jwt-go v3.1.0+incompatible - github.com/fsnotify/fsnotify v1.4.7 - github.com/hashicorp/hcl v0.0.0-20171017181929-23c074d0eceb - github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af github.com/json-iterator/go v1.1.5 - github.com/kevinburke/go-bindata v3.23.0+incompatible // indirect github.com/labstack/echo v3.2.6+incompatible - github.com/labstack/gommon v0.2.2-0.20170925052817-57409ada9da0 - github.com/magiconair/properties v1.7.6 - github.com/mattn/go-colorable v0.0.9 - github.com/mattn/go-isatty v0.0.3 - github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238 - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd - github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 - github.com/pelletier/go-toml v1.1.0 github.com/pkg/errors v0.8.0 - github.com/pmezard/go-difflib v1.0.0 github.com/sirupsen/logrus v1.0.4 - github.com/spf13/afero v1.0.2 - github.com/spf13/cast v1.2.0 - github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec github.com/spf13/pflag v1.0.0 github.com/spf13/viper v1.0.0 github.com/stretchr/testify v1.2.2 - github.com/valyala/bytebufferpool v1.0.0 - github.com/valyala/fasttemplate v0.0.0-20170224212429-dcecefd839c4 golang.org/x/crypto v0.0.0-20180228161326-91a49db82a88 - golang.org/x/net v0.0.0-20181201002055-351d144fa1fc golang.org/x/sys v0.0.0-20180302081741-dd2ff4accc09 - golang.org/x/text v0.3.0 - gopkg.in/yaml.v2 v2.1.1 +) + +require ( + github.com/BurntSushi/toml v1.3.2 // indirect + github.com/andybalholm/cascadia v1.0.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgrijalva/jwt-go v3.1.0+incompatible // indirect + github.com/fsnotify/fsnotify v1.4.7 // indirect + github.com/hashicorp/hcl v0.0.0-20171017181929-23c074d0eceb // indirect + github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af // indirect + github.com/labstack/gommon v0.2.2-0.20170925052817-57409ada9da0 // indirect + github.com/magiconair/properties v1.7.6 // indirect + github.com/mattn/go-colorable v0.0.9 // indirect + github.com/mattn/go-isatty v0.0.3 // indirect + github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 // indirect + github.com/pelletier/go-toml v1.1.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/spf13/afero v1.0.2 // indirect + github.com/spf13/cast v1.2.0 // indirect + github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasttemplate v0.0.0-20170224212429-dcecefd839c4 // indirect + golang.org/x/net v0.0.0-20181201002055-351d144fa1fc // indirect + golang.org/x/text v0.3.0 // indirect + gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect + gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect + gopkg.in/yaml.v2 v2.1.1 // indirect ) diff --git a/go.sum b/go.sum index 1bb49eb..3b536e7 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/PuerkitoBio/goquery v1.3.0 h1:2LzdaeRwZjIMW7iKEei51jiCPB33mou4AI7QCzS4NgE= github.com/PuerkitoBio/goquery v1.3.0/go.mod h1:T9ezsOHcCrDCgA8aF1Cqr3sSYbO/xgdy8/R/XiIMAhA= github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o= @@ -8,7 +10,6 @@ github.com/aws/aws-sdk-go-v2 v0.16.0 h1:X5pkFnjRNdDEX18NwDGWMaWL5ocNQX0qIYEhEcsT github.com/aws/aws-sdk-go-v2 v0.16.0/go.mod h1:pFLIN9LDjOEwHfruGweAXEq0XaD6uRkY8FsRkxhuBIg= github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.1.0+incompatible h1:FFziAwDQQ2dz1XClWMkwvukur3evtZx7x/wMHKM1i20= @@ -20,17 +21,10 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/hashicorp/hcl v0.0.0-20171017181929-23c074d0eceb h1:1OvvPvZkn/yCQ3xBcM8y4020wdkMXPHLB4+NfoGWh4U= github.com/hashicorp/hcl v0.0.0-20171017181929-23c074d0eceb/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= -github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/json-iterator/go v1.1.5 h1:gL2yXlmiIo4+t+y32d4WGwOjKGYcGOuyrg46vadswDE= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/kevinburke/go-bindata v3.16.0+incompatible h1:TFzFZop2KxGhqNwsyjgmIh5JOrpG940MZlm5gNbxr8g= -github.com/kevinburke/go-bindata v3.16.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= -github.com/kevinburke/go-bindata v3.21.0+incompatible h1:baK7hwFJDlAHrOqmE9U3u8tow1Uc5ihN9E/b7djcK2g= -github.com/kevinburke/go-bindata v3.21.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= -github.com/kevinburke/go-bindata v3.23.0+incompatible h1:rqNOXZlqrYhMVVAsQx8wuc+LaA73YcfbQ407wAykyS8= -github.com/kevinburke/go-bindata v3.23.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= github.com/labstack/echo v3.2.6+incompatible h1:28U/uXFFKBIP+VQIqq641LuYhMS7Br9ZlwEXERaohCc= github.com/labstack/echo v3.2.6+incompatible/go.mod h1:0INS7j/VjnFxD4E2wkz67b8cVwCLbBmJyDaka6Cmk1s= github.com/labstack/gommon v0.2.2-0.20170925052817-57409ada9da0 h1:7AIW1qc9sYYTZLamTsRKSmVvJDXkZZrIWXHDK4Gq4X0= @@ -65,7 +59,6 @@ github.com/spf13/pflag v1.0.0 h1:oaPbdDe/x0UncahuwiPxW1GYJyilRAdsPnq3e1yaPcI= github.com/spf13/pflag v1.0.0/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.0.0 h1:RUA/ghS2i64rlnn4ydTfblY8Og8QzcPtCcHvgMn+w/I= github.com/spf13/viper v1.0.0/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= -github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= @@ -75,8 +68,6 @@ github.com/valyala/fasttemplate v0.0.0-20170224212429-dcecefd839c4/go.mod h1:50w golang.org/x/crypto v0.0.0-20180228161326-91a49db82a88 h1:jLkAo/qlT9whgCLYC5GAJ9kcKrv3Wj8VCc4N+KJ4wpw= golang.org/x/crypto v0.0.0-20180228161326-91a49db82a88/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180301190904-22ae77b79946 h1:Ovdd3aDjGOBx9xVuWVwMZWXbFQ3542Ve+TGnNVoE/Mc= -golang.org/x/net v0.0.0-20180301190904-22ae77b79946/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc h1:a3CU5tJYVj92DY2LaA1kUkrsqD5/3mLDhx2NcNqyW+0= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -85,6 +76,11 @@ golang.org/x/sys v0.0.0-20180302081741-dd2ff4accc09/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo= +gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 h1:OAj3g0cR6Dx/R07QgQe8wkA9RNjB2u4i700xBkIT4e0= +gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/yaml.v2 v2.1.1 h1:fxK3tv8mQPVEgxu/S2LJ040LyqiajHt+syP0CdDS/Sc= gopkg.in/yaml.v2 v2.1.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/server/server.go b/server/server.go index d2ae0d3..091c5d4 100644 --- a/server/server.go +++ b/server/server.go @@ -1,8 +1,8 @@ -//go:generate sh -c "go get -u github.com/kevinburke/go-bindata/... && `go env GOPATH`/bin/go-bindata -pkg server -ignore '(^|/)\\..*' static/..." package server import ( - "bytes" + "embed" + "io" "net/http" "net/url" "path" @@ -13,13 +13,17 @@ import ( "github.com/labstack/echo/middleware" ) +//go:embed static/* +var static embed.FS + func serveAsset(c echo.Context, path string) error { - b, err := Asset(path) + f, err := static.Open(path) if err != nil { http.NotFound(c.Response(), c.Request()) return nil } - http.ServeContent(c.Response(), c.Request(), path, time.Time{}, bytes.NewReader(b)) + defer f.Close() + http.ServeContent(c.Response(), c.Request(), path, time.Time{}, f.(io.ReadSeeker)) return nil } From b1e24215564e47412d716b645e9bf00f9cc8bed2 Mon Sep 17 00:00:00 2001 From: Chris Brown Date: Sun, 30 Jul 2023 13:03:36 -0400 Subject: [PATCH 54/90] modernize build, set user-agent header for forum listing --- .dockerignore | 2 +- Dockerfile | 3 --- README.md | 2 +- server/localization.go | 13 ++++++++++++- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/.dockerignore b/.dockerignore index 40d6004..2d578ec 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,4 +2,4 @@ !*.go !server/* !go.mod -!go.lock +!go.sum diff --git a/Dockerfile b/Dockerfile index b41e4d1..6cacd63 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,10 +2,7 @@ FROM golang:1.20-alpine WORKDIR /go/src/github.com/ccbrown/gggtracker -RUN apk add --no-cache git g++ - ADD . . -RUN go generate ./... RUN go vet . RUN go test -v ./... RUN go build . diff --git a/README.md b/README.md index 6f92932..94665e0 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This is the repository for [gggtracker.com](https://gggtracker.com). If there's ### Development -If you have Go installed, you can develop and run the server in the usual way: `go get ./... && go generate ./... && go run main.go` +If you have Go installed, you can develop and run the server in the usual way: `go run main.go` ![Development](development.gif) diff --git a/server/localization.go b/server/localization.go index a420eaa..d558f98 100644 --- a/server/localization.go +++ b/server/localization.go @@ -56,12 +56,23 @@ func (l *Locale) RefreshForumIds() error { client := http.Client{ Timeout: time.Second * 10, } - resp, err := client.Get(fmt.Sprintf("https://%v/forum", l.ForumHost())) + + req, err := http.NewRequest("GET", fmt.Sprintf("https://%v/forum", l.ForumHost()), nil) + if err != nil { + return err + } + req.Header.Set("User-Agent", UserAgent) + + resp, err := client.Do(req) if err != nil { return err } defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("unexpected status code: %v", resp.StatusCode) + } + doc, err := goquery.NewDocumentFromReader(resp.Body) if err != nil { return err From 6b61fb6295799e8b5df14f944bb597e9280a5472 Mon Sep 17 00:00:00 2001 From: derekmcq <152669322+derekmcq@users.noreply.github.com> Date: Mon, 11 Dec 2023 14:35:04 -0600 Subject: [PATCH 55/90] add jp locale ("sort of" fixes ccbrown/gggtracker#108) (#120) * add jp locale * change translations to sound more natural Co-authored-by: Chris --------- Co-authored-by: Chris --- aws-sam/gggtracker.cfn.yaml | 15 +++++++++++++++ server/localization.go | 11 +++++++++++ server/static/images/locales/jp.png | Bin 0 -> 209 bytes 3 files changed, 26 insertions(+) create mode 100644 server/static/images/locales/jp.png diff --git a/aws-sam/gggtracker.cfn.yaml b/aws-sam/gggtracker.cfn.yaml index 2001a87..b6bf81a 100644 --- a/aws-sam/gggtracker.cfn.yaml +++ b/aws-sam/gggtracker.cfn.yaml @@ -89,6 +89,13 @@ Resources: DomainName: !Sub www.${DomainName} RestApiId: !Ref API Stage: !Ref APIStage + APIBasePathMappingSubdomain8: + Type: AWS::ApiGateway::BasePathMapping + DependsOn: APISubdomainName8 + Properties: + DomainName: !Sub jp.${DomainName} + RestApiId: !Ref API + Stage: !Ref APIStage APICloudWatchRole: Type: AWS::IAM::Role Properties: @@ -165,6 +172,14 @@ Resources: EndpointConfiguration: Types: - EDGE + APISubdomainName8: + Type: AWS::ApiGateway::DomainName + Properties: + CertificateArn: !Ref CertificateARN + DomainName: !Sub jp.${DomainName} + EndpointConfiguration: + Types: + - EDGE APIProxyResource: Type: AWS::ApiGateway::Resource Properties: diff --git a/server/localization.go b/server/localization.go index d558f98..f09210e 100644 --- a/server/localization.go +++ b/server/localization.go @@ -155,6 +155,17 @@ var Locales = []*Locale{ "Forum": "Foro", }, }, + { + Subdomain: "jp", + Image: "static/images/locales/jp.png", + Translations: map[string]string{ + "Activity": "アクティビティ", + "Thread": "スレッド", + "Poster": "投稿者", + "Time": "日時", + "Forum": "フォーラム", + }, + }, } func LocaleForRequest(r *http.Request) *Locale { diff --git a/server/static/images/locales/jp.png b/server/static/images/locales/jp.png new file mode 100644 index 0000000000000000000000000000000000000000..a432ef8738649d935444480e6b5f69757707a92c GIT binary patch literal 209 zcmeAS@N?(olHy`uVBq!ia0vp^B0$W~!3-oXyUC{mDc1m>5ZC|z|9^k{;)G%7RtE8t zMq%F{J^y(2+9n46oeV0w8MHSs2z)$qbq9m;E(Xm#47xiRl;0gZ^X|}@r*qbBV~}|| zXWhF)XFr|2R(Q#|8EBxpr;B5V#O36K1zZU!Noi?`uA<4w&mTN_^5Bu1;FE_>nVI71 zQVbc>>)QT6}; literal 0 HcmV?d00001 From 746cc0a9586f5ba2e3102c8c26987f6b4b221d90 Mon Sep 17 00:00:00 2001 From: Chris Brown Date: Mon, 11 Dec 2023 16:02:34 -0500 Subject: [PATCH 56/90] set docker-compose platform --- aws-sam/docker-compose.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/aws-sam/docker-compose.yaml b/aws-sam/docker-compose.yaml index 1cce8ff..3641514 100644 --- a/aws-sam/docker-compose.yaml +++ b/aws-sam/docker-compose.yaml @@ -1,7 +1,8 @@ version: '3' services: build-environment: - image: golang:1.13 + image: golang:1.20.0 + platform: linux/amd64 volumes: - ../:/go/src/github.com/ccbrown/gggtracker working_dir: /go/src/github.com/ccbrown/gggtracker/aws-sam @@ -10,6 +11,7 @@ services: environment: LD_LIBRARY_PATH: '' image: lambci/lambda:go1.x + platform: linux/amd64 volumes: - ../:/go/src/github.com/ccbrown/gggtracker working_dir: /go/src/github.com/ccbrown/gggtracker/aws-sam From cb05ff0e8fcaf0c4c51e0d98e641ff2b6e9685ed Mon Sep 17 00:00:00 2001 From: Chris Brown Date: Mon, 6 May 2024 12:32:55 -0400 Subject: [PATCH 57/90] update ga script --- server/index_handler.go | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/server/index_handler.go b/server/index_handler.go index b08c1f6..b2fa494 100644 --- a/server/index_handler.go +++ b/server/index_handler.go @@ -17,16 +17,13 @@ var index = ` {{if .Configuration.GoogleAnalytics}} - + {{end}} From 754e86315f66abccfc244ba3b7ff5a4109cbc9ae Mon Sep 17 00:00:00 2001 From: Chris Brown Date: Fri, 9 Aug 2024 23:12:49 -0400 Subject: [PATCH 58/90] support reddit again, add error handling for forum indexer --- main.go | 16 +- server/forum_indexer.go | 4 + server/forum_indexer_test.go | 16 + server/reddit_indexer.go | 10 +- server/testdata/forum-posts-logged-out.html | 508 ++++++++++++++++++++ 5 files changed, 546 insertions(+), 8 deletions(-) create mode 100644 server/testdata/forum-posts-logged-out.html diff --git a/main.go b/main.go index 1df23f8..ad5ece1 100644 --- a/main.go +++ b/main.go @@ -20,6 +20,7 @@ func main() { pflag.String("db", "./gggtracker.db", "the database file path") pflag.String("dynamodb-table", "", "if given, DynamoDB will be used instead of a database file") pflag.String("forumsession", "", "the POESESSID cookie for a forum session") + pflag.String("reddit-auth", "", "the APPLICATION:SECRET to use as Reddit auth") viper.BindPFlags(pflag.CommandLine) pflag.Parse() @@ -43,13 +44,16 @@ func main() { } defer db.Close() - redditIndexer, err := server.NewRedditIndexer(server.RedditIndexerConfiguration{ - Database: db, - }) - if err != nil { - log.Fatal(err) + if viper.GetString("reddit-auth") != "" { + redditIndexer, err := server.NewRedditIndexer(server.RedditIndexerConfiguration{ + Database: db, + Auth: viper.GetString("reddit-auth"), + }) + if err != nil { + log.Fatal(err) + } + defer redditIndexer.Close() } - defer redditIndexer.Close() if viper.GetString("forumsession") != "" { forumIndexer, err := server.NewForumIndexer(server.ForumIndexerConfiguration{ diff --git a/server/forum_indexer.go b/server/forum_indexer.go index ca84cf6..332458b 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -147,6 +147,10 @@ func ScrapeForumPosts(doc *goquery.Document, timezone *time.Location) ([]*ForumP err := error(nil) + if doc.Find(".forumPostListTable").Length() == 0 { + return nil, errors.New("forum post list not found") + } + doc.Find(".forumPostListTable > tbody > tr").EachWithBreak(func(i int, sel *goquery.Selection) bool { post := &ForumPost{ Poster: sel.Find(".post_by_account").Text(), diff --git a/server/forum_indexer_test.go b/server/forum_indexer_test.go index d2b8700..227af4b 100644 --- a/server/forum_indexer_test.go +++ b/server/forum_indexer_test.go @@ -35,6 +35,22 @@ func TestScrapeForumPosts(t *testing.T) { assert.Equal(t, "Announcements", p.ForumName) assert.Equal(t, "we had a great time too!", p.BodyHTML) assert.Equal(t, int64(1486332365), p.Time.Unix()) + + t.Run("LoggedOut", func(t *testing.T) { + f, err := os.Open("testdata/forum-posts-logged-out.html") + require.NoError(t, err) + defer f.Close() + + doc, err := goquery.NewDocumentFromReader(f) + require.NoError(t, err) + + tz, err := time.LoadLocation("America/Los_Angeles") + require.NoError(t, err) + + posts, err := ScrapeForumPosts(doc, tz) + require.Error(t, err) + assert.Equal(t, 0, len(posts)) + }) } func TestScrapeForumTimezone(t *testing.T) { diff --git a/server/reddit_indexer.go b/server/reddit_indexer.go index b14f7b7..e4519b1 100644 --- a/server/reddit_indexer.go +++ b/server/reddit_indexer.go @@ -13,6 +13,7 @@ import ( type RedditIndexerConfiguration struct { Database Database + Auth string } type RedditIndexer struct { @@ -33,6 +34,8 @@ func (indexer *RedditIndexer) Close() { indexer.closeSignal <- true } +const redditRequestInterval = time.Second * 8 + func (indexer *RedditIndexer) run() { log.Info("starting reddit indexer") @@ -58,7 +61,7 @@ func (indexer *RedditIndexer) run() { if next >= len(users) { next = 0 } - time.Sleep(time.Second * 3) + time.Sleep(redditRequestInterval) } } } @@ -136,6 +139,9 @@ func (indexer *RedditIndexer) redditActivity(user string, page string) ([]Activi return nil, "", err } req.Header.Add("User-Agent", "GGG Tracker (https://github.com/ccbrown/gggtracker) by /u/rz2yoj") + if parts := strings.Split(indexer.configuration.Auth, ":"); len(parts) == 2 { + req.SetBasicAuth(parts[0], parts[1]) + } resp, err := client.Do(req) if err != nil { @@ -184,7 +190,7 @@ func (indexer *RedditIndexer) index(user string) error { if done { break } - time.Sleep(time.Second * 3) + time.Sleep(redditRequestInterval) } if len(activity) == 0 { diff --git a/server/testdata/forum-posts-logged-out.html b/server/testdata/forum-posts-logged-out.html new file mode 100644 index 0000000..c530980 --- /dev/null +++ b/server/testdata/forum-posts-logged-out.html @@ -0,0 +1,508 @@ + + + + Path of Exile + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+ +
+ + + English + + + + + Português Brasileiro + + + + + Русский + + + + + ไทย + + + + + Deutsch + + + + + Français + + + + + Español + + + + + + + + 한국어 + + +
+ + Path of Exile + + +
+ +
+
+
+
+
+

Login Required

+
+
+

You must log in to visit this area.

+
+ +
+
+
+ +
+
+
+
+
+
+
+
+
+

+ Home +

+ +
+
+

+ Game +

+ +
+
+

+ Forum +

+ +
+
+
+

+ Events +

+ +
+
+

+ Trade +

+ +
+
+

+ Shop +

+ +
+
+
+ + + + © 2010 - 2024 + Grinding Gear Games + + + + Terms of Use, Privacy Notice and Cookies Notice + + + - + + Contact Support + + + - + + Developer API + + + 5e876bd1d0b1304cf3ea631a1f5c6916 +
+
+
+
+
+
+
+ + + + + From 7575dcef9e21fb9b420ff8ad56ea7ba604c92987 Mon Sep 17 00:00:00 2001 From: Chris Brown Date: Fri, 9 Aug 2024 23:48:24 -0400 Subject: [PATCH 59/90] remove outdated forum posters --- aws-sam/.gitignore | 1 + server/forum_indexer.go | 15 +++++++-------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/aws-sam/.gitignore b/aws-sam/.gitignore index f6f05fb..c39f315 100644 --- a/aws-sam/.gitignore +++ b/aws-sam/.gitignore @@ -1,2 +1,3 @@ /build /dist.zip +*-packaged.cfn.yaml diff --git a/server/forum_indexer.go b/server/forum_indexer.go index 332458b..60508cc 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -48,15 +48,14 @@ func (indexer *ForumIndexer) run() { "Chris", "Jonathan", "Erik", "Mark_GGG", "Samantha", "Rory", "Rhys", "Andrew_GGG", "Damien_GGG", "Joel_GGG", "Ari", "Thomas", "BrianWeissman", "Edwin_GGG", "Support", "Dylan", "MaxS", "Ammon_GGG", "Jess_GGG", "Robbie_GGG", "GGG_Neon", "Jason_GGG", "Henry_GGG", - "Michael_GGG", "Bex_GGG", "Cagan_GGG", "Daniel_GGG", "Kieren_GGG", "Yeran_GGG", "Gary_GGG", - "Dan_GGG", "Jared_GGG", "Brian_GGG", "RobbieL_GGG", "Arthur_GGG", "NickK_GGG", "Felipe_GGG", + "Michael_GGG", "Bex_GGG", "Cagan_GGG", "Kieren_GGG", "Yeran_GGG", "Gary_GGG", "Dan_GGG", + "Jared_GGG", "Brian_GGG", "RobbieL_GGG", "Arthur_GGG", "NickK_GGG", "Felipe_GGG", "Alex_GGG", "Alexcc_GGG", "Andy", "CJ_GGG", "Eben_GGG", "Emma_GGG", "Ethan_GGG", - "Fitzy_GGG", "Hartlin_GGG", "Jake_GGG", "Lionel_GGG", "Melissa_GGG", "MikeP_GGG", "Novynn", - "Rachel_GGG", "Rob_GGG", "Roman_GGG", "Sarah_GGG", "SarahB_GGG", "Tom_GGG", "Natalia_GGG", - "Jeff_GGG", "Lu_GGG", "JuliaS_GGG", "Alexander_GGG", "SamC_GGG", "AndrewE_GGG", "Kyle_GGG", - "Stacey_GGG", "Jatin_GGG", "Yolandi_GGG", "Community_Team", "Dominic_GGG", "Nick_GGG", - "Guy_GGG", "Ben_GGG", "BenH_GGG", "Nav_GGG", "Will_GGG", "Scott_GGG", "JC_GGG", "Dylan_GGG", - "Chulainn_GGG", + "Fitzy_GGG", "Hartlin_GGG", "Jake_GGG", "Melissa_GGG", "MikeP_GGG", "Novynn", "Rob_GGG", + "Roman_GGG", "Tom_GGG", "Natalia_GGG", "Jeff_GGG", "Lu_GGG", "JuliaS_GGG", "Alexander_GGG", + "SamC_GGG", "AndrewE_GGG", "Kyle_GGG", "Stacey_GGG", "Jatin_GGG", "Community_Team", + "Nick_GGG", "Guy_GGG", "Ben_GGG", "BenH_GGG", "Nav_GGG", "Will_GGG", "Scott_GGG", "JC_GGG", + "Dylan_GGG", "Chulainn_GGG", } timezone := (*time.Location)(nil) From d74410c54a49f93bebdf8f5eb6fcaefcaa4b228a Mon Sep 17 00:00:00 2001 From: Chris Brown <1731074+ccbrown@users.noreply.github.com> Date: Sun, 3 Nov 2024 23:55:10 -0500 Subject: [PATCH 60/90] add additional forum posters --- server/forum_indexer.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index 60508cc..778836c 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -55,7 +55,9 @@ func (indexer *ForumIndexer) run() { "Roman_GGG", "Tom_GGG", "Natalia_GGG", "Jeff_GGG", "Lu_GGG", "JuliaS_GGG", "Alexander_GGG", "SamC_GGG", "AndrewE_GGG", "Kyle_GGG", "Stacey_GGG", "Jatin_GGG", "Community_Team", "Nick_GGG", "Guy_GGG", "Ben_GGG", "BenH_GGG", "Nav_GGG", "Will_GGG", "Scott_GGG", "JC_GGG", - "Dylan_GGG", "Chulainn_GGG", + "Dylan_GGG", "Chulainn_GGG", "Vash_GGG", "Cameron_GGG", "Jacob_GGG", "Jenn_GGG", + "CoryA_GGG", "Sian_GGG", "Drew_GGG", "Lisa_GGG", "ThomasK_GGG", "Whai_GGG", "Scopey", + "Adam_GGG", } timezone := (*time.Location)(nil) From 02727e9a5aa78955c71fa7373727ed1047c3b14c Mon Sep 17 00:00:00 2001 From: Chris Brown <1731074+ccbrown@users.noreply.github.com> Date: Tue, 12 Nov 2024 19:09:53 -0500 Subject: [PATCH 61/90] index /r/pathofexile2 --- server/reddit_indexer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/reddit_indexer.go b/server/reddit_indexer.go index e4519b1..a5b060f 100644 --- a/server/reddit_indexer.go +++ b/server/reddit_indexer.go @@ -97,7 +97,7 @@ func ParseRedditActivity(b []byte) ([]Activity, string, error) { } for _, thing := range root.Data.Children { - if thing.Data.SubredditId != "t5_2sf6m" && thing.Data.SubredditId != "t5_2w3q8" { + if thing.Data.SubredditId != "t5_2sf6m" && thing.Data.SubredditId != "t5_2w3q8" && thing.Data.SubredditId != "t5_3910n" { continue } switch thing.Kind { From 10e735156a3dd91176a57d3aa03337c9e84a5d62 Mon Sep 17 00:00:00 2001 From: Chris <1731074+ccbrown@users.noreply.github.com> Date: Wed, 13 Nov 2024 08:50:49 -0500 Subject: [PATCH 62/90] Create FUNDING.yml --- .github/FUNDING.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..852d5ee --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,3 @@ +# These are supported funding model platforms + +github: [ccbrown] From 6ca18053d373202c1b35601c1180f22d10ee97ab Mon Sep 17 00:00:00 2001 From: Chris Brown <1731074+ccbrown@users.noreply.github.com> Date: Wed, 13 Nov 2024 19:58:54 -0500 Subject: [PATCH 63/90] add sponsor link --- server/index_handler.go | 7 ++ server/static/style.css | 139 ++++++++++++++++++++++------------------ 2 files changed, 85 insertions(+), 61 deletions(-) diff --git a/server/index_handler.go b/server/index_handler.go index b2fa494..414cf07 100644 --- a/server/index_handler.go +++ b/server/index_handler.go @@ -37,6 +37,10 @@ var index = ` {{end}} +
+

Tala moana, warrior! This website is approaching its 12th birthday! If you've found it useful over the years, you can now show your appreciation in the form of a recurring or one-time donation on GitHub! In addition to showing your appreciation, this offsets server costs and helps support further feature developments (Twitter/X support? Search? GenAI?).

+

Thanks, and stay safe out there, Exile. ❤️

+

{{call $.Translate "Activity"}}

@@ -62,6 +66,9 @@ var index = ` Please direct feedback to this thread. Want a new feature? Add it yourself!

+

+ Appreciate the site? Show your support by sponsoring me! ❤️ +

diff --git a/server/static/style.css b/server/static/style.css index 682f0b1..1a5cf08 100644 --- a/server/static/style.css +++ b/server/static/style.css @@ -1,6 +1,6 @@ a { color: #C9BA8A; - text-decoration: none; + text-decoration: none; } a:hover { @@ -52,6 +52,23 @@ td { text-align: right; } +div.notice { + background-color: #DFCF99; + padding: 0px 12px; + margin: 24px; + border: 1px solid #363636; + box-shadow: 3px 3px 16px 4px #000000; + width: 980px; + position: relative; + font-size: 14px; + color: #202020; +} + +div.notice a, div.notice a:visited, div.notice a:active, div.notice a:hover { + color: #202020; + text-decoration: underline; +} + div.content-box { background-color: #202020; padding: 12px; @@ -107,11 +124,11 @@ footer { } footer a { - text-decoration: underline; + text-decoration: underline; } td.poster { - text-align: center; + text-align: center; } .forum td.poster a { @@ -126,7 +143,7 @@ td.poster { } .reddit td.poster a { - color: #f33f3f; + color: #f33f3f; text-decoration: none; font-weight: bold; } @@ -137,124 +154,124 @@ td.poster { } .reddit td { - background-color: #07192c; + background-color: #07192c; } td.toggle { - padding: 0px; - width: 34px; - height: 34px; + padding: 0px; + width: 34px; + height: 34px; } td.time { - width: 200px; - text-align: center; + width: 200px; + text-align: center; } td.forum { - text-overflow: ellipsis; - overflow: hidden; - white-space: nowrap; - max-width: 140px; - text-align: center; + text-overflow: ellipsis; + overflow: hidden; + white-space: nowrap; + max-width: 140px; + text-align: center; } .reddit .forum a::before { - content: '/r/'; + content: '/r/'; } td.toggle { - position: relative; - padding: 0px; + position: relative; + padding: 0px; } td.toggle img { - cursor: pointer; - width: 18px; - height: 18px; - position: absolute; - top: 50%; - left: 50%; - transform: translate(-50%, -50%); - filter: opacity(70%); + cursor: pointer; + width: 18px; + height: 18px; + position: absolute; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); + filter: opacity(70%); } .expander:hover, .collapser:hover { - filter: opacity(100%); + filter: opacity(100%); } td.icon { - position: relative; - padding: 0px; - width: 35px; + position: relative; + padding: 0px; + width: 35px; } td.icon img { - position: absolute; - height: 24px; - top: 50%; - left: 50%; - transform: translate(-50%, -50%); + position: absolute; + height: 24px; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); } td.title { - max-width: 450px; - overflow: hidden; + max-width: 450px; + overflow: hidden; } td.body { - background-color: #262626; - padding: 10px; + background-color: #262626; + padding: 10px; } td.body blockquote { - background: #323232; - border-left: 5px solid #1a1a1a; - margin: 1.5em 10px; - padding: 0.5em 10px; + background: #323232; + border-left: 5px solid #1a1a1a; + margin: 1.5em 10px; + padding: 0.5em 10px; } td.body img { - max-width: 100%; + max-width: 100%; } .spoilerHidden .spoilerContent { - display: none; + display: none; } .spoilerTitle input[type="button"] { - margin-left: 8px; - color: #c9ba8a; - background-color: #0f0f0f; - border: 0; - cursor: pointer; - text-decoration: none; + margin-left: 8px; + color: #c9ba8a; + background-color: #0f0f0f; + border: 0; + cursor: pointer; + text-decoration: none; } .spoilerTitle input[type="button"]:hover { - text-decoration: underline; + text-decoration: underline; } .spoiler { - border: 1px solid #1a1a1a; - background-color: #323232; + border: 1px solid #1a1a1a; + background-color: #323232; } .spoilerTitle, .spoilerContent { - padding: 8px; + padding: 8px; } #locale-selection { - float: right; - list-style: none; - margin: 0px; + float: right; + list-style: none; + margin: 0px; } #locale-selection li { - display: inline; - opacity: 0.5; + display: inline; + opacity: 0.5; } #locale-selection li.selected-locale { - opacity: 1.0; + opacity: 1.0; } From 0f6cabf80dde9198e962814244922992e8638701 Mon Sep 17 00:00:00 2001 From: Chris Brown <1731074+ccbrown@users.noreply.github.com> Date: Wed, 13 Nov 2024 20:00:12 -0500 Subject: [PATCH 64/90] add additional forum accounts (thanks @DanNeely!) --- server/forum_indexer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index 778836c..ff23c2c 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -57,7 +57,7 @@ func (indexer *ForumIndexer) run() { "Nick_GGG", "Guy_GGG", "Ben_GGG", "BenH_GGG", "Nav_GGG", "Will_GGG", "Scott_GGG", "JC_GGG", "Dylan_GGG", "Chulainn_GGG", "Vash_GGG", "Cameron_GGG", "Jacob_GGG", "Jenn_GGG", "CoryA_GGG", "Sian_GGG", "Drew_GGG", "Lisa_GGG", "ThomasK_GGG", "Whai_GGG", "Scopey", - "Adam_GGG", + "Adam_GGG", "Nichelle_GGG", "Markus_GGG", } timezone := (*time.Location)(nil) From 2f43da42b39d45cb3968098db1f4a214225068d2 Mon Sep 17 00:00:00 2001 From: Chris Brown <1731074+ccbrown@users.noreply.github.com> Date: Wed, 13 Nov 2024 23:27:39 -0500 Subject: [PATCH 65/90] update cache buster --- server/index_handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/index_handler.go b/server/index_handler.go index 414cf07..8cd3483 100644 --- a/server/index_handler.go +++ b/server/index_handler.go @@ -11,7 +11,7 @@ var index = ` GGG Tracker - + From 293352bc2db819f5f3c7fc03d9654fc5456c571b Mon Sep 17 00:00:00 2001 From: Chris Brown <1731074+ccbrown@users.noreply.github.com> Date: Wed, 20 Nov 2024 14:41:31 -0500 Subject: [PATCH 66/90] support new account discriminators --- server/forum_indexer.go | 112 ++++++++++++++++++++++++++++------- server/forum_indexer_test.go | 6 +- server/forum_post.go | 55 +++++++++-------- server/index_handler.go | 2 +- server/static/index.js | 2 +- 5 files changed, 126 insertions(+), 51 deletions(-) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index ff23c2c..3a1c8eb 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -41,23 +41,92 @@ func (indexer *ForumIndexer) Close() { close(indexer.closeSignal) } +type ForumAccount struct { + Username string + Discriminator int +} + func (indexer *ForumIndexer) run() { log.Info("starting forum indexer") - accounts := []string{ - "Chris", "Jonathan", "Erik", "Mark_GGG", "Samantha", "Rory", "Rhys", "Andrew_GGG", - "Damien_GGG", "Joel_GGG", "Ari", "Thomas", "BrianWeissman", "Edwin_GGG", "Support", "Dylan", - "MaxS", "Ammon_GGG", "Jess_GGG", "Robbie_GGG", "GGG_Neon", "Jason_GGG", "Henry_GGG", - "Michael_GGG", "Bex_GGG", "Cagan_GGG", "Kieren_GGG", "Yeran_GGG", "Gary_GGG", "Dan_GGG", - "Jared_GGG", "Brian_GGG", "RobbieL_GGG", "Arthur_GGG", "NickK_GGG", "Felipe_GGG", - "Alex_GGG", "Alexcc_GGG", "Andy", "CJ_GGG", "Eben_GGG", "Emma_GGG", "Ethan_GGG", - "Fitzy_GGG", "Hartlin_GGG", "Jake_GGG", "Melissa_GGG", "MikeP_GGG", "Novynn", "Rob_GGG", - "Roman_GGG", "Tom_GGG", "Natalia_GGG", "Jeff_GGG", "Lu_GGG", "JuliaS_GGG", "Alexander_GGG", - "SamC_GGG", "AndrewE_GGG", "Kyle_GGG", "Stacey_GGG", "Jatin_GGG", "Community_Team", - "Nick_GGG", "Guy_GGG", "Ben_GGG", "BenH_GGG", "Nav_GGG", "Will_GGG", "Scott_GGG", "JC_GGG", - "Dylan_GGG", "Chulainn_GGG", "Vash_GGG", "Cameron_GGG", "Jacob_GGG", "Jenn_GGG", - "CoryA_GGG", "Sian_GGG", "Drew_GGG", "Lisa_GGG", "ThomasK_GGG", "Whai_GGG", "Scopey", - "Adam_GGG", "Nichelle_GGG", "Markus_GGG", + accounts := []ForumAccount{ + {Username: "Chris"}, + {Username: "Jonathan"}, + {Username: "Mark_GGG"}, + {Username: "Rory"}, + {Username: "Rhys"}, + {Username: "Joel_GGG"}, + {Username: "Ari"}, + {Username: "Thomas"}, + {Username: "Support"}, + {Username: "Jess_GGG"}, + {Username: "Robbie_GGG"}, + {Username: "GGG_Neon"}, + {Username: "Jason_GGG"}, + {Username: "Henry_GGG"}, + {Username: "Michael_GGG"}, + {Username: "Bex_GGG"}, + {Username: "Cagan_GGG"}, + {Username: "Kieren_GGG"}, + {Username: "Yeran_GGG"}, + {Username: "Gary_GGG"}, + {Username: "Dan_GGG"}, + {Username: "Jared_GGG"}, + {Username: "Brian_GGG"}, + {Username: "RobbieL_GGG"}, + {Username: "Arthur_GGG"}, + {Username: "NickK_GGG"}, + {Username: "Felipe_GGG"}, + {Username: "Alex_GGG"}, + {Username: "Alexcc_GGG"}, + {Username: "CJ_GGG"}, + {Username: "Eben_GGG"}, + {Username: "Emma_GGG"}, + {Username: "Ethan_GGG"}, + {Username: "Fitzy_GGG"}, + {Username: "Hartlin_GGG"}, + {Username: "Jake_GGG"}, + {Username: "Melissa_GGG"}, + {Username: "MikeP_GGG"}, + {Username: "Novynn"}, + {Username: "Rob_GGG"}, + {Username: "Roman_GGG"}, + {Username: "Tom_GGG"}, + {Username: "Natalia_GGG"}, + {Username: "Jeff_GGG"}, + {Username: "Lu_GGG"}, + {Username: "JuliaS_GGG"}, + {Username: "Alexander_GGG"}, + {Username: "SamC_GGG"}, + {Username: "AndrewE_GGG"}, + {Username: "Kyle_GGG"}, + {Username: "Stacey_GGG"}, + {Username: "Jatin_GGG"}, + {Username: "Community_Team"}, + {Username: "Nick_GGG"}, + {Username: "Guy_GGG"}, + {Username: "Ben_GGG"}, + {Username: "BenH_GGG"}, + {Username: "Nav_GGG"}, + {Username: "Will_GGG"}, + {Username: "Scott_GGG"}, + {Username: "JC_GGG"}, + {Username: "Dylan_GGG"}, + {Username: "Chulainn_GGG"}, + {Username: "Vash_GGG"}, + {Username: "Cameron_GGG"}, + {Username: "Jacob_GGG"}, + {Username: "Jenn_GGG"}, + {Username: "CoryA_GGG"}, + {Username: "Sian_GGG"}, + {Username: "Drew_GGG"}, + {Username: "Lisa_GGG"}, + {Username: "ThomasK_GGG"}, + {Username: "Whai_GGG"}, + {Username: "Scopey"}, + {Username: "Adam_GGG"}, + {Username: "Nichelle_GGG"}, + {Username: "Markus_GGG"}, } timezone := (*time.Location)(nil) @@ -99,7 +168,7 @@ func (indexer *ForumIndexer) run() { return default: if err := indexer.index(account, timezone); err != nil { - log.WithError(err).Error("error indexing forum account: " + account) + log.WithError(err).Error("error indexing forum account: " + account.Username) } time.Sleep(time.Second) } @@ -143,7 +212,7 @@ var postURLExpression = regexp.MustCompile("^/forum/view-post/([0-9]+)") var threadURLExpression = regexp.MustCompile("^/forum/view-thread/([0-9]+)") var forumURLExpression = regexp.MustCompile("^/forum/view-forum/([0-9]+)") -func ScrapeForumPosts(doc *goquery.Document, timezone *time.Location) ([]*ForumPost, error) { +func ScrapeForumPosts(doc *goquery.Document, poster ForumAccount, timezone *time.Location) ([]*ForumPost, error) { posts := []*ForumPost(nil) err := error(nil) @@ -154,7 +223,8 @@ func ScrapeForumPosts(doc *goquery.Document, timezone *time.Location) ([]*ForumP doc.Find(".forumPostListTable > tbody > tr").EachWithBreak(func(i int, sel *goquery.Selection) bool { post := &ForumPost{ - Poster: sel.Find(".post_by_account").Text(), + Poster: poster.Username, + PosterDiscriminator: poster.Discriminator, } body, err := sel.Find(".content").Html() @@ -197,19 +267,19 @@ func ScrapeForumPosts(doc *goquery.Document, timezone *time.Location) ([]*ForumP return posts, nil } -func (indexer *ForumIndexer) forumPosts(poster string, page int, timezone *time.Location) ([]*ForumPost, error) { - doc, err := indexer.requestDocument(fmt.Sprintf("/account/view-posts/%v/page/%v", poster, page)) +func (indexer *ForumIndexer) forumPosts(poster ForumAccount, page int, timezone *time.Location) ([]*ForumPost, error) { + doc, err := indexer.requestDocument(fmt.Sprintf("/account/view-posts/%v-%04d/page/%v", poster.Username, poster.Discriminator, page)) if err != nil { return nil, err } - posts, err := ScrapeForumPosts(doc, timezone) + posts, err := ScrapeForumPosts(doc, poster, timezone) if err != nil { return nil, err } return posts, nil } -func (indexer *ForumIndexer) index(poster string, timezone *time.Location) error { +func (indexer *ForumIndexer) index(poster ForumAccount, timezone *time.Location) error { logger := log.WithFields(log.Fields{ "poster": poster, }) diff --git a/server/forum_indexer_test.go b/server/forum_indexer_test.go index 227af4b..4d48f9e 100644 --- a/server/forum_indexer_test.go +++ b/server/forum_indexer_test.go @@ -21,7 +21,8 @@ func TestScrapeForumPosts(t *testing.T) { tz, err := time.LoadLocation("America/Los_Angeles") require.NoError(t, err) - posts, err := ScrapeForumPosts(doc, tz) + poster := ForumAccount{Username: "Chris"} + posts, err := ScrapeForumPosts(doc, poster, tz) require.NoError(t, err) assert.Equal(t, 10, len(posts)) @@ -31,6 +32,7 @@ func TestScrapeForumPosts(t *testing.T) { assert.Equal(t, 54, p.ForumId) assert.Equal(t, 1830139, p.ThreadId) assert.Equal(t, "Chris", p.Poster) + assert.Equal(t, 0, p.PosterDiscriminator) assert.Equal(t, "Photos of the Fan Meetup", p.ThreadTitle) assert.Equal(t, "Announcements", p.ForumName) assert.Equal(t, "we had a great time too!", p.BodyHTML) @@ -47,7 +49,7 @@ func TestScrapeForumPosts(t *testing.T) { tz, err := time.LoadLocation("America/Los_Angeles") require.NoError(t, err) - posts, err := ScrapeForumPosts(doc, tz) + posts, err := ScrapeForumPosts(doc, poster, tz) require.Error(t, err) assert.Equal(t, 0, len(posts)) }) diff --git a/server/forum_post.go b/server/forum_post.go index 1cd6e41..bd9971f 100644 --- a/server/forum_post.go +++ b/server/forum_post.go @@ -7,39 +7,42 @@ import ( ) type ForumPost struct { - Id int `json:"id"` - BodyHTML string `json:"body_html"` - Time time.Time `json:"time"` - Poster string `json:"poster"` - ThreadId int `json:"thread_id"` - ThreadTitle string `json:"thread_title"` - ForumId int `json:"forum_id"` - ForumName string `json:"forum_name"` + Id int `json:"id"` + BodyHTML string `json:"body_html"` + Time time.Time `json:"time"` + Poster string `json:"poster"` + PosterDiscriminator int `json:"poster_discriminator"` + ThreadId int `json:"thread_id"` + ThreadTitle string `json:"thread_title"` + ForumId int `json:"forum_id"` + ForumName string `json:"forum_name"` } type forumPostWithHost struct { - Id int `json:"id"` - BodyHTML string `json:"body_html"` - Time time.Time `json:"time"` - Poster string `json:"poster"` - ThreadId int `json:"thread_id"` - ThreadTitle string `json:"thread_title"` - ForumId int `json:"forum_id"` - ForumName string `json:"forum_name"` - Host string `json:"host"` + Id int `json:"id"` + BodyHTML string `json:"body_html"` + Time time.Time `json:"time"` + Poster string `json:"poster"` + PosterDiscriminator int `json:"poster_discriminator"` + ThreadId int `json:"thread_id"` + ThreadTitle string `json:"thread_title"` + ForumId int `json:"forum_id"` + ForumName string `json:"forum_name"` + Host string `json:"host"` } func (p ForumPost) MarshalJSON() ([]byte, error) { return json.Marshal(&forumPostWithHost{ - Id: p.Id, - BodyHTML: p.BodyHTML, - Time: p.Time, - Poster: p.Poster, - ThreadId: p.ThreadId, - ThreadTitle: p.ThreadTitle, - ForumId: p.ForumId, - ForumName: p.ForumName, - Host: p.Host(), + Id: p.Id, + BodyHTML: p.BodyHTML, + Time: p.Time, + Poster: p.Poster, + PosterDiscriminator: p.PosterDiscriminator, + ThreadId: p.ThreadId, + ThreadTitle: p.ThreadTitle, + ForumId: p.ForumId, + ForumName: p.ForumName, + Host: p.Host(), }) } diff --git a/server/index_handler.go b/server/index_handler.go index 8cd3483..f5ac837 100644 --- a/server/index_handler.go +++ b/server/index_handler.go @@ -72,7 +72,7 @@ var index = ` - + ` diff --git a/server/static/index.js b/server/static/index.js index 4b65dbe..9c8aed3 100644 --- a/server/static/index.js +++ b/server/static/index.js @@ -72,7 +72,7 @@ function loadActivity() { if (type == 'forum_post') { $tr.append($('		').append($('') - .attr('href', 'https://' + activity.host + '/account/view-profile/' + encodeURIComponent(activity.poster)) + .attr('href', 'https://' + activity.host + '/account/view-profile/' + encodeURIComponent(activity.poster) + '-' + String(activity.poster_discriminator || 0).padStart(4, '0')) .text(activity.poster) )); } else { From c58459354670d572e01332f3714bb1906777aaef Mon Sep 17 00:00:00 2001 From: Chris <1731074+ccbrown@users.noreply.github.com> Date: Wed, 20 Nov 2024 15:07:16 -0500 Subject: [PATCH 67/90] fix readme badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 94665e0..bfd6a82 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Welcome! [![Build Status](https://travis-ci.org/ccbrown/gggtracker.svg?branch=master)](https://travis-ci.org/ccbrown/gggtracker) +# Welcome! ![Build Status](https://github.com/ccbrown/gggtracker/actions/workflows/push.yml/badge.svg) This is the repository for [gggtracker.com](https://gggtracker.com). If there's something you think the site is missing, please either a.) open an issue to request the feature or b.) develop the feature yourself and put in a pull request. From bde81e80452719946bbb3d725dba0314f308b371 Mon Sep 17 00:00:00 2001 From: Chris Brown <1731074+ccbrown@users.noreply.github.com> Date: Mon, 25 Nov 2024 15:23:30 -0500 Subject: [PATCH 68/90] add Jarod_GGG --- server/forum_indexer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index 3a1c8eb..4723e15 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -127,6 +127,7 @@ func (indexer *ForumIndexer) run() { {Username: "Adam_GGG"}, {Username: "Nichelle_GGG"}, {Username: "Markus_GGG"}, + {Username: "Jarod_GGG"}, } timezone := (*time.Location)(nil) From bcc3536ce2e94f474738e81fe9128565821da924 Mon Sep 17 00:00:00 2001 From: Chris Brown <1731074+ccbrown@users.noreply.github.com> Date: Wed, 11 Dec 2024 22:05:27 -0500 Subject: [PATCH 69/90] improve handling of errors due to maintenance --- server/forum_indexer.go | 17 ++++-- server/forum_indexer_test.go | 16 ++++++ server/testdata/forum-maintenance.html | 74 ++++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 3 deletions(-) create mode 100644 server/testdata/forum-maintenance.html diff --git a/server/forum_indexer.go b/server/forum_indexer.go index 4723e15..ab422dc 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -169,7 +169,12 @@ func (indexer *ForumIndexer) run() { return default: if err := indexer.index(account, timezone); err != nil { - log.WithError(err).Error("error indexing forum account: " + account.Username) + if errors.Is(err, ErrForumMaintenance) { + log.Info("forum is under maintenance") + time.Sleep(30 * time.Second) + } else { + log.WithError(err).Error("error indexing forum account: " + account.Username) + } } time.Sleep(time.Second) } @@ -213,13 +218,19 @@ var postURLExpression = regexp.MustCompile("^/forum/view-post/([0-9]+)") var threadURLExpression = regexp.MustCompile("^/forum/view-thread/([0-9]+)") var forumURLExpression = regexp.MustCompile("^/forum/view-forum/([0-9]+)") +var ErrForumMaintenance = errors.New("forum is in maintenance") + func ScrapeForumPosts(doc *goquery.Document, poster ForumAccount, timezone *time.Location) ([]*ForumPost, error) { posts := []*ForumPost(nil) err := error(nil) if doc.Find(".forumPostListTable").Length() == 0 { - return nil, errors.New("forum post list not found") + err = errors.New("forum post list not found") + if topBar := doc.Find(".topBar"); topBar.Length() == 1 && topBar.Text() == "Down For Maintenance" { + err = ErrForumMaintenance + } + return nil, err } doc.Find(".forumPostListTable > tbody > tr").EachWithBreak(func(i int, sel *goquery.Selection) bool { @@ -292,7 +303,7 @@ func (indexer *ForumIndexer) index(poster ForumAccount, timezone *time.Location) for page := 1; ; page++ { posts, err := indexer.forumPosts(poster, page, timezone) if err != nil { - logger.WithError(err).Error("error requesting forum posts") + return fmt.Errorf("error getting forum posts: %w", err) } done := len(posts) == 0 diff --git a/server/forum_indexer_test.go b/server/forum_indexer_test.go index 4d48f9e..4e6b756 100644 --- a/server/forum_indexer_test.go +++ b/server/forum_indexer_test.go @@ -53,6 +53,22 @@ func TestScrapeForumPosts(t *testing.T) { require.Error(t, err) assert.Equal(t, 0, len(posts)) }) + + t.Run("Maintenance", func(t *testing.T) { + f, err := os.Open("testdata/forum-maintenance.html") + require.NoError(t, err) + defer f.Close() + + doc, err := goquery.NewDocumentFromReader(f) + require.NoError(t, err) + + tz, err := time.LoadLocation("America/Los_Angeles") + require.NoError(t, err) + + posts, err := ScrapeForumPosts(doc, poster, tz) + assert.Equal(t, err, ErrForumMaintenance) + assert.Equal(t, 0, len(posts)) + }) } func TestScrapeForumTimezone(t *testing.T) { diff --git a/server/testdata/forum-maintenance.html b/server/testdata/forum-maintenance.html new file mode 100644 index 0000000..6e1f145 --- /dev/null +++ b/server/testdata/forum-maintenance.html @@ -0,0 +1,74 @@ + + + + Path of Exile + + + + + + + + + + + + + + + +
+
+
+
+ + + + + + + + + +
+ + Path of Exile + +
+
+
+
+
+
+

Down For Maintenance

+
+

+ pathofexile.com is currently down for maintenance. Please try again later.

+Thanks for your patience! +

+
+
+
+
+ + +
+
+
+
+
+
+
+
+
+
+ © 2010 - 2019 Grinding Gear Games + Contact Support
+ +
+
+
+
+
+ + From d1e1115ce7ec59edccc2d7a987f07b3a89bf8bbe Mon Sep 17 00:00:00 2001 From: Chris Brown <1731074+ccbrown@users.noreply.github.com> Date: Fri, 3 Jan 2025 19:27:05 -0600 Subject: [PATCH 70/90] add Joel_GGG#1496 --- server/forum_indexer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index ab422dc..fb034e9 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -128,6 +128,7 @@ func (indexer *ForumIndexer) run() { {Username: "Nichelle_GGG"}, {Username: "Markus_GGG"}, {Username: "Jarod_GGG"}, + {Username: "Joel_GGG", Discriminator: 1496}, } timezone := (*time.Location)(nil) From fcfbc5c377191736ddd8c9a1ef80bf677433b686 Mon Sep 17 00:00:00 2001 From: Chris Brown <1731074+ccbrown@users.noreply.github.com> Date: Tue, 7 Jan 2025 01:47:02 -0500 Subject: [PATCH 71/90] add vinky and edmund --- server/forum_indexer.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index fb034e9..6d124db 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -129,6 +129,8 @@ func (indexer *ForumIndexer) run() { {Username: "Markus_GGG"}, {Username: "Jarod_GGG"}, {Username: "Joel_GGG", Discriminator: 1496}, + {Username: "Vinky_GGG"}, + {Username: "Edmund_GGG", Discriminator: 4844}, } timezone := (*time.Location)(nil) From c503345c0a89ab698737a1a7398768acd2903ae1 Mon Sep 17 00:00:00 2001 From: Abraham Egnor Date: Tue, 7 Jan 2025 01:51:06 -0500 Subject: [PATCH 72/90] Add optional filtering of help posts (#140) * add filtering of help posts * make fetchActivity more readable * fold filtering into db layer * move toggle to top of table * show loading indicator for filtered mode * fix style and autoformat --- server/activity_handler.go | 16 +++++++++++++++- server/bolt_database.go | 4 ++-- server/database.go | 2 +- server/database_test.go | 8 +++++--- server/dynamodb_database.go | 4 ++-- server/index_handler.go | 1 + server/localization.go | 11 +++++++++-- server/rss_handler.go | 2 +- server/static/index.js | 26 +++++++++++++++++++++++--- server/static/style.css | 6 ++++++ 10 files changed, 65 insertions(+), 15 deletions(-) diff --git a/server/activity_handler.go b/server/activity_handler.go index 7f883f1..fc45567 100644 --- a/server/activity_handler.go +++ b/server/activity_handler.go @@ -16,7 +16,21 @@ type jsonResponse struct { func ActivityHandler(db Database) echo.HandlerFunc { return func(c echo.Context) error { - activity, next, err := db.Activity(LocaleForRequest(c.Request()), c.QueryParam("next"), 50) + locale := LocaleForRequest(c.Request()) + filter := func(a Activity) bool { + return true + } + if c.QueryParam("nohelp") == "true" { + filter = func(a Activity) bool { + if fp, ok := a.(*ForumPost); ok { + if fp.ForumId == locale.HelpForumId { + return false + } + } + return true + } + } + activity, next, err := db.Activity(locale, c.QueryParam("next"), 50, filter) if err != nil { return err } diff --git a/server/bolt_database.go b/server/bolt_database.go index 5a83aec..5b23285 100644 --- a/server/bolt_database.go +++ b/server/bolt_database.go @@ -43,7 +43,7 @@ func (db *BoltDatabase) AddActivity(activity []Activity) error { }) } -func (db *BoltDatabase) Activity(locale *Locale, start string, count int) ([]Activity, string, error) { +func (db *BoltDatabase) Activity(locale *Locale, start string, count int, filter func(a Activity) bool) ([]Activity, string, error) { ret := []Activity(nil) next := "" if err := db.db.View(func(tx *bolt.Tx) error { @@ -66,7 +66,7 @@ func (db *BoltDatabase) Activity(locale *Locale, start string, count int) ([]Act activity, err := unmarshalActivity(k, v) if err != nil { return err - } else if activity != nil && locale.ActivityFilter(activity) { + } else if activity != nil && locale.ActivityFilter(activity) && filter(activity) { ret = append(ret, activity) next = base64.RawURLEncoding.EncodeToString(k) } diff --git a/server/database.go b/server/database.go index e144320..8118b45 100644 --- a/server/database.go +++ b/server/database.go @@ -17,7 +17,7 @@ const ( type Database interface { AddActivity(activity []Activity) error - Activity(locale *Locale, start string, count int) ([]Activity, string, error) + Activity(locale *Locale, start string, count int, filter func(a Activity) bool) ([]Activity, string, error) Close() error } diff --git a/server/database_test.go b/server/database_test.go index 6654c54..4193c8b 100644 --- a/server/database_test.go +++ b/server/database_test.go @@ -31,21 +31,23 @@ func testDatabase_ForumPosts(t *testing.T, db Database) { db.AddActivity([]Activity{post1, post2}) - posts, next, err := db.Activity(locale, "", 1) + all := func(a Activity) bool { return true } + + posts, next, err := db.Activity(locale, "", 1, all) require.NoError(t, err) require.Equal(t, 1, len(posts)) assert.Equal(t, post1.Id, posts[0].(*ForumPost).Id) assert.Equal(t, post1.Poster, posts[0].(*ForumPost).Poster) assert.Equal(t, post1.Time.Unix(), posts[0].(*ForumPost).Time.Unix()) - posts, next, err = db.Activity(locale, next, 1) + posts, next, err = db.Activity(locale, next, 1, all) require.NoError(t, err) require.Equal(t, 1, len(posts)) assert.Equal(t, post2.Id, posts[0].(*ForumPost).Id) assert.Equal(t, post2.Poster, posts[0].(*ForumPost).Poster) assert.Equal(t, post2.Time.Unix(), posts[0].(*ForumPost).Time.Unix()) - posts, _, err = db.Activity(locale, next, 1) + posts, _, err = db.Activity(locale, next, 1, all) require.NoError(t, err) require.Equal(t, 0, len(posts)) } diff --git a/server/dynamodb_database.go b/server/dynamodb_database.go index b339081..d498949 100644 --- a/server/dynamodb_database.go +++ b/server/dynamodb_database.go @@ -82,7 +82,7 @@ func (db *DynamoDBDatabase) AddActivity(activity []Activity) error { return nil } -func (db *DynamoDBDatabase) Activity(locale *Locale, start string, count int) ([]Activity, string, error) { +func (db *DynamoDBDatabase) Activity(locale *Locale, start string, count int, filter func(a Activity) bool) ([]Activity, string, error) { var activity []Activity var startKey map[string]dynamodb.AttributeValue @@ -120,7 +120,7 @@ func (db *DynamoDBDatabase) Activity(locale *Locale, start string, count int) ([ for _, item := range result.Items { if a, err := unmarshalActivity(item["rk"].B, item["v"].B); err != nil { return nil, "", err - } else if a != nil { + } else if a != nil && filter(a) { activity = append(activity, a) } } diff --git a/server/index_handler.go b/server/index_handler.go index f5ac837..927177e 100644 --- a/server/index_handler.go +++ b/server/index_handler.go @@ -43,6 +43,7 @@ var index = `

{{call $.Translate "Activity"}}

+
diff --git a/server/localization.go b/server/localization.go index f09210e..860211d 100644 --- a/server/localization.go +++ b/server/localization.go @@ -17,6 +17,7 @@ type Locale struct { IncludeReddit bool Translations map[string]string ParseTime func(s string, tz *time.Location) (time.Time, error) + HelpForumId int forumIds atomic.Value } @@ -95,6 +96,7 @@ var Locales = []*Locale{ { IncludeReddit: true, Image: "static/images/locales/gb.png", + HelpForumId: 584, }, { Subdomain: "br", @@ -106,6 +108,7 @@ var Locales = []*Locale{ "Time": "Hora", "Forum": "Fórum", }, + HelpForumId: 774, }, { Subdomain: "ru", @@ -119,8 +122,9 @@ var Locales = []*Locale{ }, }, { - Subdomain: "th", - Image: "static/images/locales/th.png", + Subdomain: "th", + Image: "static/images/locales/th.png", + HelpForumId: 1011, }, { Subdomain: "de", @@ -132,6 +136,7 @@ var Locales = []*Locale{ "Time": "Datum", "Forum": "Forum", }, + HelpForumId: 1123, }, { Subdomain: "fr", @@ -143,6 +148,7 @@ var Locales = []*Locale{ "Time": "Date", "Forum": "Forum", }, + HelpForumId: 1051, }, { Subdomain: "es", @@ -154,6 +160,7 @@ var Locales = []*Locale{ "Time": "Fecha", "Forum": "Foro", }, + HelpForumId: 1193, }, { Subdomain: "jp", diff --git a/server/rss_handler.go b/server/rss_handler.go index 8982272..cf520bc 100644 --- a/server/rss_handler.go +++ b/server/rss_handler.go @@ -43,7 +43,7 @@ type rssResponse struct { func RSSHandler(db Database) echo.HandlerFunc { return func(c echo.Context) error { - activity, _, err := db.Activity(LocaleForRequest(c.Request()), c.QueryParam("next"), 50) + activity, _, err := db.Activity(LocaleForRequest(c.Request()), c.QueryParam("next"), 50, func(a Activity) bool { return true }) if err != nil { return err } diff --git a/server/static/index.js b/server/static/index.js index 9c8aed3..f402423 100644 --- a/server/static/index.js +++ b/server/static/index.js @@ -16,14 +16,20 @@ var POE = { }; function loadActivity() { - var page = location.hash.replace(/^#page=/, ''); + var params = new URLSearchParams(location.hash.replace(/^#/, '')); + var page = params.get('page') || ''; + var nohelp = params.get('nohelp') || ''; if (currentPage !== undefined && page == currentPage) { return; } var previousPage = currentPage; currentPage = page; - $.get('activity.json?next=' + page, function(data) { + if (nohelp == 'true') { + $('#activity-table tbody').empty().append($('').append($('
').attr('colspan', 6).text('Loading...'))) + } + + $.get('activity.json?next=' + page + '&nohelp=' + nohelp, function(data) { var $tbody = $('#activity-table tbody'); $tbody.empty(); @@ -135,7 +141,21 @@ function loadActivity() { $tbody.append($tr); } - $('#activity-nav').empty().append($('').text('Next Page').attr('href', '#page=' + data.next).click(function() { + var nohelpText; + var nohelpHref; + if (nohelp != 'true') { + nohelpText = 'Hide Help Forum'; + nohelpHref = '#page=' + page + '&nohelp=true'; + } else { + nohelpText = 'Show Help Forum'; + nohelpHref = '#page=' + page + '&nohelp=false'; + } + $('#help-toggle').empty().append($('').text(nohelpText).attr('href', nohelpHref).click(function() { + currentPage = undefined; + window.scrollTo(0, 0); + })); + + $('#activity-nav').empty().append($('').text('Next Page').attr('href', '#page=' + data.next + '&nohelp=' + nohelp).click(function() { window.scrollTo(0, 0); })); }).fail(function() { diff --git a/server/static/style.css b/server/static/style.css index 1a5cf08..f6714c3 100644 --- a/server/static/style.css +++ b/server/static/style.css @@ -99,6 +99,12 @@ div.container { opacity: 1.0; } +#help-toggle { + position: absolute; + top: 24px; + right: 50px; +} + div.content-box h1 { padding: 6px; padding-top: 0px; From 3f52589535004157679d2842f938e65d9c419d90 Mon Sep 17 00:00:00 2001 From: Chris <1731074+ccbrown@users.noreply.github.com> Date: Sat, 11 Jan 2025 15:13:14 -0500 Subject: [PATCH 73/90] improvements and polish for the help filter (#143) * improvements and polish for the help filter * add translations --- server/activity_handler.go | 7 ++-- server/bolt_database.go | 2 +- server/dynamodb_database.go | 22 +++++++++-- server/index_handler.go | 15 ++++---- server/localization.go | 74 +++++++++++++++++++++++-------------- server/static/index.js | 37 +++++++++---------- 6 files changed, 95 insertions(+), 62 deletions(-) diff --git a/server/activity_handler.go b/server/activity_handler.go index fc45567..6573a85 100644 --- a/server/activity_handler.go +++ b/server/activity_handler.go @@ -17,10 +17,8 @@ type jsonResponse struct { func ActivityHandler(db Database) echo.HandlerFunc { return func(c echo.Context) error { locale := LocaleForRequest(c.Request()) - filter := func(a Activity) bool { - return true - } - if c.QueryParam("nohelp") == "true" { + var filter func(Activity) bool + if c.QueryParams().Has("nohelp") && c.QueryParam("nohelp") != "false" { filter = func(a Activity) bool { if fp, ok := a.(*ForumPost); ok { if fp.ForumId == locale.HelpForumId { @@ -52,6 +50,7 @@ func ActivityHandler(db Database) echo.HandlerFunc { Data: a, }) } + c.Response().Header().Add("Cache-Control", "max-age=120") return c.JSON(200, response) } } diff --git a/server/bolt_database.go b/server/bolt_database.go index 5b23285..5ded537 100644 --- a/server/bolt_database.go +++ b/server/bolt_database.go @@ -66,7 +66,7 @@ func (db *BoltDatabase) Activity(locale *Locale, start string, count int, filter activity, err := unmarshalActivity(k, v) if err != nil { return err - } else if activity != nil && locale.ActivityFilter(activity) && filter(activity) { + } else if activity != nil && locale.ActivityFilter(activity) && (filter == nil || filter(activity)) { ret = append(ret, activity) next = base64.RawURLEncoding.EncodeToString(k) } diff --git a/server/dynamodb_database.go b/server/dynamodb_database.go index d498949..cf7bdbc 100644 --- a/server/dynamodb_database.go +++ b/server/dynamodb_database.go @@ -106,28 +106,42 @@ func (db *DynamoDBDatabase) Activity(locale *Locale, start string, count int, fi } for len(activity) < count { + batchSize := count - len(activity) + if filter != nil { + // if we're filtering results, fetch extra + batchSize = count * 4 + if batchSize < 50 { + batchSize = 50 + } else if batchSize > 1000 { + batchSize = 1000 + } + } result, err := db.client.QueryRequest(&dynamodb.QueryInput{ TableName: aws.String(db.tableName), KeyConditionExpression: aws.String(condition), ExpressionAttributeValues: attributeValues, ExclusiveStartKey: startKey, - Limit: aws.Int64(int64(count - len(activity))), + Limit: aws.Int64(int64(batchSize)), ScanIndexForward: aws.Bool(false), }).Send(context.Background()) if err != nil { return nil, "", err } + startKey = result.LastEvaluatedKey for _, item := range result.Items { if a, err := unmarshalActivity(item["rk"].B, item["v"].B); err != nil { return nil, "", err - } else if a != nil && filter(a) { + } else if a != nil && (filter == nil || filter(a)) { activity = append(activity, a) + if len(activity) == count { + startKey = item + break + } } } - if result.LastEvaluatedKey == nil { + if result.LastEvaluatedKey == nil || len(activity) == count { break } - startKey = result.LastEvaluatedKey } var next string diff --git a/server/index_handler.go b/server/index_handler.go index 927177e..2a291b1 100644 --- a/server/index_handler.go +++ b/server/index_handler.go @@ -11,7 +11,7 @@ var index = ` GGG Tracker - + @@ -37,13 +37,14 @@ var index = ` {{end}} -
-

Tala moana, warrior! This website is approaching its 12th birthday! If you've found it useful over the years, you can now show your appreciation in the form of a recurring or one-time donation on GitHub! In addition to showing your appreciation, this offsets server costs and helps support further feature developments (Twitter/X support? Search? GenAI?).

-

Thanks, and stay safe out there, Exile. ❤️

-

{{call $.Translate "Activity"}}

-
+ {{if ne $.Locale.HelpForumId 0}} + + {{end}} @@ -73,7 +74,7 @@ var index = ` - + ` diff --git a/server/localization.go b/server/localization.go index 860211d..61f5a4d 100644 --- a/server/localization.go +++ b/server/localization.go @@ -102,11 +102,13 @@ var Locales = []*Locale{ Subdomain: "br", Image: "static/images/locales/br.png", Translations: map[string]string{ - "Activity": "Atividade", - "Thread": "Discussão", - "Poster": "Autor", - "Time": "Hora", - "Forum": "Fórum", + "Activity": "Atividade", + "Thread": "Discussão", + "Poster": "Autor", + "Time": "Hora", + "Forum": "Fórum", + "Hide Help Forum": "Ocultar Fórum de Ajuda", + "Show Help Forum": "Mostrar Fórum de Ajuda", }, HelpForumId: 774, }, @@ -114,27 +116,41 @@ var Locales = []*Locale{ Subdomain: "ru", Image: "static/images/locales/ru.png", Translations: map[string]string{ - "Activity": "Активность", - "Thread": "Тема", - "Poster": "Автор", - "Time": "Время", - "Forum": "Форум", + "Activity": "Активность", + "Thread": "Тема", + "Poster": "Автор", + "Time": "Время", + "Forum": "Форум", + "Hide Help Forum": "Скрыть форум помощи", + "Show Help Forum": "Показать Форум Помощи", }, + HelpForumId: 1281, }, { - Subdomain: "th", - Image: "static/images/locales/th.png", + Subdomain: "th", + Image: "static/images/locales/th.png", + Translations: map[string]string{ + "Activity": "กิจกรรม", + "Thread": "กระทู้", + "Poster": "ผู้โพสต์", + "Time": "เวลา", + "Forum": "ฟอรั่ม", + "Hide Help Forum": "ซ่อนฟอรั่มช่วยเหลือ", + "Show Help Forum": "แสดงฟอรั่มช่วยเหลือ", + }, HelpForumId: 1011, }, { Subdomain: "de", Image: "static/images/locales/de.png", Translations: map[string]string{ - "Activity": "Aktivität", - "Thread": "Beitrag", - "Poster": "Verfasser", - "Time": "Datum", - "Forum": "Forum", + "Activity": "Aktivität", + "Thread": "Beitrag", + "Poster": "Verfasser", + "Time": "Datum", + "Forum": "Forum", + "Hide Help Forum": "Hilfeforum ausblenden", + "Show Help Forum": "Hilfeforum anzeigen", }, HelpForumId: 1123, }, @@ -142,11 +158,13 @@ var Locales = []*Locale{ Subdomain: "fr", Image: "static/images/locales/fr.png", Translations: map[string]string{ - "Activity": "Activité", - "Thread": "Fil de discussion", - "Poster": "Posteur", - "Time": "Date", - "Forum": "Forum", + "Activity": "Activité", + "Thread": "Fil de discussion", + "Poster": "Posteur", + "Time": "Date", + "Forum": "Forum", + "Hide Help Forum": "Masquer le forum d'aide", + "Show Help Forum": "Afficher le forum d'aide", }, HelpForumId: 1051, }, @@ -154,11 +172,13 @@ var Locales = []*Locale{ Subdomain: "es", Image: "static/images/locales/es.png", Translations: map[string]string{ - "Activity": "Actividad", - "Thread": "Tema", - "Poster": "Autor", - "Time": "Fecha", - "Forum": "Foro", + "Activity": "Actividad", + "Thread": "Tema", + "Poster": "Autor", + "Time": "Fecha", + "Forum": "Foro", + "Hide Help Forum": "Ocultar el foro de ayuda", + "Show Help Forum": "Mostrar el foro de ayuda", }, HelpForumId: 1193, }, diff --git a/server/static/index.js b/server/static/index.js index f402423..72d6e0e 100644 --- a/server/static/index.js +++ b/server/static/index.js @@ -1,4 +1,5 @@ var currentPage = undefined; +var currentHideHelp = undefined; var POE = { Forum: { @@ -18,18 +19,21 @@ var POE = { function loadActivity() { var params = new URLSearchParams(location.hash.replace(/^#/, '')); var page = params.get('page') || ''; - var nohelp = params.get('nohelp') || ''; - if (currentPage !== undefined && page == currentPage) { + var hideHelp = params.has('nohelp') && params.get('nohelp') !== 'false'; + if (currentPage !== undefined && page == currentPage && currentHideHelp !== undefined && hideHelp == currentHideHelp) { return; } var previousPage = currentPage; + var previousHideHelp = currentHideHelp; + currentPage = page; + currentHideHelp = hideHelp; - if (nohelp == 'true') { - $('#activity-table tbody').empty().append($('').append($('').append($('
').attr('colspan', 6).text('Loading...'))) - } + var canonicalNohelpParam = hideHelp ? '&nohelp' : ''; + + $('#activity-table tbody').empty().append($('
').attr('colspan', 6).text('Loading...'))) - $.get('activity.json?next=' + page + '&nohelp=' + nohelp, function(data) { + $.get('activity.json?next=' + page + canonicalNohelpParam, function(data) { var $tbody = $('#activity-table tbody'); $tbody.empty(); @@ -141,28 +145,23 @@ function loadActivity() { $tbody.append($tr); } - var nohelpText; - var nohelpHref; - if (nohelp != 'true') { - nohelpText = 'Hide Help Forum'; - nohelpHref = '#page=' + page + '&nohelp=true'; + if (hideHelp) { + $('#hide-help-forum').hide(); + $('#show-help-forum').attr('href', page ? '#page=' + page : '#').show(); } else { - nohelpText = 'Show Help Forum'; - nohelpHref = '#page=' + page + '&nohelp=false'; + $('#show-help-forum').hide(); + $('#hide-help-forum').attr('href', (page ? '#page=' + page + '&' : '#') + 'nohelp').show(); } - $('#help-toggle').empty().append($('').text(nohelpText).attr('href', nohelpHref).click(function() { - currentPage = undefined; - window.scrollTo(0, 0); - })); - $('#activity-nav').empty().append($('').text('Next Page').attr('href', '#page=' + data.next + '&nohelp=' + nohelp).click(function() { + $('#activity-nav').empty().append($('').text('Next Page').attr('href', '#page=' + data.next + canonicalNohelpParam).click(function() { window.scrollTo(0, 0); })); }).fail(function() { alert('Something went wrong. Better luck next time.'); currentPage = previousPage if (currentPage !== undefined) { - window.location.hash = 'page=' + currentPage; + var previousNohelpParam = previousHideHelp ? '&nohelp' : ''; + window.location.hash = 'page=' + currentPage + previousNohelpParam; } else { window.location.hash = ''; } From c95373c61f6197668566ec157bcae0ec6badbde2 Mon Sep 17 00:00:00 2001 From: Chris Brown <1731074+ccbrown@users.noreply.github.com> Date: Sun, 19 Jan 2025 22:17:21 -0500 Subject: [PATCH 74/90] add Clint to forum indexer --- server/forum_indexer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index 6d124db..447eae8 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -131,6 +131,7 @@ func (indexer *ForumIndexer) run() { {Username: "Joel_GGG", Discriminator: 1496}, {Username: "Vinky_GGG"}, {Username: "Edmund_GGG", Discriminator: 4844}, + {Username: "Clint"}, } timezone := (*time.Location)(nil) From a320f591cbbe8a49a06b5d3c723647a8e52ef984 Mon Sep 17 00:00:00 2001 From: Chris Brown <1731074+ccbrown@users.noreply.github.com> Date: Sun, 19 Jan 2025 22:22:06 -0500 Subject: [PATCH 75/90] add LeightonJ_GGG to forum indexer --- server/forum_indexer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index 447eae8..1ba0e1b 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -132,6 +132,7 @@ func (indexer *ForumIndexer) run() { {Username: "Vinky_GGG"}, {Username: "Edmund_GGG", Discriminator: 4844}, {Username: "Clint"}, + {Username: "LeightonJ_GGG"}, } timezone := (*time.Location)(nil) From 161a73fe851ef8a89bb55d5f4b5cdea53d3cdcba Mon Sep 17 00:00:00 2001 From: Chris Brown <1731074+ccbrown@users.noreply.github.com> Date: Wed, 22 Jan 2025 17:09:54 -0500 Subject: [PATCH 76/90] add Tai_GGG --- server/forum_indexer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index 1ba0e1b..2415c96 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -133,6 +133,7 @@ func (indexer *ForumIndexer) run() { {Username: "Edmund_GGG", Discriminator: 4844}, {Username: "Clint"}, {Username: "LeightonJ_GGG"}, + {Username: "Tai_GGG"}, } timezone := (*time.Location)(nil) From 6668709d0548eb5fdda89cf6737c7438c8e079e3 Mon Sep 17 00:00:00 2001 From: Chris Brown <1731074+ccbrown@users.noreply.github.com> Date: Thu, 30 Jan 2025 19:56:00 -0500 Subject: [PATCH 77/90] add ShaunB_GGG --- server/forum_indexer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index 2415c96..b2c7e0f 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -134,6 +134,7 @@ func (indexer *ForumIndexer) run() { {Username: "Clint"}, {Username: "LeightonJ_GGG"}, {Username: "Tai_GGG"}, + {Username: "ShaunB_GGG"}, } timezone := (*time.Location)(nil) From 086a1e0e8344902cb9502bd46938551d7f12d51d Mon Sep 17 00:00:00 2001 From: Chris Brown <1731074+ccbrown@users.noreply.github.com> Date: Tue, 4 Feb 2025 21:18:35 -0500 Subject: [PATCH 78/90] add MatthewD_GGG for Reddit --- server/reddit_indexer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/reddit_indexer.go b/server/reddit_indexer.go index a5b060f..225b4aa 100644 --- a/server/reddit_indexer.go +++ b/server/reddit_indexer.go @@ -45,7 +45,7 @@ func (indexer *RedditIndexer) run() { "KamilOrmanJanowski", "Daniel_GGG", "Jeff_GGG", "NapfelGGG", "Baltic_GGG", "Novynn", "Felipe_GGG", "Mel_GGG", "Sarah_GGG", "riandrake", "Kieren_GGG", "Openarl", "Natalia_GGG", "AlexDenfordGGG", "Stacey_GGG", "ZaccieA", "viperesque", "rach_ggg", "Community_Team", - "M59Gar", "Dominic_GGG", "Nick_GGG", + "M59Gar", "Dominic_GGG", "Nick_GGG", "MatthewD_GGG", } next := 0 From 8fa27055d1cccc309b9c9f8c968a6c387cc6b1fb Mon Sep 17 00:00:00 2001 From: Chris Brown <1731074+ccbrown@users.noreply.github.com> Date: Thu, 6 Mar 2025 14:38:25 -0500 Subject: [PATCH 79/90] add Ayelen_GGG and Timothy_GGG --- server/forum_indexer.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index b2c7e0f..2a3f5d3 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -135,6 +135,8 @@ func (indexer *ForumIndexer) run() { {Username: "LeightonJ_GGG"}, {Username: "Tai_GGG"}, {Username: "ShaunB_GGG"}, + {Username: "Ayelen_GGG"}, + {Username: "Timothy_GGG"}, } timezone := (*time.Location)(nil) From 6b0b08bde2fb0d9d37f5840804c790cf48999c40 Mon Sep 17 00:00:00 2001 From: Chris Brown <1731074+ccbrown@users.noreply.github.com> Date: Fri, 14 Mar 2025 21:18:51 -0400 Subject: [PATCH 80/90] add BenMH_GGG --- server/forum_indexer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index 2a3f5d3..9564928 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -137,6 +137,7 @@ func (indexer *ForumIndexer) run() { {Username: "ShaunB_GGG"}, {Username: "Ayelen_GGG"}, {Username: "Timothy_GGG"}, + {Username: "BenMH_GGG"}, } timezone := (*time.Location)(nil) From f69608c69f4585691b06aae22dfc8c62a4b908db Mon Sep 17 00:00:00 2001 From: Chris Brown <1731074+ccbrown@users.noreply.github.com> Date: Mon, 24 Mar 2025 14:03:56 -0400 Subject: [PATCH 81/90] add Ian_GGG --- server/forum_indexer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index 9564928..d4327aa 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -138,6 +138,7 @@ func (indexer *ForumIndexer) run() { {Username: "Ayelen_GGG"}, {Username: "Timothy_GGG"}, {Username: "BenMH_GGG"}, + {Username: "Ian_GGG"}, } timezone := (*time.Location)(nil) From e2578632abcd92510dfd1f853f4513a2058417ef Mon Sep 17 00:00:00 2001 From: Chris Brown <1731074+ccbrown@users.noreply.github.com> Date: Mon, 31 Mar 2025 17:46:56 -0400 Subject: [PATCH 82/90] add EthanH_GGG --- server/forum_indexer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index d4327aa..54d01e4 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -139,6 +139,7 @@ func (indexer *ForumIndexer) run() { {Username: "Timothy_GGG"}, {Username: "BenMH_GGG"}, {Username: "Ian_GGG"}, + {Username: "EthanH_GGG"}, } timezone := (*time.Location)(nil) From 9c3438e6b3551c9a44dabe5ee7916b7da3c2eacb Mon Sep 17 00:00:00 2001 From: Chris Brown <1731074+ccbrown@users.noreply.github.com> Date: Fri, 2 May 2025 13:11:35 -0400 Subject: [PATCH 83/90] add Yrone_GGG#9576 --- server/forum_indexer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index 54d01e4..8b5504b 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -140,6 +140,7 @@ func (indexer *ForumIndexer) run() { {Username: "BenMH_GGG"}, {Username: "Ian_GGG"}, {Username: "EthanH_GGG"}, + {Username: "Yrone_GGG", Discriminator: 9576}, } timezone := (*time.Location)(nil) From 3c25bf96cf21a544d72e93f62c38e0b591dfe905 Mon Sep 17 00:00:00 2001 From: Chris Brown <1731074+ccbrown@users.noreply.github.com> Date: Mon, 12 May 2025 10:53:48 -0400 Subject: [PATCH 84/90] add Sam_GGG#2420 --- server/forum_indexer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index 8b5504b..6766011 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -141,6 +141,7 @@ func (indexer *ForumIndexer) run() { {Username: "Ian_GGG"}, {Username: "EthanH_GGG"}, {Username: "Yrone_GGG", Discriminator: 9576}, + {Username: "Sam_GGG", Discriminator: 2420}, } timezone := (*time.Location)(nil) From 9596fc172f2d3f89e61bb054693bce4984c16490 Mon Sep 17 00:00:00 2001 From: Chris Brown <1731074+ccbrown@users.noreply.github.com> Date: Thu, 26 Jun 2025 01:08:43 -0400 Subject: [PATCH 85/90] add /u/Belakay_ggg and Ting_GGG --- server/forum_indexer.go | 1 + server/reddit_indexer.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index 6766011..4cd1e7d 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -142,6 +142,7 @@ func (indexer *ForumIndexer) run() { {Username: "EthanH_GGG"}, {Username: "Yrone_GGG", Discriminator: 9576}, {Username: "Sam_GGG", Discriminator: 2420}, + {Username: "Ting_GGG", Discriminator: 2357}, } timezone := (*time.Location)(nil) diff --git a/server/reddit_indexer.go b/server/reddit_indexer.go index 225b4aa..d2d097a 100644 --- a/server/reddit_indexer.go +++ b/server/reddit_indexer.go @@ -45,7 +45,7 @@ func (indexer *RedditIndexer) run() { "KamilOrmanJanowski", "Daniel_GGG", "Jeff_GGG", "NapfelGGG", "Baltic_GGG", "Novynn", "Felipe_GGG", "Mel_GGG", "Sarah_GGG", "riandrake", "Kieren_GGG", "Openarl", "Natalia_GGG", "AlexDenfordGGG", "Stacey_GGG", "ZaccieA", "viperesque", "rach_ggg", "Community_Team", - "M59Gar", "Dominic_GGG", "Nick_GGG", "MatthewD_GGG", + "M59Gar", "Dominic_GGG", "Nick_GGG", "MatthewD_GGG", "Belakay_ggg", } next := 0 From dcc76800e69f0a85cdd4b6686b46fd52bfa9b9d3 Mon Sep 17 00:00:00 2001 From: Chris Brown <1731074+ccbrown@users.noreply.github.com> Date: Sat, 28 Jun 2025 22:00:53 -0500 Subject: [PATCH 86/90] add Ramon_GGG --- server/forum_indexer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index 4cd1e7d..039a87b 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -143,6 +143,7 @@ func (indexer *ForumIndexer) run() { {Username: "Yrone_GGG", Discriminator: 9576}, {Username: "Sam_GGG", Discriminator: 2420}, {Username: "Ting_GGG", Discriminator: 2357}, + {Username: "Ramon_GGG"}, } timezone := (*time.Location)(nil) From 065a6ecb68372017ab1669040e564e065603557d Mon Sep 17 00:00:00 2001 From: Chris Brown <1731074+ccbrown@users.noreply.github.com> Date: Thu, 24 Jul 2025 00:44:34 -0400 Subject: [PATCH 87/90] rm yrone discriminator --- server/forum_indexer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index 039a87b..ab451af 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -140,7 +140,7 @@ func (indexer *ForumIndexer) run() { {Username: "BenMH_GGG"}, {Username: "Ian_GGG"}, {Username: "EthanH_GGG"}, - {Username: "Yrone_GGG", Discriminator: 9576}, + {Username: "Yrone_GGG"}, {Username: "Sam_GGG", Discriminator: 2420}, {Username: "Ting_GGG", Discriminator: 2357}, {Username: "Ramon_GGG"}, From 7ce0fd663be34341f93a6b0b0af1d16e5cdbd810 Mon Sep 17 00:00:00 2001 From: Chris Brown <1731074+ccbrown@users.noreply.github.com> Date: Mon, 11 Aug 2025 16:43:42 -0400 Subject: [PATCH 88/90] add Sameer_GGG --- server/forum_indexer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index ab451af..afb27de 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -144,6 +144,7 @@ func (indexer *ForumIndexer) run() { {Username: "Sam_GGG", Discriminator: 2420}, {Username: "Ting_GGG", Discriminator: 2357}, {Username: "Ramon_GGG"}, + {Username: "Sameer_GGG"}, } timezone := (*time.Location)(nil) From 7658d440daccb2655d75a54c03467456b7223737 Mon Sep 17 00:00:00 2001 From: Chris Brown <1731074+ccbrown@users.noreply.github.com> Date: Sat, 23 Aug 2025 10:20:31 -0500 Subject: [PATCH 89/90] add TobyM_GGG and Lachlan_GGG --- server/forum_indexer.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index afb27de..2110bda 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -145,6 +145,8 @@ func (indexer *ForumIndexer) run() { {Username: "Ting_GGG", Discriminator: 2357}, {Username: "Ramon_GGG"}, {Username: "Sameer_GGG"}, + {Username: "TobyM_GGG"}, + {Username: "Lachlan_GGG"}, } timezone := (*time.Location)(nil) From 3a9b71f5662391d93fc92bdc02d9c29f3909a007 Mon Sep 17 00:00:00 2001 From: Chris Brown <1731074+ccbrown@users.noreply.github.com> Date: Sat, 25 Oct 2025 13:20:30 -0400 Subject: [PATCH 90/90] add Ringatu_GGG --- server/forum_indexer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index 2110bda..915b049 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -147,6 +147,7 @@ func (indexer *ForumIndexer) run() { {Username: "Sameer_GGG"}, {Username: "TobyM_GGG"}, {Username: "Lachlan_GGG"}, + {Username: "Ringatu_GGG"}, } timezone := (*time.Location)(nil)