Skip to content

Commit 5ed72da

Browse files
ccbrownccbrown
authored
add dynamodb support (#28)
* add dynamodb support * don't export dynamodb table creation function * compatibility fix for older db entries
1 parent c927a0d commit 5ed72da

14 files changed

+488
-142
lines changed

‎aws-sam/README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# aws-sam
2+
3+
This directory contains resources for deploying the tracker using the AWS Serverless Application Model (SAM). By deploying via serverless technologies, you can create a cost-effective (free or nearly free) scalable deployment.

‎aws-sam/gggtracker.cfn.yaml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
AWSTemplateFormatVersion: '2010-09-09'
2+
Description: github.com/ccbrown/gggtracker
3+
Resources:
4+
DynamoDBTable:
5+
Type: AWS::DynamoDB::Table
6+
Properties:
7+
AttributeDefinitions:
8+
- AttributeName: hk
9+
AttributeType: B
10+
- AttributeName: rk
11+
AttributeType: B
12+
KeySchema:
13+
- AttributeName: hk
14+
KeyType: HASH
15+
- AttributeName: rk
16+
KeyType: RANGE
17+
ProvisionedThroughput:
18+
ReadCapacityUnits: 25
19+
WriteCapacityUnits: 25

‎main.go‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,10 @@ import (
44
"fmt"
55
"net/http"
66
"path"
7+
"strings"
78

9+
"github.com/aws/aws-sdk-go-v2/aws/external"
10+
"github.com/aws/aws-sdk-go-v2/service/dynamodb"
811
"github.com/labstack/echo"
912
"github.com/labstack/echo/middleware"
1013
log "github.com/sirupsen/logrus"
@@ -18,17 +21,29 @@ func main() {
1821
pflag.IntP("port", "p", 8080, "the port to listen on")
1922
pflag.String("staticdir", "./server/static", "the static files to serve")
2023
pflag.String("ga", "", "a google analytics account")
21-
pflag.String("db", "./gggtracker.db", "the database path")
24+
pflag.String("db", "./gggtracker.db", "the database file path")
25+
pflag.String("dynamodb-table", "", "if given, DynamoDB will be used instead of a database file")
2226
pflag.String("forumsession", "", "the POESESSID cookie for a forum session")
2327
viper.BindPFlags(pflag.CommandLine)
2428
pflag.Parse()
2529

2630
viper.SetEnvPrefix("gggtracker")
31+
viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
2732
viper.AutomaticEnv()
2833

2934
e := echo.New()
3035

31-
db, err := server.OpenDatabase(viper.GetString("db"))
36+
var db server.Database
37+
var err error
38+
if tableName := viper.GetString("dynamodb-table"); tableName != "" {
39+
config, err := external.LoadDefaultAWSConfig()
40+
if err != nil {
41+
log.Fatal(err)
42+
}
43+
db, err = server.NewDynamoDBDatabase(dynamodb.New(config), tableName)
44+
} else {
45+
db, err = server.NewBoltDatabase(viper.GetString("db"))
46+
}
3247
if err != nil {
3348
log.Fatal(err)
3449
}

‎server/activity_handler.go‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,12 @@ type jsonResponse struct {
1414
Next string `json:"next"`
1515
}
1616

17-
func ActivityHandler(db *Database) echo.HandlerFunc {
17+
func ActivityHandler(db Database) echo.HandlerFunc {
1818
return func(c echo.Context) error {
19-
activity, next := db.Activity(c.QueryParam("next"), 50, LocaleForRequest(c.Request()).ActivityFilter)
19+
activity, next, err := db.Activity(LocaleForRequest(c.Request()), c.QueryParam("next"), 50)
20+
if err != nil {
21+
return err
22+
}
2023
response := jsonResponse{
2124
Next: next,
2225
}

‎server/bolt_database.go‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
package server
2+
3+
import (
4+
"encoding/base64"
5+
6+
"github.com/boltdb/bolt"
7+
)
8+
9+
type BoltDatabase struct {
10+
db *bolt.DB
11+
}
12+
13+
func NewBoltDatabase(path string) (*BoltDatabase, error) {
14+
db, err := bolt.Open(path, 0600, nil)
15+
if err != nil {
16+
return nil, err
17+
}
18+
19+
db.Update(func(tx *bolt.Tx) error {
20+
_, err := tx.CreateBucketIfNotExists([]byte("activity"))
21+
if err != nil {
22+
return err
23+
}
24+
return nil
25+
})
26+
27+
return &BoltDatabase{
28+
db: db,
29+
}, nil
30+
}
31+
32+
func (db *BoltDatabase) AddActivity(activity []Activity) error {
33+
return db.db.Update(func(tx *bolt.Tx) error {
34+
b := tx.Bucket([]byte("activity"))
35+
for _, a := range activity {
36+
k, v, err := marshalActivity(a)
37+
if err != nil {
38+
return err
39+
}
40+
b.Put(k, v)
41+
}
42+
return nil
43+
})
44+
}
45+
46+
func (db *BoltDatabase) Activity(locale *Locale, start string, count int) ([]Activity, string, error) {
47+
ret := []Activity(nil)
48+
next := ""
49+
if err := db.db.View(func(tx *bolt.Tx) error {
50+
c := tx.Bucket([]byte("activity")).Cursor()
51+
var k, v []byte
52+
if start == "" {
53+
k, v = c.Last()
54+
} else {
55+
s, err := base64.RawURLEncoding.DecodeString(start)
56+
if err != nil {
57+
k, v = c.Last()
58+
} else {
59+
k, v = c.Seek(s)
60+
if k != nil {
61+
k, v = c.Prev()
62+
}
63+
}
64+
}
65+
for len(ret) < count && k != nil {
66+
activity, err := unmarshalActivity(k, v)
67+
if err != nil {
68+
return err
69+
} else if activity != nil && locale.ActivityFilter(activity) {
70+
ret = append(ret, activity)
71+
next = base64.RawURLEncoding.EncodeToString(k)
72+
}
73+
k, v = c.Prev()
74+
}
75+
return nil
76+
}); err != nil {
77+
return nil, "", err
78+
}
79+
return ret, next, nil
80+
}
81+
82+
func (db *BoltDatabase) Close() error {
83+
return db.db.Close()
84+
}

‎server/bolt_database_test.go‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
package server
2+
3+
import (
4+
"io/ioutil"
5+
"os"
6+
"path"
7+
"testing"
8+
9+
"github.com/stretchr/testify/require"
10+
)
11+
12+
func TestBoltDatabase(t *testing.T) {
13+
dir, err := ioutil.TempDir("testdata", "db")
14+
require.NoError(t, err)
15+
defer os.RemoveAll(dir)
16+
17+
db, err := NewBoltDatabase(path.Join(dir, "test.db"))
18+
require.NoError(t, err)
19+
defer db.Close()
20+
21+
testDatabase(t, db)
22+
}

‎server/database.go‎

Lines changed: 49 additions & 112 deletions
Original file line numberDiff line numberDiff line change
@@ -1,133 +1,70 @@
11
package server
22

33
import (
4-
"encoding/base64"
54
"encoding/binary"
6-
"encoding/json"
75

8-
"github.com/boltdb/bolt"
6+
json "github.com/json-iterator/go"
97
)
108

11-
type Database struct {
12-
db *bolt.DB
13-
}
14-
15-
func OpenDatabase(path string) (*Database, error) {
16-
db, err := bolt.Open(path, 0600, nil)
17-
if err != nil {
18-
return nil, err
19-
}
20-
21-
db.Update(func(tx *bolt.Tx) error {
22-
_, err := tx.CreateBucketIfNotExists([]byte("activity"))
23-
if err != nil {
24-
return err
25-
}
26-
return nil
27-
})
28-
29-
return &Database{
30-
db: db,
31-
}, nil
32-
}
33-
34-
func (db *Database) Close() {
35-
db.db.Close()
36-
}
37-
389
const (
3910
ForumPostType = iota
4011
RedditCommentType
4112
RedditPostType
4213
)
4314

44-
func (db *Database) AddActivity(activity []Activity) {
45-
err := db.db.Update(func(tx *bolt.Tx) error {
46-
b := tx.Bucket([]byte("activity"))
47-
for _, a := range activity {
48-
buf, err := json.Marshal(a)
49-
if err != nil {
50-
return err
51-
}
52-
k := make([]byte, 10)
53-
binary.BigEndian.PutUint64(k, uint64(a.ActivityTime().Unix())<<24)
54-
switch a.(type) {
55-
case *ForumPost:
56-
k[5] = ForumPostType
57-
case *RedditComment:
58-
k[5] = RedditCommentType
59-
case *RedditPost:
60-
k[5] = RedditPostType
61-
}
62-
binary.BigEndian.PutUint32(k[6:], a.ActivityKey())
63-
b.Put(k, buf)
64-
}
65-
return nil
66-
})
15+
type Database interface {
16+
AddActivity(activity []Activity) error
17+
Activity(locale *Locale, start string, count int) ([]Activity, string, error)
18+
Close() error
19+
}
20+
21+
func marshalActivity(a Activity) (key, value []byte, err error) {
22+
buf, err := json.Marshal(a)
6723
if err != nil {
68-
panic(err)
24+
return nil, nil, err
6925
}
26+
k := make([]byte, 10)
27+
binary.BigEndian.PutUint64(k, uint64(a.ActivityTime().Unix())<<24)
28+
switch a.(type) {
29+
case *ForumPost:
30+
k[5] = ForumPostType
31+
case *RedditComment:
32+
k[5] = RedditCommentType
33+
case *RedditPost:
34+
k[5] = RedditPostType
35+
}
36+
binary.BigEndian.PutUint32(k[6:], a.ActivityKey())
37+
return k, buf, nil
7038
}
7139

72-
func (db *Database) Activity(start string, count int, filter func(Activity) bool) ([]Activity, string) {
73-
ret := []Activity(nil)
74-
next := ""
75-
err := db.db.View(func(tx *bolt.Tx) error {
76-
c := tx.Bucket([]byte("activity")).Cursor()
77-
var k, v []byte
78-
if start == "" {
79-
k, v = c.Last()
80-
} else {
81-
s, err := base64.RawURLEncoding.DecodeString(start)
82-
if err != nil {
83-
k, v = c.Last()
84-
} else {
85-
k, v = c.Seek(s)
86-
if k != nil {
87-
k, v = c.Prev()
88-
}
89-
}
40+
func unmarshalActivity(key, value []byte) (Activity, error) {
41+
switch key[5] {
42+
case ForumPostType:
43+
post := &ForumPost{}
44+
err := json.Unmarshal(value, post)
45+
if err != nil {
46+
return nil, err
9047
}
91-
for len(ret) < count && k != nil {
92-
var activity Activity
93-
switch k[5] {
94-
case ForumPostType:
95-
post := &ForumPost{}
96-
err := json.Unmarshal(v, post)
97-
if err != nil {
98-
return err
99-
}
100-
if post.Host == "" {
101-
post.Host = "www.pathofexile.com"
102-
}
103-
if post.Id != 0 {
104-
activity = post
105-
}
106-
case RedditCommentType:
107-
comment := &RedditComment{}
108-
err := json.Unmarshal(v, comment)
109-
if err != nil {
110-
return err
111-
}
112-
activity = comment
113-
case RedditPostType:
114-
post := &RedditPost{}
115-
err := json.Unmarshal(v, post)
116-
if err != nil {
117-
return err
118-
}
119-
activity = post
120-
}
121-
if activity != nil && (filter == nil || filter(activity)) {
122-
ret = append(ret, activity)
123-
next = base64.RawURLEncoding.EncodeToString(k)
124-
}
125-
k, v = c.Prev()
48+
if post.Host == "" {
49+
post.Host = "www.pathofexile.com"
12650
}
127-
return nil
128-
})
129-
if err != nil {
130-
panic(err)
51+
if post.Id != 0 {
52+
return post, nil
53+
}
54+
case RedditCommentType:
55+
comment := &RedditComment{}
56+
err := json.Unmarshal(value, comment)
57+
if err != nil {
58+
return nil, err
59+
}
60+
return comment, nil
61+
case RedditPostType:
62+
post := &RedditPost{}
63+
err := json.Unmarshal(value, post)
64+
if err != nil {
65+
return nil, err
66+
}
67+
return post, nil
13168
}
132-
return ret, next
69+
return nil, nil
13370
}

0 commit comments

Comments
 (0)