diff --git a/.dockerignore b/.dockerignore index bbfd30f..2d578ec 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,5 @@ * !*.go !server/* +!go.mod +!go.sum diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..852d5ee --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,3 @@ +# These are supported funding model platforms + +github: [ccbrown] diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml new file mode 100644 index 0000000..f033944 --- /dev/null +++ b/.github/workflows/push.yml @@ -0,0 +1,12 @@ +name: Push +on: [push] +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Build + run: | + make docker-image diff --git a/.gitignore b/.gitignore index 3990fde..4edd60f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ *.db gggtracker +/server/bindata.go +/vendor diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index bf46dbe..0000000 --- a/.travis.yml +++ /dev/null @@ -1,6 +0,0 @@ -language: go -sudo: required -services: - - docker -script: - - make docker-image diff --git a/Dockerfile b/Dockerfile index f94d04f..6cacd63 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,10 @@ -FROM golang:alpine +FROM golang:1.20-alpine WORKDIR /go/src/github.com/ccbrown/gggtracker -ADD . . -RUN apk add --no-cache git && go get -t ./... -RUN go vet . && go test -v ./... +ADD . . +RUN go vet . +RUN go test -v ./... RUN go build . -ENTRYPOINT ["gggtracker"] +ENTRYPOINT ["./gggtracker"] diff --git a/README.md b/README.md index 96d1ea7..bfd6a82 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ -# Welcome! [![Build Status](https://travis-ci.org/ccbrown/gggtracker.svg?branch=master)](https://travis-ci.org/ccbrown/gggtracker) +# Welcome! ![Build Status](https://github.com/ccbrown/gggtracker/actions/workflows/push.yml/badge.svg) -This is the repository for gggtracker.com. If there's something you think the site is missing, please either a.) open an issue to request the feature or b.) develop the feature yourself and put in a pull request. +This is the repository for [gggtracker.com](https://gggtracker.com). If there's something you think the site is missing, please either a.) open an issue to request the feature or b.) develop the feature yourself and put in a pull request. ### Development -If you have Go installed, you can develop and run the server in the usual way: `go get ./... && go run main.go` +If you have Go installed, you can develop and run the server in the usual way: `go run main.go` ![Development](development.gif) diff --git a/aws-sam/.gitignore b/aws-sam/.gitignore new file mode 100644 index 0000000..c39f315 --- /dev/null +++ b/aws-sam/.gitignore @@ -0,0 +1,3 @@ +/build +/dist.zip +*-packaged.cfn.yaml diff --git a/aws-sam/Makefile b/aws-sam/Makefile new file mode 100644 index 0000000..55590af --- /dev/null +++ b/aws-sam/Makefile @@ -0,0 +1,17 @@ +.PHONY: build build-environment-build lambda-environment-build + +dist.zip: build + zip -j dist.zip build/main + +build: + go generate ../... + docker-compose run --rm build-environment make build-environment-build + docker-compose run --rm lambda-environment make lambda-environment-build + +build-environment-build: + rm -rf ./build + go build -o ./build/main . + ./build/main --help + +lambda-environment-build: + ./build/main --help || (ldd ./build/main && exit 1) diff --git a/aws-sam/README.md b/aws-sam/README.md new file mode 100644 index 0000000..437266d --- /dev/null +++ b/aws-sam/README.md @@ -0,0 +1,3 @@ +# aws-sam + +This directory contains resources for deploying the tracker using the AWS Serverless Application Model (SAM). By deploying via serverless technologies, you can create a cost-effective (free or nearly free) scalable deployment. diff --git a/aws-sam/docker-compose.yaml b/aws-sam/docker-compose.yaml new file mode 100644 index 0000000..3641514 --- /dev/null +++ b/aws-sam/docker-compose.yaml @@ -0,0 +1,17 @@ +version: '3' +services: + build-environment: + image: golang:1.20.0 + platform: linux/amd64 + volumes: + - ../:/go/src/github.com/ccbrown/gggtracker + working_dir: /go/src/github.com/ccbrown/gggtracker/aws-sam + lambda-environment: + entrypoint: [] + environment: + LD_LIBRARY_PATH: '' + image: lambci/lambda:go1.x + platform: linux/amd64 + volumes: + - ../:/go/src/github.com/ccbrown/gggtracker + working_dir: /go/src/github.com/ccbrown/gggtracker/aws-sam diff --git a/aws-sam/gggtracker.cfn.yaml b/aws-sam/gggtracker.cfn.yaml new file mode 100644 index 0000000..b6bf81a --- /dev/null +++ b/aws-sam/gggtracker.cfn.yaml @@ -0,0 +1,290 @@ +AWSTemplateFormatVersion: '2010-09-09' +Description: github.com/ccbrown/gggtracker +Parameters: + BinaryMediaType: + Type: String + AllowedValues: + - '*~1*' + - '*/*' + Description: See https://forums.aws.amazon.com/thread.jspa?messageID=797934 + CodeS3Bucket: + Type: String + Description: The bucket that contains the code to deploy. + CodeS3Key: + Type: String + Description: The key for the code to deploy. + CertificateARN: + Type: String + Description: A certificate corresponding to DomainName. + DomainName: + Type: String + Description: The desired domain name. It'll be up to you to actually change your DNS after the stack is deployed. + GoogleAnalytics: + Type: String + Default: '' +Resources: + API: + Type: AWS::ApiGateway::RestApi + Properties: + BinaryMediaTypes: + - !Ref BinaryMediaType + Name: !Ref AWS::StackName + APIAccount: + Type: AWS::ApiGateway::Account + Properties: + CloudWatchRoleArn: !GetAtt APICloudWatchRole.Arn + APIBasePathMapping: + Type: AWS::ApiGateway::BasePathMapping + DependsOn: APIDomainName + Properties: + DomainName: !Ref DomainName + RestApiId: !Ref API + Stage: !Ref APIStage + APIBasePathMappingSubdomain1: + Type: AWS::ApiGateway::BasePathMapping + DependsOn: APISubdomainName1 + Properties: + DomainName: !Sub br.${DomainName} + RestApiId: !Ref API + Stage: !Ref APIStage + APIBasePathMappingSubdomain2: + Type: AWS::ApiGateway::BasePathMapping + DependsOn: APISubdomainName2 + Properties: + DomainName: !Sub ru.${DomainName} + RestApiId: !Ref API + Stage: !Ref APIStage + APIBasePathMappingSubdomain3: + Type: AWS::ApiGateway::BasePathMapping + DependsOn: APISubdomainName3 + Properties: + DomainName: !Sub th.${DomainName} + RestApiId: !Ref API + Stage: !Ref APIStage + APIBasePathMappingSubdomain4: + Type: AWS::ApiGateway::BasePathMapping + DependsOn: APISubdomainName4 + Properties: + DomainName: !Sub de.${DomainName} + RestApiId: !Ref API + Stage: !Ref APIStage + APIBasePathMappingSubdomain5: + Type: AWS::ApiGateway::BasePathMapping + DependsOn: APISubdomainName5 + Properties: + DomainName: !Sub fr.${DomainName} + RestApiId: !Ref API + Stage: !Ref APIStage + APIBasePathMappingSubdomain6: + Type: AWS::ApiGateway::BasePathMapping + DependsOn: APISubdomainName6 + Properties: + DomainName: !Sub es.${DomainName} + RestApiId: !Ref API + Stage: !Ref APIStage + APIBasePathMappingSubdomain7: + Type: AWS::ApiGateway::BasePathMapping + DependsOn: APISubdomainName7 + Properties: + DomainName: !Sub www.${DomainName} + RestApiId: !Ref API + Stage: !Ref APIStage + APIBasePathMappingSubdomain8: + Type: AWS::ApiGateway::BasePathMapping + DependsOn: APISubdomainName8 + Properties: + DomainName: !Sub jp.${DomainName} + RestApiId: !Ref API + Stage: !Ref APIStage + APICloudWatchRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: apigateway.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AmazonAPIGatewayPushToCloudWatchLogs + APIDomainName: + Type: AWS::ApiGateway::DomainName + Properties: + CertificateArn: !Ref CertificateARN + DomainName: !Ref DomainName + EndpointConfiguration: + Types: + - EDGE + APISubdomainName1: + Type: AWS::ApiGateway::DomainName + Properties: + CertificateArn: !Ref CertificateARN + DomainName: !Sub br.${DomainName} + EndpointConfiguration: + Types: + - EDGE + APISubdomainName2: + Type: AWS::ApiGateway::DomainName + Properties: + CertificateArn: !Ref CertificateARN + DomainName: !Sub ru.${DomainName} + EndpointConfiguration: + Types: + - EDGE + APISubdomainName3: + Type: AWS::ApiGateway::DomainName + Properties: + CertificateArn: !Ref CertificateARN + DomainName: !Sub th.${DomainName} + EndpointConfiguration: + Types: + - EDGE + APISubdomainName4: + Type: AWS::ApiGateway::DomainName + Properties: + CertificateArn: !Ref CertificateARN + DomainName: !Sub de.${DomainName} + EndpointConfiguration: + Types: + - EDGE + APISubdomainName5: + Type: AWS::ApiGateway::DomainName + Properties: + CertificateArn: !Ref CertificateARN + DomainName: !Sub fr.${DomainName} + EndpointConfiguration: + Types: + - EDGE + APISubdomainName6: + Type: AWS::ApiGateway::DomainName + Properties: + CertificateArn: !Ref CertificateARN + DomainName: !Sub es.${DomainName} + EndpointConfiguration: + Types: + - EDGE + APISubdomainName7: + Type: AWS::ApiGateway::DomainName + Properties: + CertificateArn: !Ref CertificateARN + DomainName: !Sub www.${DomainName} + EndpointConfiguration: + Types: + - EDGE + APISubdomainName8: + Type: AWS::ApiGateway::DomainName + Properties: + CertificateArn: !Ref CertificateARN + DomainName: !Sub jp.${DomainName} + EndpointConfiguration: + Types: + - EDGE + APIProxyResource: + Type: AWS::ApiGateway::Resource + Properties: + ParentId: !GetAtt API.RootResourceId + PathPart: '{proxy+}' + RestApiId: !Ref API + APIRootMethod: + Type: AWS::ApiGateway::Method + Properties: + AuthorizationType: NONE + HttpMethod: ANY + Integration: + Type: AWS_PROXY + IntegrationHttpMethod: POST + Uri: !Sub arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${APIFunction.Arn}/invocations + ResourceId: !GetAtt API.RootResourceId + RestApiId: !Ref API + APIProxyMethod: + Type: AWS::ApiGateway::Method + Properties: + AuthorizationType: NONE + HttpMethod: ANY + Integration: + Type: AWS_PROXY + IntegrationHttpMethod: POST + Uri: !Sub arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${APIFunction.Arn}/invocations + ResourceId: !Ref APIProxyResource + RestApiId: !Ref API + APIDeployment: + Type: AWS::ApiGateway::Deployment + DependsOn: + - APIRootMethod + - APIProxyMethod + Properties: + RestApiId: !Ref API + APIStage: + Type: AWS::ApiGateway::Stage + DependsOn: + - APIAccount + Properties: + DeploymentId: !Ref APIDeployment + MethodSettings: + - HttpMethod: '*' + LoggingLevel: ERROR + MetricsEnabled: true + ResourcePath: /* + RestApiId: !Ref API + StageName: stage + DynamoDBTable: + Type: AWS::DynamoDB::Table + Properties: + AttributeDefinitions: + - AttributeName: hk + AttributeType: B + - AttributeName: rk + AttributeType: B + KeySchema: + - AttributeName: hk + KeyType: HASH + - AttributeName: rk + KeyType: RANGE + ProvisionedThroughput: + ReadCapacityUnits: 25 + WriteCapacityUnits: 50 + APIFunction: + Type: AWS::Lambda::Function + Properties: + Code: + S3Bucket: !Ref CodeS3Bucket + S3Key: !Ref CodeS3Key + Environment: + Variables: + GGGTRACKER_DYNAMODB_TABLE: !Ref DynamoDBTable + GGGTRACKER_GA: !Ref GoogleAnalytics + Handler: main + MemorySize: 128 + Role: !GetAtt AppFunctionRole.Arn + Runtime: go1.x + Timeout: 60 + APIFunctionAPIPermission: + Type: AWS::Lambda::Permission + Properties: + Action: lambda:InvokeFunction + FunctionName: !GetAtt APIFunction.Arn + Principal: apigateway.amazonaws.com + SourceArn: !Sub arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}:${API}/*/*/* + AppFunctionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: lambda.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole + Policies: + - PolicyName: gggtracker + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: dynamodb:* + Resource: + - !Sub arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${DynamoDBTable} + - !Sub arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${DynamoDBTable}/* diff --git a/aws-sam/main.go b/aws-sam/main.go new file mode 100644 index 0000000..f9ca730 --- /dev/null +++ b/aws-sam/main.go @@ -0,0 +1,160 @@ +package main + +import ( + "bufio" + "bytes" + "encoding/base64" + "flag" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "os" + "strings" + + "github.com/aws/aws-lambda-go/events" + "github.com/aws/aws-lambda-go/lambda" + "github.com/aws/aws-sdk-go-v2/aws/external" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/crypto/ssh/terminal" + "golang.org/x/sys/unix" + + "github.com/ccbrown/gggtracker/server" +) + +type ResponseWriter struct { + header http.Header + statusCode int + body *bytes.Buffer +} + +func (w *ResponseWriter) Header() http.Header { + if w.header == nil { + w.header = http.Header{} + } + return w.header +} + +func (w *ResponseWriter) Write(b []byte) (int, error) { + if w.body == nil { + w.body = &bytes.Buffer{} + } + return w.body.Write(b) +} + +func (w *ResponseWriter) WriteHeader(statusCode int) { + w.statusCode = statusCode +} + +func (w *ResponseWriter) APIGatewayProxyResponse() (*events.APIGatewayProxyResponse, error) { + ret := &events.APIGatewayProxyResponse{} + if w.statusCode != 0 { + ret.StatusCode = w.statusCode + } else { + ret.StatusCode = http.StatusOK + } + for key, values := range w.header { + if len(values) > 1 { + return nil, fmt.Errorf("header has multiple values: " + key) + } else if len(values) == 1 { + if ret.Headers == nil { + ret.Headers = map[string]string{} + } + ret.Headers[key] = values[0] + } + } + if w.body != nil { + ret.Body = base64.StdEncoding.EncodeToString(w.body.Bytes()) + ret.IsBase64Encoded = true + } + return ret, nil +} + +func NewRequest(request *events.APIGatewayProxyRequest) (*http.Request, error) { + resource, err := url.ParseRequestURI(request.Path) + if err != nil { + return nil, errors.Wrap(err, "unable to parse request URI") + } + if len(request.QueryStringParameters) > 0 { + query := url.Values{} + for key, value := range request.QueryStringParameters { + query.Set(key, value) + } + resource.RawQuery = query.Encode() + } + req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(request.HTTPMethod + " " + resource.RequestURI() + " HTTP/1.0\r\n\r\n"))) + if err != nil { + return nil, errors.Wrap(err, "unable to create request") + } + + req.Proto = "HTTP/1.1" + req.ProtoMinor = 1 + + if request.Body != "" { + var body []byte + if request.IsBase64Encoded { + body, err = base64.StdEncoding.DecodeString(request.Body) + if err != nil { + return nil, errors.Wrap(err, "unable to decode base64 body") + } + } else { + body = []byte(request.Body) + } + req.ContentLength = int64(len(body)) + req.Body = ioutil.NopCloser(bytes.NewReader(body)) + } + + for key, value := range request.Headers { + req.Header.Set(key, value) + } + req.Host = req.Header.Get("Host") + req.RemoteAddr = request.RequestContext.Identity.SourceIP + + return req, nil +} + +func Handler(handler http.Handler) func(*events.APIGatewayProxyRequest) (*events.APIGatewayProxyResponse, error) { + return func(request *events.APIGatewayProxyRequest) (*events.APIGatewayProxyResponse, error) { + req, err := NewRequest(request) + if err != nil { + return nil, err + } + resp := &ResponseWriter{} + handler.ServeHTTP(resp, req) + return resp.APIGatewayProxyResponse() + } +} + +func StartHTTPHandler() { + config, err := external.LoadDefaultAWSConfig() + if err != nil { + logrus.Fatal(err) + } + db, err := server.NewDynamoDBDatabase(dynamodb.New(config), os.Getenv("GGGTRACKER_DYNAMODB_TABLE")) + if err != nil { + logrus.Fatal(err) + } + defer db.Close() + + e := server.New(db, os.Getenv("GGGTRACKER_GA")) + lambda.Start(Handler(e)) +} + +func main() { + if !terminal.IsTerminal(unix.Stdout) { + logrus.SetFormatter(&logrus.JSONFormatter{}) + } + + flags := flag.NewFlagSet(os.Args[0], flag.ContinueOnError) + if err := flags.Parse(os.Args[1:]); err != nil { + if err == flag.ErrHelp { + // Exit with no error if --help was given. This is used to test the build. + os.Exit(0) + } + logrus.Fatal(err) + } + + StartHTTPHandler() +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..6e82471 --- /dev/null +++ b/go.mod @@ -0,0 +1,48 @@ +module github.com/ccbrown/gggtracker + +go 1.20 + +require ( + github.com/PuerkitoBio/goquery v1.3.0 + github.com/aws/aws-lambda-go v1.7.0 + github.com/aws/aws-sdk-go-v2 v0.16.0 + github.com/boltdb/bolt v1.3.1 + github.com/json-iterator/go v1.1.5 + github.com/labstack/echo v3.2.6+incompatible + github.com/pkg/errors v0.8.0 + github.com/sirupsen/logrus v1.0.4 + github.com/spf13/pflag v1.0.0 + github.com/spf13/viper v1.0.0 + github.com/stretchr/testify v1.2.2 + golang.org/x/crypto v0.0.0-20180228161326-91a49db82a88 + golang.org/x/sys v0.0.0-20180302081741-dd2ff4accc09 +) + +require ( + github.com/BurntSushi/toml v1.3.2 // indirect + github.com/andybalholm/cascadia v1.0.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgrijalva/jwt-go v3.1.0+incompatible // indirect + github.com/fsnotify/fsnotify v1.4.7 // indirect + github.com/hashicorp/hcl v0.0.0-20171017181929-23c074d0eceb // indirect + github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af // indirect + github.com/labstack/gommon v0.2.2-0.20170925052817-57409ada9da0 // indirect + github.com/magiconair/properties v1.7.6 // indirect + github.com/mattn/go-colorable v0.0.9 // indirect + github.com/mattn/go-isatty v0.0.3 // indirect + github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 // indirect + github.com/pelletier/go-toml v1.1.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/spf13/afero v1.0.2 // indirect + github.com/spf13/cast v1.2.0 // indirect + github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasttemplate v0.0.0-20170224212429-dcecefd839c4 // indirect + golang.org/x/net v0.0.0-20181201002055-351d144fa1fc // indirect + golang.org/x/text v0.3.0 // indirect + gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect + gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect + gopkg.in/yaml.v2 v2.1.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..3b536e7 --- /dev/null +++ b/go.sum @@ -0,0 +1,86 @@ +github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/PuerkitoBio/goquery v1.3.0 h1:2LzdaeRwZjIMW7iKEei51jiCPB33mou4AI7QCzS4NgE= +github.com/PuerkitoBio/goquery v1.3.0/go.mod h1:T9ezsOHcCrDCgA8aF1Cqr3sSYbO/xgdy8/R/XiIMAhA= +github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o= +github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/aws/aws-lambda-go v1.7.0 h1:g3Ad7aw27B2lhQLIuK7Aha+cWSaHr7ZNlngveHkhZyo= +github.com/aws/aws-lambda-go v1.7.0/go.mod h1:zUsUQhAUjYzR8AuduJPCfhBuKWUaDbQiPOG+ouzmE1A= +github.com/aws/aws-sdk-go-v2 v0.16.0 h1:X5pkFnjRNdDEX18NwDGWMaWL5ocNQX0qIYEhEcsTy64= +github.com/aws/aws-sdk-go-v2 v0.16.0/go.mod h1:pFLIN9LDjOEwHfruGweAXEq0XaD6uRkY8FsRkxhuBIg= +github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.1.0+incompatible h1:FFziAwDQQ2dz1XClWMkwvukur3evtZx7x/wMHKM1i20= +github.com/dgrijalva/jwt-go v3.1.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/hashicorp/hcl v0.0.0-20171017181929-23c074d0eceb h1:1OvvPvZkn/yCQ3xBcM8y4020wdkMXPHLB4+NfoGWh4U= +github.com/hashicorp/hcl v0.0.0-20171017181929-23c074d0eceb/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/json-iterator/go v1.1.5 h1:gL2yXlmiIo4+t+y32d4WGwOjKGYcGOuyrg46vadswDE= +github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/labstack/echo v3.2.6+incompatible h1:28U/uXFFKBIP+VQIqq641LuYhMS7Br9ZlwEXERaohCc= +github.com/labstack/echo v3.2.6+incompatible/go.mod h1:0INS7j/VjnFxD4E2wkz67b8cVwCLbBmJyDaka6Cmk1s= +github.com/labstack/gommon v0.2.2-0.20170925052817-57409ada9da0 h1:7AIW1qc9sYYTZLamTsRKSmVvJDXkZZrIWXHDK4Gq4X0= +github.com/labstack/gommon v0.2.2-0.20170925052817-57409ada9da0/go.mod h1:/tj9csK2iPSBvn+3NLM9e52usepMtrd5ilFYA+wQNJ4= +github.com/magiconair/properties v1.7.6 h1:U+1DqNen04MdEPgFiIwdOUiqZ8qPa37xgogX/sd3+54= +github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238 h1:+MZW2uvHgN8kYvksEN3f7eFL2wpzk0GxmlFsMybWc7E= +github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/pelletier/go-toml v1.1.0 h1:cmiOvKzEunMsAxyhXSzpL5Q1CRKpVv0KQsnAIcSEVYM= +github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.0.4 h1:gzbtLsZC3Ic5PptoRG+kQj4L60qjK7H7XszrU163JNQ= +github.com/sirupsen/logrus v1.0.4/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/spf13/afero v1.0.2 h1:5bRmqmInNmNFkI9NG9O0Xc/Lgl9wOWWUUA/O8XZqTCo= +github.com/spf13/afero v1.0.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.2.0 h1:HHl1DSRbEQN2i8tJmtS6ViPyHx35+p51amrdsiTCrkg= +github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= +github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec h1:2ZXvIUGghLpdTVHR1UfvfrzoVlZaE/yOWC5LueIHZig= +github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.0 h1:oaPbdDe/x0UncahuwiPxW1GYJyilRAdsPnq3e1yaPcI= +github.com/spf13/pflag v1.0.0/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.0.0 h1:RUA/ghS2i64rlnn4ydTfblY8Og8QzcPtCcHvgMn+w/I= +github.com/spf13/viper v1.0.0/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v0.0.0-20170224212429-dcecefd839c4 h1:gKMu1Bf6QINDnvyZuTaACm9ofY+PRh+5vFz4oxBZeF8= +github.com/valyala/fasttemplate v0.0.0-20170224212429-dcecefd839c4/go.mod h1:50wTf68f99/Zt14pr046Tgt3Lp2vLyFZKzbFXTOabXw= +golang.org/x/crypto v0.0.0-20180228161326-91a49db82a88 h1:jLkAo/qlT9whgCLYC5GAJ9kcKrv3Wj8VCc4N+KJ4wpw= +golang.org/x/crypto v0.0.0-20180228161326-91a49db82a88/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc h1:a3CU5tJYVj92DY2LaA1kUkrsqD5/3mLDhx2NcNqyW+0= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/sys v0.0.0-20180302081741-dd2ff4accc09 h1:wNPZbZUOH0tyqngVRXeF2iQm19+ssqyebJTCFBvxsow= +golang.org/x/sys v0.0.0-20180302081741-dd2ff4accc09/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo= +gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 h1:OAj3g0cR6Dx/R07QgQe8wkA9RNjB2u4i700xBkIT4e0= +gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= +gopkg.in/yaml.v2 v2.1.1 h1:fxK3tv8mQPVEgxu/S2LJ040LyqiajHt+syP0CdDS/Sc= +gopkg.in/yaml.v2 v2.1.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/main.go b/main.go index 7af8c20..ad5ece1 100644 --- a/main.go +++ b/main.go @@ -2,12 +2,11 @@ package main import ( "fmt" - "net/http" - "path" + "strings" - log "github.com/Sirupsen/logrus" - "github.com/labstack/echo" - "github.com/labstack/echo/middleware" + "github.com/aws/aws-sdk-go-v2/aws/external" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" + log "github.com/sirupsen/logrus" "github.com/spf13/pflag" "github.com/spf13/viper" @@ -16,31 +15,45 @@ import ( func main() { pflag.IntP("port", "p", 8080, "the port to listen on") - pflag.String("staticdir", "./server/static", "the static files to serve") + pflag.String("staticdir", "", "this argument is ignored and will be removed") pflag.String("ga", "", "a google analytics account") - pflag.String("db", "./gggtracker.db", "the database path") + pflag.String("db", "./gggtracker.db", "the database file path") + pflag.String("dynamodb-table", "", "if given, DynamoDB will be used instead of a database file") pflag.String("forumsession", "", "the POESESSID cookie for a forum session") + pflag.String("reddit-auth", "", "the APPLICATION:SECRET to use as Reddit auth") viper.BindPFlags(pflag.CommandLine) pflag.Parse() viper.SetEnvPrefix("gggtracker") + viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) viper.AutomaticEnv() - e := echo.New() - - db, err := server.OpenDatabase(viper.GetString("db")) + var db server.Database + var err error + if tableName := viper.GetString("dynamodb-table"); tableName != "" { + config, err := external.LoadDefaultAWSConfig() + if err != nil { + log.Fatal(err) + } + db, err = server.NewDynamoDBDatabase(dynamodb.New(config), tableName) + } else { + db, err = server.NewBoltDatabase(viper.GetString("db")) + } if err != nil { log.Fatal(err) } defer db.Close() - redditIndexer, err := server.NewRedditIndexer(server.RedditIndexerConfiguration{ - Database: db, - }) - if err != nil { - log.Fatal(err) + if viper.GetString("reddit-auth") != "" { + redditIndexer, err := server.NewRedditIndexer(server.RedditIndexerConfiguration{ + Database: db, + Auth: viper.GetString("reddit-auth"), + }) + if err != nil { + log.Fatal(err) + } + defer redditIndexer.Close() } - defer redditIndexer.Close() if viper.GetString("forumsession") != "" { forumIndexer, err := server.NewForumIndexer(server.ForumIndexerConfiguration{ @@ -53,18 +66,6 @@ func main() { defer forumIndexer.Close() } - e.Use(middleware.Recover()) - - e.GET("/", server.IndexHandler(server.IndexConfiguration{ - GoogleAnalytics: viper.GetString("ga"), - })) - e.GET("/activity.json", server.ActivityHandler(db)) - e.GET("/rss", server.RSSHandler(db)) - e.GET("/rss.php", func(c echo.Context) error { - return c.Redirect(http.StatusMovedPermanently, server.AbsoluteURL(c, "/rss")) - }) - e.File("/favicon.ico", path.Join(viper.GetString("staticdir"), "favicon.ico")) - e.Static("/static", viper.GetString("staticdir")) - + e := server.New(db, viper.GetString("ga")) log.Fatal(e.Start(fmt.Sprintf(":%v", viper.GetInt("port")))) } diff --git a/server/activity_handler.go b/server/activity_handler.go index 6530cc1..6573a85 100644 --- a/server/activity_handler.go +++ b/server/activity_handler.go @@ -14,9 +14,24 @@ type jsonResponse struct { Next string `json:"next"` } -func ActivityHandler(db *Database) echo.HandlerFunc { +func ActivityHandler(db Database) echo.HandlerFunc { return func(c echo.Context) error { - activity, next := db.Activity(c.QueryParam("next"), 50) + locale := LocaleForRequest(c.Request()) + var filter func(Activity) bool + if c.QueryParams().Has("nohelp") && c.QueryParam("nohelp") != "false" { + filter = func(a Activity) bool { + if fp, ok := a.(*ForumPost); ok { + if fp.ForumId == locale.HelpForumId { + return false + } + } + return true + } + } + activity, next, err := db.Activity(locale, c.QueryParam("next"), 50, filter) + if err != nil { + return err + } response := jsonResponse{ Next: next, } @@ -35,6 +50,7 @@ func ActivityHandler(db *Database) echo.HandlerFunc { Data: a, }) } + c.Response().Header().Add("Cache-Control", "max-age=120") return c.JSON(200, response) } } diff --git a/server/bolt_database.go b/server/bolt_database.go new file mode 100644 index 0000000..5ded537 --- /dev/null +++ b/server/bolt_database.go @@ -0,0 +1,84 @@ +package server + +import ( + "encoding/base64" + + "github.com/boltdb/bolt" +) + +type BoltDatabase struct { + db *bolt.DB +} + +func NewBoltDatabase(path string) (*BoltDatabase, error) { + db, err := bolt.Open(path, 0600, nil) + if err != nil { + return nil, err + } + + db.Update(func(tx *bolt.Tx) error { + _, err := tx.CreateBucketIfNotExists([]byte("activity")) + if err != nil { + return err + } + return nil + }) + + return &BoltDatabase{ + db: db, + }, nil +} + +func (db *BoltDatabase) AddActivity(activity []Activity) error { + return db.db.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("activity")) + for _, a := range activity { + k, v, err := marshalActivity(a) + if err != nil { + return err + } + b.Put(k, v) + } + return nil + }) +} + +func (db *BoltDatabase) Activity(locale *Locale, start string, count int, filter func(a Activity) bool) ([]Activity, string, error) { + ret := []Activity(nil) + next := "" + if err := db.db.View(func(tx *bolt.Tx) error { + c := tx.Bucket([]byte("activity")).Cursor() + var k, v []byte + if start == "" { + k, v = c.Last() + } else { + s, err := base64.RawURLEncoding.DecodeString(start) + if err != nil { + k, v = c.Last() + } else { + k, v = c.Seek(s) + if k != nil { + k, v = c.Prev() + } + } + } + for len(ret) < count && k != nil { + activity, err := unmarshalActivity(k, v) + if err != nil { + return err + } else if activity != nil && locale.ActivityFilter(activity) && (filter == nil || filter(activity)) { + ret = append(ret, activity) + next = base64.RawURLEncoding.EncodeToString(k) + } + k, v = c.Prev() + } + return nil + }); err != nil { + return nil, "", err + } + return ret, next, nil +} + +func (db *BoltDatabase) Close() error { + return db.db.Close() +} diff --git a/server/bolt_database_test.go b/server/bolt_database_test.go new file mode 100644 index 0000000..9e1c361 --- /dev/null +++ b/server/bolt_database_test.go @@ -0,0 +1,22 @@ +package server + +import ( + "io/ioutil" + "os" + "path" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBoltDatabase(t *testing.T) { + dir, err := ioutil.TempDir("testdata", "db") + require.NoError(t, err) + defer os.RemoveAll(dir) + + db, err := NewBoltDatabase(path.Join(dir, "test.db")) + require.NoError(t, err) + defer db.Close() + + testDatabase(t, db) +} diff --git a/server/common.go b/server/common.go index 459666c..048b4cc 100644 --- a/server/common.go +++ b/server/common.go @@ -1,6 +1,8 @@ package server import ( + "strings" + "github.com/labstack/echo" ) @@ -14,3 +16,17 @@ func AbsoluteURL(c echo.Context, resource string) string { } return scheme + "://" + c.Request().Host + resource } + +func SubdomainURL(c echo.Context, subdomain string) string { + scheme := c.Scheme() + if c.Request().Header.Get(echo.HeaderXForwardedProto) == "https" { + scheme = "https" + } + locale := LocaleForRequest(c.Request()) + host := strings.TrimPrefix(c.Request().Host, "www.") + host = strings.TrimPrefix(host, locale.Subdomain+".") + if subdomain != "" { + host = subdomain + "." + host + } + return scheme + "://" + host +} diff --git a/server/database.go b/server/database.go index 00a4891..8118b45 100644 --- a/server/database.go +++ b/server/database.go @@ -1,124 +1,88 @@ package server import ( - "encoding/base64" + "bytes" + "compress/gzip" "encoding/binary" - "encoding/json" + "io/ioutil" - "github.com/boltdb/bolt" + json "github.com/json-iterator/go" ) -type Database struct { - db *bolt.DB -} - -func OpenDatabase(path string) (*Database, error) { - db, err := bolt.Open(path, 0600, nil) - if err != nil { - return nil, err - } - - db.Update(func(tx *bolt.Tx) error { - _, err := tx.CreateBucketIfNotExists([]byte("activity")) - if err != nil { - return err - } - return nil - }) - - return &Database{ - db: db, - }, nil -} - -func (db *Database) Close() { - db.db.Close() -} - const ( ForumPostType = iota RedditCommentType RedditPostType ) -func (db *Database) AddActivity(activity []Activity) { - err := db.db.Update(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte("activity")) - for _, a := range activity { - buf, err := json.Marshal(a) - if err != nil { - return err - } - k := make([]byte, 10) - binary.BigEndian.PutUint64(k, uint64(a.ActivityTime().Unix())<<24) - switch a.(type) { - case *ForumPost: - k[5] = ForumPostType - case *RedditComment: - k[5] = RedditCommentType - case *RedditPost: - k[5] = RedditPostType - } - binary.BigEndian.PutUint32(k[6:], a.ActivityKey()) - b.Put(k, buf) - } - return nil - }) - if err != nil { - panic(err) +type Database interface { + AddActivity(activity []Activity) error + Activity(locale *Locale, start string, count int, filter func(a Activity) bool) ([]Activity, string, error) + Close() error +} + +const gzipMarker = 0 + +func marshalActivity(a Activity) (key, value []byte, err error) { + buf := &bytes.Buffer{} + buf.Write([]byte{gzipMarker}) + w := gzip.NewWriter(buf) + if err := json.NewEncoder(w).Encode(a); err != nil { + return nil, nil, err + } + w.Close() + k := make([]byte, 10) + binary.BigEndian.PutUint64(k, uint64(a.ActivityTime().Unix())<<24) + switch a.(type) { + case *ForumPost: + k[5] = ForumPostType + case *RedditComment: + k[5] = RedditCommentType + case *RedditPost: + k[5] = RedditPostType } + binary.BigEndian.PutUint32(k[6:], a.ActivityKey()) + return k, buf.Bytes(), nil } -func (db *Database) Activity(start string, count int) ([]Activity, string) { - ret := []Activity(nil) - next := "" - err := db.db.View(func(tx *bolt.Tx) error { - c := tx.Bucket([]byte("activity")).Cursor() - var k, v []byte - if start == "" { - k, v = c.Last() - } else { - s, err := base64.RawURLEncoding.DecodeString(start) - if err != nil { - k, v = c.Last() - } else { - k, v = c.Seek(s) - if k != nil { - k, v = c.Prev() - } - } +func unmarshalActivity(key, value []byte) (Activity, error) { + if len(value) > 0 && value[0] == gzipMarker { + r, err := gzip.NewReader(bytes.NewReader(value[1:])) + if err != nil { + return nil, err } - for i := 0; i < count && k != nil; i++ { - switch k[5] { - case ForumPostType: - post := &ForumPost{} - err := json.Unmarshal(v, post) - if err != nil { - return err - } - ret = append(ret, post) - case RedditCommentType: - comment := &RedditComment{} - err := json.Unmarshal(v, comment) - if err != nil { - return err - } - ret = append(ret, comment) - case RedditPostType: - post := &RedditPost{} - err := json.Unmarshal(v, post) - if err != nil { - return err - } - ret = append(ret, post) - } - next = base64.RawURLEncoding.EncodeToString(k) - k, v = c.Prev() + defer r.Close() + buf, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + value = buf + } + + switch key[5] { + case ForumPostType: + post := &ForumPost{} + err := json.Unmarshal(value, post) + if err != nil { + return nil, err + } + if post.Id != 0 { + return post, nil + } + case RedditCommentType: + comment := &RedditComment{} + err := json.Unmarshal(value, comment) + if err != nil { + return nil, err + } + return comment, nil + case RedditPostType: + post := &RedditPost{} + err := json.Unmarshal(value, post) + if err != nil { + return nil, err } - return nil - }) - if err != nil { - panic(err) + return post, nil } - return ret, next + return nil, nil } diff --git a/server/database_test.go b/server/database_test.go index 55e7847..4193c8b 100644 --- a/server/database_test.go +++ b/server/database_test.go @@ -1,9 +1,6 @@ package server import ( - "io/ioutil" - "os" - "path" "testing" "time" @@ -11,14 +8,14 @@ import ( "github.com/stretchr/testify/require" ) -func TestDatabase_ForumPosts(t *testing.T) { - dir, err := ioutil.TempDir("testdata", "db") - require.NoError(t, err) - defer os.RemoveAll(dir) +func testDatabase(t *testing.T, db Database) { + t.Run("ForumPosts", func(t *testing.T) { + testDatabase_ForumPosts(t, db) + }) +} - db, err := OpenDatabase(path.Join(dir, "test.db")) - require.NoError(t, err) - defer db.Close() +func testDatabase_ForumPosts(t *testing.T, db Database) { + locale := Locales[0] post1 := &ForumPost{ Id: 9000, @@ -34,18 +31,23 @@ func TestDatabase_ForumPosts(t *testing.T) { db.AddActivity([]Activity{post1, post2}) - posts, next := db.Activity("", 1) + all := func(a Activity) bool { return true } + + posts, next, err := db.Activity(locale, "", 1, all) + require.NoError(t, err) require.Equal(t, 1, len(posts)) assert.Equal(t, post1.Id, posts[0].(*ForumPost).Id) assert.Equal(t, post1.Poster, posts[0].(*ForumPost).Poster) assert.Equal(t, post1.Time.Unix(), posts[0].(*ForumPost).Time.Unix()) - posts, next = db.Activity(next, 1) + posts, next, err = db.Activity(locale, next, 1, all) + require.NoError(t, err) require.Equal(t, 1, len(posts)) assert.Equal(t, post2.Id, posts[0].(*ForumPost).Id) assert.Equal(t, post2.Poster, posts[0].(*ForumPost).Poster) assert.Equal(t, post2.Time.Unix(), posts[0].(*ForumPost).Time.Unix()) - posts, _ = db.Activity(next, 1) + posts, _, err = db.Activity(locale, next, 1, all) + require.NoError(t, err) require.Equal(t, 0, len(posts)) } diff --git a/server/dynamodb_database.go b/server/dynamodb_database.go new file mode 100644 index 0000000..cf7bdbc --- /dev/null +++ b/server/dynamodb_database.go @@ -0,0 +1,156 @@ +package server + +import ( + "context" + "encoding/base64" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" +) + +type DynamoDBDatabase struct { + client *dynamodb.Client + tableName string +} + +func NewDynamoDBDatabase(client *dynamodb.Client, tableName string) (*DynamoDBDatabase, error) { + return &DynamoDBDatabase{ + client: client, + tableName: tableName, + }, nil +} + +func dynamoDBActivityHashKey(locale *Locale) []byte { + return []byte("activity_by_locale:" + locale.Subdomain) +} + +func (db *DynamoDBDatabase) AddActivity(activity []Activity) error { + for _, locale := range Locales { + var remaining []Activity + for _, a := range activity { + if locale.ActivityFilter(a) { + remaining = append(remaining, a) + } + } + + for len(remaining) > 0 { + batch := remaining + const maxBatchSize = 25 + if len(batch) > maxBatchSize { + batch = batch[:maxBatchSize] + } + + writeRequests := make([]dynamodb.WriteRequest, len(batch)) + for i, a := range batch { + k, v, err := marshalActivity(a) + if err != nil { + return err + } + writeRequests[i] = dynamodb.WriteRequest{ + PutRequest: &dynamodb.PutRequest{ + Item: map[string]dynamodb.AttributeValue{ + "hk": dynamodb.AttributeValue{ + B: dynamoDBActivityHashKey(locale), + }, + "rk": dynamodb.AttributeValue{ + B: []byte(k), + }, + "v": dynamodb.AttributeValue{ + B: []byte(v), + }, + }, + }, + } + } + unprocessed := map[string][]dynamodb.WriteRequest{ + db.tableName: writeRequests, + } + + for len(unprocessed) > 0 { + result, err := db.client.BatchWriteItemRequest(&dynamodb.BatchWriteItemInput{ + RequestItems: unprocessed, + }).Send(context.Background()) + if err != nil { + return err + } + unprocessed = result.UnprocessedItems + } + + remaining = remaining[len(batch):] + } + } + return nil +} + +func (db *DynamoDBDatabase) Activity(locale *Locale, start string, count int, filter func(a Activity) bool) ([]Activity, string, error) { + var activity []Activity + + var startKey map[string]dynamodb.AttributeValue + if start != "" { + rk, _ := base64.RawURLEncoding.DecodeString(start) + startKey = map[string]dynamodb.AttributeValue{ + "hk": dynamodb.AttributeValue{ + B: dynamoDBActivityHashKey(locale), + }, + "rk": dynamodb.AttributeValue{ + B: rk, + }, + } + } + + condition := "hk = :hash" + attributeValues := map[string]dynamodb.AttributeValue{ + ":hash": dynamodb.AttributeValue{ + B: dynamoDBActivityHashKey(locale), + }, + } + + for len(activity) < count { + batchSize := count - len(activity) + if filter != nil { + // if we're filtering results, fetch extra + batchSize = count * 4 + if batchSize < 50 { + batchSize = 50 + } else if batchSize > 1000 { + batchSize = 1000 + } + } + result, err := db.client.QueryRequest(&dynamodb.QueryInput{ + TableName: aws.String(db.tableName), + KeyConditionExpression: aws.String(condition), + ExpressionAttributeValues: attributeValues, + ExclusiveStartKey: startKey, + Limit: aws.Int64(int64(batchSize)), + ScanIndexForward: aws.Bool(false), + }).Send(context.Background()) + if err != nil { + return nil, "", err + } + startKey = result.LastEvaluatedKey + for _, item := range result.Items { + if a, err := unmarshalActivity(item["rk"].B, item["v"].B); err != nil { + return nil, "", err + } else if a != nil && (filter == nil || filter(a)) { + activity = append(activity, a) + if len(activity) == count { + startKey = item + break + } + } + } + if result.LastEvaluatedKey == nil || len(activity) == count { + break + } + } + + var next string + if startKey != nil { + next = base64.RawURLEncoding.EncodeToString(startKey["rk"].B) + } + return activity, next, nil +} + +func (db *DynamoDBDatabase) Close() error { + return nil +} diff --git a/server/dynamodb_database_test.go b/server/dynamodb_database_test.go new file mode 100644 index 0000000..e448f0b --- /dev/null +++ b/server/dynamodb_database_test.go @@ -0,0 +1,111 @@ +package server + +import ( + "context" + "crypto/rand" + "encoding/base64" + "os" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/awserr" + "github.com/aws/aws-sdk-go-v2/aws/defaults" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" + "github.com/stretchr/testify/require" +) + +func newDynamoDBTestClient() (*dynamodb.Client, error) { + endpoint := os.Getenv("DYNAMODB_ENDPOINT") + + config := defaults.Config() + config.Region = "us-east-1" + config.EndpointResolver = aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) { + if endpoint != "" { + return aws.Endpoint{ + URL: endpoint, + }, nil + } + return aws.Endpoint{ + URL: "http://localhost:8000", + }, nil + }) + + credentialsBuf := make([]byte, 20) + if _, err := rand.Read(credentialsBuf); err != nil { + return nil, err + } + credentials := base64.RawURLEncoding.EncodeToString(credentialsBuf) + config.Credentials = aws.NewStaticCredentialsProvider(credentials, credentials, "") + config.Retryer = aws.DefaultRetryer{ + NumMaxRetries: 0, + } + + client := dynamodb.New(config) + if endpoint == "" { + if _, err := client.ListTablesRequest(&dynamodb.ListTablesInput{}).Send(context.Background()); err != nil { + if err, ok := err.(awserr.Error); ok && err.Code() == "RequestError" { + return nil, nil + } + } + } + return client, nil +} + +func createDynamoDBTable(client *dynamodb.Client, tableName string) error { + if _, err := client.CreateTableRequest(&dynamodb.CreateTableInput{ + AttributeDefinitions: []dynamodb.AttributeDefinition{ + { + AttributeName: aws.String("hk"), + AttributeType: dynamodb.ScalarAttributeTypeB, + }, { + AttributeName: aws.String("rk"), + AttributeType: dynamodb.ScalarAttributeTypeB, + }, + }, + KeySchema: []dynamodb.KeySchemaElement{ + { + AttributeName: aws.String("hk"), + KeyType: dynamodb.KeyTypeHash, + }, { + AttributeName: aws.String("rk"), + KeyType: dynamodb.KeyTypeRange, + }, + }, + ProvisionedThroughput: &dynamodb.ProvisionedThroughput{ + ReadCapacityUnits: aws.Int64(25), + WriteCapacityUnits: aws.Int64(25), + }, + TableName: &tableName, + }).Send(context.Background()); err != nil { + return err + } + return client.WaitUntilTableExists(context.Background(), &dynamodb.DescribeTableInput{ + TableName: aws.String(tableName), + }) +} + +func TestDynamoDBDatabase(t *testing.T) { + client, err := newDynamoDBTestClient() + require.NoError(t, err) + if client == nil { + t.Skip("launch a local dynamodb container to run this test: docker run --rm -it -p 8000:8000 dwmkerr/dynamodb -inMemory") + } + + const tableName = "TestDynamoDBDatabase" + + if _, err := client.DeleteTableRequest(&dynamodb.DeleteTableInput{ + TableName: aws.String(tableName), + }).Send(context.Background()); err == nil { + require.NoError(t, client.WaitUntilTableNotExists(context.Background(), &dynamodb.DescribeTableInput{ + TableName: aws.String(tableName), + })) + } + + require.NoError(t, createDynamoDBTable(client, tableName)) + + db, err := NewDynamoDBDatabase(client, tableName) + require.NoError(t, err) + defer db.Close() + + testDatabase(t, db) +} diff --git a/server/forum_indexer.go b/server/forum_indexer.go index ede70ad..915b049 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -13,72 +13,191 @@ import ( "github.com/PuerkitoBio/goquery" - log "github.com/Sirupsen/logrus" + log "github.com/sirupsen/logrus" ) +const UserAgent = "gggtracker.com (github.com/ccbrown/gggtracker)" + type ForumIndexer struct { configuration ForumIndexerConfiguration - closeSignal chan bool + closeSignal chan struct{} } type ForumIndexerConfiguration struct { - Database *Database + Database Database Session string } func NewForumIndexer(configuration ForumIndexerConfiguration) (*ForumIndexer, error) { ret := &ForumIndexer{ configuration: configuration, - closeSignal: make(chan bool), + closeSignal: make(chan struct{}), } go ret.run() return ret, nil } func (indexer *ForumIndexer) Close() { - indexer.closeSignal <- true + close(indexer.closeSignal) +} + +type ForumAccount struct { + Username string + Discriminator int } func (indexer *ForumIndexer) run() { log.Info("starting forum indexer") - posters := []string{ - "Chris", "Jonathan", "Erik", "Mark_GGG", "Samantha", "Rory", "Rhys", "Qarl", "Andrew_GGG", - "Damien_GGG", "Joel_GGG", "Ari", "Thomas", "BrianWeissman", "Edwin_GGG", "Support", "Dylan", - "MaxS", "Ammon_GGG", "Jess_GGG", "Robbie_GGG", "GGG_Neon", "Jason_GGG", "Henry_GGG", - "Michael_GGG", "Bex_GGG", "Cagan_GGG", "Daniel_GGG", "Kieren_GGG", "Yeran_GGG", "Gary_GGG", - "Dan_GGG", "Jared_GGG", "Brian_GGG", "RobbieL_GGG", "Arthur_GGG", "NickK_GGG", "Felipe_GGG", - "Alex_GGG", "Alexcc_GGG", "Andy", "CJ_GGG", "Eben_GGG", "Emma_GGG", "Ethan_GGG", - "Fitzy_GGG", "Hartlin_GGG", "Jake_GGG", "Lionel_GGG", "Melissa_GGG", "MikeP_GGG", "Novynn", - "Rachel_GGG", "Rob_GGG", "Roman_GGG", "Sarah_GGG", "SarahB_GGG", "Tom_GGG", "Natalia_GGG", + accounts := []ForumAccount{ + {Username: "Chris"}, + {Username: "Jonathan"}, + {Username: "Mark_GGG"}, + {Username: "Rory"}, + {Username: "Rhys"}, + {Username: "Joel_GGG"}, + {Username: "Ari"}, + {Username: "Thomas"}, + {Username: "Support"}, + {Username: "Jess_GGG"}, + {Username: "Robbie_GGG"}, + {Username: "GGG_Neon"}, + {Username: "Jason_GGG"}, + {Username: "Henry_GGG"}, + {Username: "Michael_GGG"}, + {Username: "Bex_GGG"}, + {Username: "Cagan_GGG"}, + {Username: "Kieren_GGG"}, + {Username: "Yeran_GGG"}, + {Username: "Gary_GGG"}, + {Username: "Dan_GGG"}, + {Username: "Jared_GGG"}, + {Username: "Brian_GGG"}, + {Username: "RobbieL_GGG"}, + {Username: "Arthur_GGG"}, + {Username: "NickK_GGG"}, + {Username: "Felipe_GGG"}, + {Username: "Alex_GGG"}, + {Username: "Alexcc_GGG"}, + {Username: "CJ_GGG"}, + {Username: "Eben_GGG"}, + {Username: "Emma_GGG"}, + {Username: "Ethan_GGG"}, + {Username: "Fitzy_GGG"}, + {Username: "Hartlin_GGG"}, + {Username: "Jake_GGG"}, + {Username: "Melissa_GGG"}, + {Username: "MikeP_GGG"}, + {Username: "Novynn"}, + {Username: "Rob_GGG"}, + {Username: "Roman_GGG"}, + {Username: "Tom_GGG"}, + {Username: "Natalia_GGG"}, + {Username: "Jeff_GGG"}, + {Username: "Lu_GGG"}, + {Username: "JuliaS_GGG"}, + {Username: "Alexander_GGG"}, + {Username: "SamC_GGG"}, + {Username: "AndrewE_GGG"}, + {Username: "Kyle_GGG"}, + {Username: "Stacey_GGG"}, + {Username: "Jatin_GGG"}, + {Username: "Community_Team"}, + {Username: "Nick_GGG"}, + {Username: "Guy_GGG"}, + {Username: "Ben_GGG"}, + {Username: "BenH_GGG"}, + {Username: "Nav_GGG"}, + {Username: "Will_GGG"}, + {Username: "Scott_GGG"}, + {Username: "JC_GGG"}, + {Username: "Dylan_GGG"}, + {Username: "Chulainn_GGG"}, + {Username: "Vash_GGG"}, + {Username: "Cameron_GGG"}, + {Username: "Jacob_GGG"}, + {Username: "Jenn_GGG"}, + {Username: "CoryA_GGG"}, + {Username: "Sian_GGG"}, + {Username: "Drew_GGG"}, + {Username: "Lisa_GGG"}, + {Username: "ThomasK_GGG"}, + {Username: "Whai_GGG"}, + {Username: "Scopey"}, + {Username: "Adam_GGG"}, + {Username: "Nichelle_GGG"}, + {Username: "Markus_GGG"}, + {Username: "Jarod_GGG"}, + {Username: "Joel_GGG", Discriminator: 1496}, + {Username: "Vinky_GGG"}, + {Username: "Edmund_GGG", Discriminator: 4844}, + {Username: "Clint"}, + {Username: "LeightonJ_GGG"}, + {Username: "Tai_GGG"}, + {Username: "ShaunB_GGG"}, + {Username: "Ayelen_GGG"}, + {Username: "Timothy_GGG"}, + {Username: "BenMH_GGG"}, + {Username: "Ian_GGG"}, + {Username: "EthanH_GGG"}, + {Username: "Yrone_GGG"}, + {Username: "Sam_GGG", Discriminator: 2420}, + {Username: "Ting_GGG", Discriminator: 2357}, + {Username: "Ramon_GGG"}, + {Username: "Sameer_GGG"}, + {Username: "TobyM_GGG"}, + {Username: "Lachlan_GGG"}, + {Username: "Ringatu_GGG"}, } - next := 0 timezone := (*time.Location)(nil) - for { + for timezone == nil { select { case <-indexer.closeSignal: return default: - if timezone == nil { - tz, err := indexer.sessionTimezone() - if err != nil { - log.WithError(err).Error("error getting forum timezone") + if tz, err := indexer.sessionTimezone(); err != nil { + log.WithError(err).Error("error getting forum timezone") + } else { + timezone = tz + log.WithField("timezone", timezone).Info("forum timezone obtained") + break + } + time.Sleep(time.Second) + } + } + + for { + for _, locale := range Locales { + select { + case <-indexer.closeSignal: + return + default: + logger := log.WithField("host", locale.ForumHost()) + if err := locale.RefreshForumIds(); err != nil { + logger.WithError(err).Error("error refreshing forum ids") } else { - timezone = tz - log.WithFields(log.Fields{ - "timezone": timezone, - }).Info("forum timezone obtained") + logger.Info("refreshed forum ids") } - } else { - indexer.index(posters[next], timezone) - next += 1 - if next >= len(posters) { - next = 0 + time.Sleep(time.Second) + } + } + for _, account := range accounts { + select { + case <-indexer.closeSignal: + return + default: + if err := indexer.index(account, timezone); err != nil { + if errors.Is(err, ErrForumMaintenance) { + log.Info("forum is under maintenance") + time.Sleep(30 * time.Second) + } else { + log.WithError(err).Error("error indexing forum account: " + account.Username) + } } + time.Sleep(time.Second) } - time.Sleep(time.Second) } } } @@ -99,7 +218,14 @@ func (indexer *ForumIndexer) requestDocument(resource string) (*goquery.Document Jar: jar, Timeout: time.Second * 10, } - resp, err := client.Get(urlString) + + req, err := http.NewRequest("GET", urlString, nil) + req.Header.Set("User-Agent", UserAgent) + if err != nil { + return nil, err + } + + resp, err := client.Do(req) if err != nil { return nil, err } @@ -108,18 +234,29 @@ func (indexer *ForumIndexer) requestDocument(resource string) (*goquery.Document return goquery.NewDocumentFromReader(resp.Body) } -var postURLExpression = regexp.MustCompile("^/forum/view-thread/([0-9]+)/page/([0-9]+)#p([0-9]+)") +var postURLExpression = regexp.MustCompile("^/forum/view-post/([0-9]+)") var threadURLExpression = regexp.MustCompile("^/forum/view-thread/([0-9]+)") var forumURLExpression = regexp.MustCompile("^/forum/view-forum/([0-9]+)") -func ScrapeForumPosts(doc *goquery.Document, timezone *time.Location) ([]*ForumPost, error) { +var ErrForumMaintenance = errors.New("forum is in maintenance") + +func ScrapeForumPosts(doc *goquery.Document, poster ForumAccount, timezone *time.Location) ([]*ForumPost, error) { posts := []*ForumPost(nil) err := error(nil) - doc.Find(".forumPostListTable tr").EachWithBreak(func(i int, sel *goquery.Selection) bool { + if doc.Find(".forumPostListTable").Length() == 0 { + err = errors.New("forum post list not found") + if topBar := doc.Find(".topBar"); topBar.Length() == 1 && topBar.Text() == "Down For Maintenance" { + err = ErrForumMaintenance + } + return nil, err + } + + doc.Find(".forumPostListTable > tbody > tr").EachWithBreak(func(i int, sel *goquery.Selection) bool { post := &ForumPost{ - Poster: sel.Find(".post_by_account").Text(), + Poster: poster.Username, + PosterDiscriminator: poster.Discriminator, } body, err := sel.Find(".content").Html() @@ -128,23 +265,21 @@ func ScrapeForumPosts(doc *goquery.Document, timezone *time.Location) ([]*ForumP } post.BodyHTML = body - text := sel.Find(".post_date").Text() - t, err := time.ParseInLocation("Jan _2, 2006 3:04:05 PM", text, timezone) - if err != nil { + timeText := sel.Find(".post_date").Text() + + if post.Time, err = time.ParseInLocation("Jan _2, 2006, 3:04:05 PM", timeText, timezone); err != nil { + log.WithField("text", timeText).Error("unable to parse time") return false } - post.Time = t sel.Find("a").Each(func(i int, sel *goquery.Selection) { href := sel.AttrOr("href", "") if match := postURLExpression.FindStringSubmatch(href); match != nil { n, _ := strconv.Atoi(match[1]) - post.ThreadId = n - n, _ = strconv.Atoi(match[2]) - post.PageNumber = n - n, _ = strconv.Atoi(match[3]) post.Id = n } else if match := threadURLExpression.FindStringSubmatch(href); match != nil { + n, _ := strconv.Atoi(match[1]) + post.ThreadId = n post.ThreadTitle = sel.Text() } else if match := forumURLExpression.FindStringSubmatch(href); match != nil { n, _ := strconv.Atoi(match[1]) @@ -164,39 +299,45 @@ func ScrapeForumPosts(doc *goquery.Document, timezone *time.Location) ([]*ForumP return posts, nil } -func (indexer *ForumIndexer) forumPosts(poster string, page int, timezone *time.Location) ([]*ForumPost, error) { - doc, err := indexer.requestDocument(fmt.Sprintf("/account/view-posts/%v/page/%v", poster, page)) +func (indexer *ForumIndexer) forumPosts(poster ForumAccount, page int, timezone *time.Location) ([]*ForumPost, error) { + doc, err := indexer.requestDocument(fmt.Sprintf("/account/view-posts/%v-%04d/page/%v", poster.Username, poster.Discriminator, page)) if err != nil { return nil, err } - return ScrapeForumPosts(doc, timezone) + posts, err := ScrapeForumPosts(doc, poster, timezone) + if err != nil { + return nil, err + } + return posts, nil } -func (indexer *ForumIndexer) index(poster string, timezone *time.Location) { +func (indexer *ForumIndexer) index(poster ForumAccount, timezone *time.Location) error { logger := log.WithFields(log.Fields{ "poster": poster, }) - cutoff := time.Now().Add(time.Hour * -12) + pageCutoff := time.Now().Add(-12 * time.Hour) + cutoff := time.Now().Add(-14 * 24 * time.Hour) activity := []Activity(nil) for page := 1; ; page++ { posts, err := indexer.forumPosts(poster, page, timezone) if err != nil { - logger.WithError(err).Error("error requesting forum posts") + return fmt.Errorf("error getting forum posts: %w", err) } done := len(posts) == 0 for _, post := range posts { - if post.Time.Before(cutoff) { + if post.Time.Before(pageCutoff) { done = true } + if post.Time.Before(cutoff) { + break + } activity = append(activity, post) } - logger.WithFields(log.Fields{ - "count": len(posts), - }).Info("received forum posts") + logger.WithField("count", len(activity)).Info("received forum posts") if done { break @@ -204,13 +345,15 @@ func (indexer *ForumIndexer) index(poster string, timezone *time.Location) { time.Sleep(time.Second) } - if len(activity) > 0 { - indexer.configuration.Database.AddActivity(activity) + if len(activity) == 0 { + return nil } + + return indexer.configuration.Database.AddActivity(activity) } func ScrapeForumTimezone(doc *goquery.Document) (*time.Location, error) { - sel := doc.Find("#timezone option[selected]") + sel := doc.Find(`select[name="preferences[timezone]"] option[selected]`) if sel == nil || sel.AttrOr("value", "") == "" { return nil, errors.New("unable to find timezone selection") } diff --git a/server/forum_indexer_test.go b/server/forum_indexer_test.go index f1600e5..4e6b756 100644 --- a/server/forum_indexer_test.go +++ b/server/forum_indexer_test.go @@ -21,7 +21,8 @@ func TestScrapeForumPosts(t *testing.T) { tz, err := time.LoadLocation("America/Los_Angeles") require.NoError(t, err) - posts, err := ScrapeForumPosts(doc, tz) + poster := ForumAccount{Username: "Chris"} + posts, err := ScrapeForumPosts(doc, poster, tz) require.NoError(t, err) assert.Equal(t, 10, len(posts)) @@ -31,11 +32,43 @@ func TestScrapeForumPosts(t *testing.T) { assert.Equal(t, 54, p.ForumId) assert.Equal(t, 1830139, p.ThreadId) assert.Equal(t, "Chris", p.Poster) + assert.Equal(t, 0, p.PosterDiscriminator) assert.Equal(t, "Photos of the Fan Meetup", p.ThreadTitle) assert.Equal(t, "Announcements", p.ForumName) - assert.Equal(t, 1, p.PageNumber) assert.Equal(t, "we had a great time too!", p.BodyHTML) assert.Equal(t, int64(1486332365), p.Time.Unix()) + + t.Run("LoggedOut", func(t *testing.T) { + f, err := os.Open("testdata/forum-posts-logged-out.html") + require.NoError(t, err) + defer f.Close() + + doc, err := goquery.NewDocumentFromReader(f) + require.NoError(t, err) + + tz, err := time.LoadLocation("America/Los_Angeles") + require.NoError(t, err) + + posts, err := ScrapeForumPosts(doc, poster, tz) + require.Error(t, err) + assert.Equal(t, 0, len(posts)) + }) + + t.Run("Maintenance", func(t *testing.T) { + f, err := os.Open("testdata/forum-maintenance.html") + require.NoError(t, err) + defer f.Close() + + doc, err := goquery.NewDocumentFromReader(f) + require.NoError(t, err) + + tz, err := time.LoadLocation("America/Los_Angeles") + require.NoError(t, err) + + posts, err := ScrapeForumPosts(doc, poster, tz) + assert.Equal(t, err, ErrForumMaintenance) + assert.Equal(t, 0, len(posts)) + }) } func TestScrapeForumTimezone(t *testing.T) { diff --git a/server/forum_post.go b/server/forum_post.go index 09afa7e..bd9971f 100644 --- a/server/forum_post.go +++ b/server/forum_post.go @@ -1,20 +1,49 @@ package server import ( + "encoding/json" "fmt" "time" ) type ForumPost struct { - Id int `json:"id"` - BodyHTML string `json:"body_html"` - Time time.Time `json:"time"` - Poster string `json:"poster"` - ThreadId int `json:"thread_id"` - ThreadTitle string `json:"thread_title"` - PageNumber int `json:"page_number"` - ForumId int `json:"forum_id"` - ForumName string `json:"forum_name"` + Id int `json:"id"` + BodyHTML string `json:"body_html"` + Time time.Time `json:"time"` + Poster string `json:"poster"` + PosterDiscriminator int `json:"poster_discriminator"` + ThreadId int `json:"thread_id"` + ThreadTitle string `json:"thread_title"` + ForumId int `json:"forum_id"` + ForumName string `json:"forum_name"` +} + +type forumPostWithHost struct { + Id int `json:"id"` + BodyHTML string `json:"body_html"` + Time time.Time `json:"time"` + Poster string `json:"poster"` + PosterDiscriminator int `json:"poster_discriminator"` + ThreadId int `json:"thread_id"` + ThreadTitle string `json:"thread_title"` + ForumId int `json:"forum_id"` + ForumName string `json:"forum_name"` + Host string `json:"host"` +} + +func (p ForumPost) MarshalJSON() ([]byte, error) { + return json.Marshal(&forumPostWithHost{ + Id: p.Id, + BodyHTML: p.BodyHTML, + Time: p.Time, + Poster: p.Poster, + PosterDiscriminator: p.PosterDiscriminator, + ThreadId: p.ThreadId, + ThreadTitle: p.ThreadTitle, + ForumId: p.ForumId, + ForumName: p.ForumName, + Host: p.Host(), + }) } func (p *ForumPost) ActivityTime() time.Time { @@ -25,6 +54,15 @@ func (p *ForumPost) ActivityKey() uint32 { return uint32(p.Id) } +func (p *ForumPost) Host() string { + for _, l := range Locales { + if l.ForumIds()[p.ForumId] { + return l.ForumHost() + } + } + return "www.pathofexile.com" +} + func (p *ForumPost) PostURL() string { - return fmt.Sprintf("https://www.pathofexile.com/forum/view-post/%v", p.Id) + return fmt.Sprintf("https://%v/forum/view-post/%v", p.Host(), p.Id) } diff --git a/server/index_handler.go b/server/index_handler.go index 404cfb9..2a291b1 100644 --- a/server/index_handler.go +++ b/server/index_handler.go @@ -11,22 +11,19 @@ var index = ` GGG Tracker - + {{if .Configuration.GoogleAnalytics}} - + {{end}} @@ -34,19 +31,30 @@ var index = `
+
-

Activity

+

{{call $.Translate "Activity"}}

+ {{if ne $.Locale.HelpForumId 0}} + + {{end}} - - - - + + + + @@ -60,10 +68,13 @@ var index = ` Please direct feedback to this thread. Want a new feature? Add it yourself!

+

+ Appreciate the site? Show your support by sponsoring me! ❤️ +

- + ` @@ -71,16 +82,34 @@ type IndexConfiguration struct { GoogleAnalytics string } -func IndexHandler(configuration IndexConfiguration) echo.HandlerFunc { +var indexTemplate *template.Template + +func init() { t, err := template.New("index").Parse(index) if err != nil { panic(err) } + indexTemplate = t +} + +func IndexHandler(configuration IndexConfiguration) echo.HandlerFunc { return func(c echo.Context) error { - return t.Execute(c.Response(), struct { + c.Response().Header().Set("Content-Type", "text/html; charset=utf-8") + locale := LocaleForRequest(c.Request()) + return indexTemplate.Execute(c.Response(), struct { Configuration IndexConfiguration + Locales []*Locale + Locale *Locale + Translate func(string) string + SubdomainURL func(string) string }{ Configuration: configuration, + Locales: Locales, + Locale: locale, + Translate: locale.Translate, + SubdomainURL: func(subdomain string) string { + return SubdomainURL(c, subdomain) + }, }) } } diff --git a/server/localization.go b/server/localization.go new file mode 100644 index 0000000..61f5a4d --- /dev/null +++ b/server/localization.go @@ -0,0 +1,213 @@ +package server + +import ( + "fmt" + "net/http" + "strconv" + "strings" + "sync/atomic" + "time" + + "github.com/PuerkitoBio/goquery" +) + +type Locale struct { + Subdomain string + Image string + IncludeReddit bool + Translations map[string]string + ParseTime func(s string, tz *time.Location) (time.Time, error) + HelpForumId int + + forumIds atomic.Value +} + +func (l *Locale) Translate(s string) string { + if translated, ok := l.Translations[s]; ok { + return translated + } + return s +} + +func (l *Locale) ActivityFilter(a Activity) bool { + switch a := a.(type) { + case *ForumPost: + return a.Host() == l.ForumHost() + case *RedditComment: + return l.IncludeReddit + case *RedditPost: + return l.IncludeReddit + } + return false +} + +func (l *Locale) ForumHost() string { + if l.Subdomain != "" { + return l.Subdomain + ".pathofexile.com" + } + return "www.pathofexile.com" +} + +func (l *Locale) ForumIds() map[int]bool { + ret, _ := l.forumIds.Load().(map[int]bool) + return ret +} + +func (l *Locale) RefreshForumIds() error { + client := http.Client{ + Timeout: time.Second * 10, + } + + req, err := http.NewRequest("GET", fmt.Sprintf("https://%v/forum", l.ForumHost()), nil) + if err != nil { + return err + } + req.Header.Set("User-Agent", UserAgent) + + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("unexpected status code: %v", resp.StatusCode) + } + + doc, err := goquery.NewDocumentFromReader(resp.Body) + if err != nil { + return err + } + + forumIds := map[int]bool{} + doc.Find(".forumTable tbody tr").Each(func(i int, sel *goquery.Selection) { + if idStr := sel.AttrOr("data-id", ""); idStr != "" { + if id, err := strconv.Atoi(idStr); err == nil { + forumIds[id] = true + } + } + }) + l.forumIds.Store(forumIds) + + return nil +} + +var Locales = []*Locale{ + { + IncludeReddit: true, + Image: "static/images/locales/gb.png", + HelpForumId: 584, + }, + { + Subdomain: "br", + Image: "static/images/locales/br.png", + Translations: map[string]string{ + "Activity": "Atividade", + "Thread": "Discussão", + "Poster": "Autor", + "Time": "Hora", + "Forum": "Fórum", + "Hide Help Forum": "Ocultar Fórum de Ajuda", + "Show Help Forum": "Mostrar Fórum de Ajuda", + }, + HelpForumId: 774, + }, + { + Subdomain: "ru", + Image: "static/images/locales/ru.png", + Translations: map[string]string{ + "Activity": "Активность", + "Thread": "Тема", + "Poster": "Автор", + "Time": "Время", + "Forum": "Форум", + "Hide Help Forum": "Скрыть форум помощи", + "Show Help Forum": "Показать Форум Помощи", + }, + HelpForumId: 1281, + }, + { + Subdomain: "th", + Image: "static/images/locales/th.png", + Translations: map[string]string{ + "Activity": "กิจกรรม", + "Thread": "กระทู้", + "Poster": "ผู้โพสต์", + "Time": "เวลา", + "Forum": "ฟอรั่ม", + "Hide Help Forum": "ซ่อนฟอรั่มช่วยเหลือ", + "Show Help Forum": "แสดงฟอรั่มช่วยเหลือ", + }, + HelpForumId: 1011, + }, + { + Subdomain: "de", + Image: "static/images/locales/de.png", + Translations: map[string]string{ + "Activity": "Aktivität", + "Thread": "Beitrag", + "Poster": "Verfasser", + "Time": "Datum", + "Forum": "Forum", + "Hide Help Forum": "Hilfeforum ausblenden", + "Show Help Forum": "Hilfeforum anzeigen", + }, + HelpForumId: 1123, + }, + { + Subdomain: "fr", + Image: "static/images/locales/fr.png", + Translations: map[string]string{ + "Activity": "Activité", + "Thread": "Fil de discussion", + "Poster": "Posteur", + "Time": "Date", + "Forum": "Forum", + "Hide Help Forum": "Masquer le forum d'aide", + "Show Help Forum": "Afficher le forum d'aide", + }, + HelpForumId: 1051, + }, + { + Subdomain: "es", + Image: "static/images/locales/es.png", + Translations: map[string]string{ + "Activity": "Actividad", + "Thread": "Tema", + "Poster": "Autor", + "Time": "Fecha", + "Forum": "Foro", + "Hide Help Forum": "Ocultar el foro de ayuda", + "Show Help Forum": "Mostrar el foro de ayuda", + }, + HelpForumId: 1193, + }, + { + Subdomain: "jp", + Image: "static/images/locales/jp.png", + Translations: map[string]string{ + "Activity": "アクティビティ", + "Thread": "スレッド", + "Poster": "投稿者", + "Time": "日時", + "Forum": "フォーラム", + }, + }, +} + +func LocaleForRequest(r *http.Request) *Locale { + subdomain := "" + if r.Host != "" { + subdomain = strings.Split(r.Host, ".")[0] + } + + if subdomain != "" { + for _, l := range Locales { + if l.Subdomain == subdomain { + return l + } + } + } + + return Locales[0] +} diff --git a/server/localization_test.go b/server/localization_test.go new file mode 100644 index 0000000..a50ff22 --- /dev/null +++ b/server/localization_test.go @@ -0,0 +1,16 @@ +package server + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestLocale_RefreshForumIds(t *testing.T) { + for _, l := range Locales { + t.Run(l.ForumHost(), func(t *testing.T) { + assert.NoError(t, l.RefreshForumIds()) + assert.NotEmpty(t, l.ForumIds()) + }) + } +} diff --git a/server/reddit_comment.go b/server/reddit_comment.go index 8de747d..d5bae23 100644 --- a/server/reddit_comment.go +++ b/server/reddit_comment.go @@ -13,6 +13,9 @@ type RedditComment struct { PostId string `json:"post_id"` PostTitle string `json:"post_title"` Time time.Time `json:"time"` + + // Added in August 2022. Comments stored before then may not have this. + Subreddit string `json:"subreddit"` } func (c *RedditComment) ActivityTime() time.Time { @@ -25,9 +28,17 @@ func (c *RedditComment) ActivityKey() uint32 { } func (c *RedditComment) PostURL() string { - return fmt.Sprintf("https://www.reddit.com/r/pathofexile/comments/%v/", c.PostId) + subreddit := "pathofexile" + if c.Subreddit != "" { + subreddit = c.Subreddit + } + return fmt.Sprintf("https://www.reddit.com/r/%v/comments/%v/", subreddit, c.PostId) } func (c *RedditComment) CommentURL() string { - return fmt.Sprintf("https://www.reddit.com/r/pathofexile/comments/%v/-/%v/?context=3", c.PostId, c.Id) + subreddit := "pathofexile" + if c.Subreddit != "" { + subreddit = c.Subreddit + } + return fmt.Sprintf("https://www.reddit.com/r/%v/comments/%v/-/%v/?context=3", subreddit, c.PostId, c.Id) } diff --git a/server/reddit_indexer.go b/server/reddit_indexer.go index 68ce28c..d2d097a 100644 --- a/server/reddit_indexer.go +++ b/server/reddit_indexer.go @@ -8,11 +8,12 @@ import ( "strings" "time" - log "github.com/Sirupsen/logrus" + log "github.com/sirupsen/logrus" ) type RedditIndexerConfiguration struct { - Database *Database + Database Database + Auth string } type RedditIndexer struct { @@ -33,13 +34,18 @@ func (indexer *RedditIndexer) Close() { indexer.closeSignal <- true } +const redditRequestInterval = time.Second * 8 + func (indexer *RedditIndexer) run() { log.Info("starting reddit indexer") users := []string{ - "chris_wilson", "Bex_GGG", "Negitivefrags", "Omnitect", "qarldev", "BrianWeissman_GGG", - "Mark_GGG", "RhysGGG", "Dan_GGG", "Rory_Rackham", "Blake_GGG", "Fitzy_GGG", "Hartlin_GGG", - "Hrishi_GGG", "Baltic_GGG", "KamilOrmanJanowski", + "chris_wilson", "Bex_GGG", "Negitivefrags", "Omnitect", "Mark_GGG", "RhysGGG", "Dan_GGG", + "Rory_Rackham", "Blake_GGG", "Fitzy_GGG", "Hartlin_GGG", "Hrishi_GGG", "Baltic_GGG", + "KamilOrmanJanowski", "Daniel_GGG", "Jeff_GGG", "NapfelGGG", "Baltic_GGG", "Novynn", + "Felipe_GGG", "Mel_GGG", "Sarah_GGG", "riandrake", "Kieren_GGG", "Openarl", "Natalia_GGG", + "AlexDenfordGGG", "Stacey_GGG", "ZaccieA", "viperesque", "rach_ggg", "Community_Team", + "M59Gar", "Dominic_GGG", "Nick_GGG", "MatthewD_GGG", "Belakay_ggg", } next := 0 @@ -48,12 +54,14 @@ func (indexer *RedditIndexer) run() { case <-indexer.closeSignal: return default: - indexer.index(users[next]) + if err := indexer.index(users[next]); err != nil { + log.WithError(err).Error("error indexing reddit user: " + users[next]) + } next += 1 if next >= len(users) { next = 0 } - time.Sleep(time.Second * 2) + time.Sleep(redditRequestInterval) } } } @@ -78,6 +86,7 @@ func ParseRedditActivity(b []byte) ([]Activity, string, error) { CreatedUTC float64 `json:"created_utc"` LinkId string `json:"link_id"` LinkTitle string `json:"link_title"` + Subreddit string `json:"subreddit"` } `json:"data"` } `json:"children"` } `json:"data"` @@ -88,7 +97,7 @@ func ParseRedditActivity(b []byte) ([]Activity, string, error) { } for _, thing := range root.Data.Children { - if thing.Data.SubredditId != "t5_2sf6m" { + if thing.Data.SubredditId != "t5_2sf6m" && thing.Data.SubredditId != "t5_2w3q8" && thing.Data.SubredditId != "t5_3910n" { continue } switch thing.Kind { @@ -100,6 +109,7 @@ func ParseRedditActivity(b []byte) ([]Activity, string, error) { PostId: strings.TrimPrefix(thing.Data.LinkId, "t3_"), PostTitle: thing.Data.LinkTitle, Time: time.Unix(int64(thing.Data.CreatedUTC), 0), + Subreddit: thing.Data.Subreddit, }) case "t3": activity = append(activity, &RedditPost{ @@ -110,6 +120,7 @@ func ParseRedditActivity(b []byte) ([]Activity, string, error) { Title: thing.Data.Title, URL: thing.Data.URL, Time: time.Unix(int64(thing.Data.CreatedUTC), 0), + Subreddit: thing.Data.Subreddit, }) } } @@ -128,6 +139,9 @@ func (indexer *RedditIndexer) redditActivity(user string, page string) ([]Activi return nil, "", err } req.Header.Add("User-Agent", "GGG Tracker (https://github.com/ccbrown/gggtracker) by /u/rz2yoj") + if parts := strings.Split(indexer.configuration.Auth, ":"); len(parts) == 2 { + req.SetBasicAuth(parts[0], parts[1]) + } resp, err := client.Do(req) if err != nil { @@ -142,12 +156,13 @@ func (indexer *RedditIndexer) redditActivity(user string, page string) ([]Activi return ParseRedditActivity(body) } -func (indexer *RedditIndexer) index(user string) { +func (indexer *RedditIndexer) index(user string) error { logger := log.WithFields(log.Fields{ "user": user, }) - cutoff := time.Now().Add(time.Hour * -12) + pageCutoff := time.Now().Add(-12 * time.Hour) + cutoff := time.Now().Add(-14 * 24 * time.Hour) activity := []Activity(nil) for page := ""; ; { @@ -157,25 +172,30 @@ func (indexer *RedditIndexer) index(user string) { logger.WithError(err).Error("error requesting reddit activity") } - done := len(things) == 0 + done := len(things) == 0 || next == "" for _, thing := range things { - if thing.ActivityTime().Before(cutoff) { + if thing.ActivityTime().Before(pageCutoff) { done = true } + if thing.ActivityTime().Before(cutoff) { + break + } activity = append(activity, thing) } logger.WithFields(log.Fields{ - "count": len(things), + "count": len(activity), }).Info("received reddit activity") if done { break } - time.Sleep(time.Second * 2) + time.Sleep(redditRequestInterval) } - if len(activity) > 0 { - indexer.configuration.Database.AddActivity(activity) + if len(activity) == 0 { + return nil } + + return indexer.configuration.Database.AddActivity(activity) } diff --git a/server/reddit_indexer_test.go b/server/reddit_indexer_test.go index 13b1f94..6079a0d 100644 --- a/server/reddit_indexer_test.go +++ b/server/reddit_indexer_test.go @@ -30,6 +30,7 @@ func TestParseRedditComments(t *testing.T) { assert.Equal(t, "/r/pathofexile/comments/5q12qc/another_update_on_singapore_fibre_cuts/", post.Permalink) assert.Equal(t, "Another Update on Singapore Fibre Cuts", post.Title) assert.Equal(t, time.Unix(1485316926, 0), post.Time) + assert.Equal(t, "pathofexile", post.Subreddit) comment, ok := activity[1].(*RedditComment) require.True(t, ok) @@ -37,4 +38,5 @@ func TestParseRedditComments(t *testing.T) { assert.Equal(t, "5plxw0", comment.PostId) assert.Equal(t, "Development Manifesto: Solo Self-Found Support in 2.6.0", comment.PostTitle) assert.Equal(t, time.Unix(1485287813, 0), comment.Time) + assert.Equal(t, "pathofexile", comment.Subreddit) } diff --git a/server/reddit_post.go b/server/reddit_post.go index fa238a2..f910bd8 100644 --- a/server/reddit_post.go +++ b/server/reddit_post.go @@ -13,6 +13,9 @@ type RedditPost struct { BodyHTML string `json:"body_html,omitempty"` Permalink string `json:"permalink"` Time time.Time `json:"time"` + + // Added in August 2022. Comments stored before then may not have this. + Subreddit string `json:"subreddit"` } func (p *RedditPost) ActivityTime() time.Time { diff --git a/server/rss_handler.go b/server/rss_handler.go index 03540be..cf520bc 100644 --- a/server/rss_handler.go +++ b/server/rss_handler.go @@ -41,9 +41,12 @@ type rssResponse struct { Channel rssChannel `xml:"channel"` } -func RSSHandler(db *Database) echo.HandlerFunc { +func RSSHandler(db Database) echo.HandlerFunc { return func(c echo.Context) error { - activity, _ := db.Activity(c.QueryParam("next"), 50) + activity, _, err := db.Activity(LocaleForRequest(c.Request()), c.QueryParam("next"), 50, func(a Activity) bool { return true }) + if err != nil { + return err + } response := rssResponse{ Version: "2.0", Atom: "http://www.w3.org/2005/Atom", diff --git a/server/server.go b/server/server.go new file mode 100644 index 0000000..091c5d4 --- /dev/null +++ b/server/server.go @@ -0,0 +1,54 @@ +package server + +import ( + "embed" + "io" + "net/http" + "net/url" + "path" + "path/filepath" + "time" + + "github.com/labstack/echo" + "github.com/labstack/echo/middleware" +) + +//go:embed static/* +var static embed.FS + +func serveAsset(c echo.Context, path string) error { + f, err := static.Open(path) + if err != nil { + http.NotFound(c.Response(), c.Request()) + return nil + } + defer f.Close() + http.ServeContent(c.Response(), c.Request(), path, time.Time{}, f.(io.ReadSeeker)) + return nil +} + +func New(db Database, ga string) *echo.Echo { + e := echo.New() + e.Use(middleware.Recover()) + + e.GET("/", IndexHandler(IndexConfiguration{ + GoogleAnalytics: ga, + })) + e.GET("/activity.json", ActivityHandler(db)) + e.GET("/rss", RSSHandler(db)) + e.GET("/rss.php", func(c echo.Context) error { + return c.Redirect(http.StatusMovedPermanently, AbsoluteURL(c, "/rss")) + }) + e.GET("/favicon.ico", func(c echo.Context) error { + return serveAsset(c, "static/favicon.ico") + }) + e.GET("/static/*", func(c echo.Context) error { + p, err := url.PathUnescape(c.Param("*")) + if err != nil { + return err + } + return serveAsset(c, filepath.Join("static", path.Clean("/"+p))) + }) + + return e +} diff --git a/server/static/images/locales/br.png b/server/static/images/locales/br.png new file mode 100644 index 0000000..ab9c25c Binary files /dev/null and b/server/static/images/locales/br.png differ diff --git a/server/static/images/locales/de.png b/server/static/images/locales/de.png new file mode 100644 index 0000000..ae1db16 Binary files /dev/null and b/server/static/images/locales/de.png differ diff --git a/server/static/images/locales/es.png b/server/static/images/locales/es.png new file mode 100644 index 0000000..8bf37a2 Binary files /dev/null and b/server/static/images/locales/es.png differ diff --git a/server/static/images/locales/fr.png b/server/static/images/locales/fr.png new file mode 100644 index 0000000..6caf5bc Binary files /dev/null and b/server/static/images/locales/fr.png differ diff --git a/server/static/images/locales/gb.png b/server/static/images/locales/gb.png new file mode 100644 index 0000000..2aad20b Binary files /dev/null and b/server/static/images/locales/gb.png differ diff --git a/server/static/images/locales/jp.png b/server/static/images/locales/jp.png new file mode 100644 index 0000000..a432ef8 Binary files /dev/null and b/server/static/images/locales/jp.png differ diff --git a/server/static/images/locales/ru.png b/server/static/images/locales/ru.png new file mode 100644 index 0000000..edd28c1 Binary files /dev/null and b/server/static/images/locales/ru.png differ diff --git a/server/static/images/locales/th.png b/server/static/images/locales/th.png new file mode 100644 index 0000000..7d632fd Binary files /dev/null and b/server/static/images/locales/th.png differ diff --git a/server/static/index.js b/server/static/index.js index 734075f..72d6e0e 100644 --- a/server/static/index.js +++ b/server/static/index.js @@ -1,4 +1,5 @@ var currentPage = undefined; +var currentHideHelp = undefined; var POE = { Forum: { @@ -16,20 +17,30 @@ var POE = { }; function loadActivity() { - var page = location.hash.replace(/^#page=/, ''); - if (currentPage !== undefined && page == currentPage) { + var params = new URLSearchParams(location.hash.replace(/^#/, '')); + var page = params.get('page') || ''; + var hideHelp = params.has('nohelp') && params.get('nohelp') !== 'false'; + if (currentPage !== undefined && page == currentPage && currentHideHelp !== undefined && hideHelp == currentHideHelp) { return; } var previousPage = currentPage; + var previousHideHelp = currentHideHelp; + currentPage = page; + currentHideHelp = hideHelp; + + var canonicalNohelpParam = hideHelp ? '&nohelp' : ''; + + $('#activity-table tbody').empty().append($('').append($('').addClass(type == 'forum_post' ? 'forum' : 'reddit').addClass(type.replace('/_/', '-')); var $toggleTD = $('
ThreadPosterTimeForum{{call $.Translate "Thread"}}{{call $.Translate "Poster"}}{{call $.Translate "Time"}}{{call $.Translate "Forum"}}
').attr('colspan', 6).text('Loading...'))) - $.get('activity.json?next=' + page, function(data) { + $.get('activity.json?next=' + page + canonicalNohelpParam, function(data) { var $tbody = $('#activity-table tbody'); $tbody.empty(); for (var i = 0; i < data.activity.length; ++i) { var type = data.activity[i].type; var activity = data.activity[i].data; + var subreddit = activity.subreddit || 'pathofexile'; var $tr = $('
'); @@ -37,12 +48,12 @@ function loadActivity() { if (type == 'forum_post') { $tr.append($('').append($('') - .attr('href', 'https://www.pathofexile.com/forum/view-thread/' + activity.thread_id + '/filter-account-type/staff') + .attr('href', 'https://' + activity.host + '/forum/view-thread/' + activity.thread_id + '/filter-account-type/staff') .append($('')) )); } else if (type == 'reddit_comment') { $tr.append($('').append($('') - .attr('href', 'https://www.reddit.com/r/pathofexile/comments/' + activity.post_id) + .attr('href', 'https://www.reddit.com/r/' + subreddit + '/comments/' + activity.post_id) .append($('')) )); } else { @@ -54,7 +65,7 @@ function loadActivity() { if (type == 'forum_post') { $tr.append($('').append($('') - .attr('href', 'https://www.pathofexile.com/forum/view-post/' + activity.id) + .attr('href', 'https://' + activity.host + '/forum/view-post/' + activity.id) .text(activity.thread_title) )); } else if (type == "reddit_post") { @@ -64,14 +75,14 @@ function loadActivity() { )); } else if (type == "reddit_comment") { $tr.append($('').append($('') - .attr('href', 'https://www.reddit.com/r/pathofexile/comments/' + activity.post_id + '/-/' + activity.id + '/?context=3') + .attr('href', 'https://www.reddit.com/r/' + subreddit + '/comments/' + activity.post_id + '/-/' + activity.id + '/?context=3') .text(activity.post_title) )); } if (type == 'forum_post') { $tr.append($('').append($('') - .attr('href', 'https://www.pathofexile.com/account/view-profile/' + encodeURIComponent(activity.poster)) + .attr('href', 'https://' + activity.host + '/account/view-profile/' + encodeURIComponent(activity.poster) + '-' + String(activity.poster_discriminator || 0).padStart(4, '0')) .text(activity.poster) )); } else { @@ -85,13 +96,13 @@ function loadActivity() { if (type == 'forum_post') { $tr.append($('').append($('') - .attr('href', 'https://www.pathofexile.com/forum/view-forum/' + encodeURIComponent(activity.forum_id)) + .attr('href', 'https://' + activity.host + '/forum/view-forum/' + encodeURIComponent(activity.forum_id)) .text(activity.forum_name) )); } else { $tr.append($('').append($('') - .attr('href', 'https://www.reddit.com/r/pathofexile') - .text('pathofexile') + .attr('href', 'https://www.reddit.com/r/' + subreddit) + .text(subreddit) )); } @@ -107,7 +118,7 @@ function loadActivity() { $body.find('a').each(function() { var r = $(this).attr('href'); if (r && (r.indexOf(':') < 0 || r.indexOf('/') <= r.indexOf(':'))) { - var root = type == 'forum_post' ? 'https://www.pathofexile.com' : 'https://www.reddit.com'; + var root = type == 'forum_post' ? 'https://' + activity.host : 'https://www.reddit.com'; $(this).attr('href', root + (r[0] == '/' ? '' : '/') + r); } }); @@ -134,14 +145,23 @@ function loadActivity() { $tbody.append($tr); } - $('#activity-nav').empty().append($('').text('Next Page').attr('href', '#page=' + data.next).click(function() { + if (hideHelp) { + $('#hide-help-forum').hide(); + $('#show-help-forum').attr('href', page ? '#page=' + page : '#').show(); + } else { + $('#show-help-forum').hide(); + $('#hide-help-forum').attr('href', (page ? '#page=' + page + '&' : '#') + 'nohelp').show(); + } + + $('#activity-nav').empty().append($('').text('Next Page').attr('href', '#page=' + data.next + canonicalNohelpParam).click(function() { window.scrollTo(0, 0); })); }).fail(function() { alert('Something went wrong. Better luck next time.'); currentPage = previousPage if (currentPage !== undefined) { - window.location.hash = 'page=' + currentPage; + var previousNohelpParam = previousHideHelp ? '&nohelp' : ''; + window.location.hash = 'page=' + currentPage + previousNohelpParam; } else { window.location.hash = ''; } diff --git a/server/static/style.css b/server/static/style.css index c03a29c..f6714c3 100644 --- a/server/static/style.css +++ b/server/static/style.css @@ -1,6 +1,6 @@ a { color: #C9BA8A; - text-decoration: none; + text-decoration: none; } a:hover { @@ -52,6 +52,23 @@ td { text-align: right; } +div.notice { + background-color: #DFCF99; + padding: 0px 12px; + margin: 24px; + border: 1px solid #363636; + box-shadow: 3px 3px 16px 4px #000000; + width: 980px; + position: relative; + font-size: 14px; + color: #202020; +} + +div.notice a, div.notice a:visited, div.notice a:active, div.notice a:hover { + color: #202020; + text-decoration: underline; +} + div.content-box { background-color: #202020; padding: 12px; @@ -82,6 +99,12 @@ div.container { opacity: 1.0; } +#help-toggle { + position: absolute; + top: 24px; + right: 50px; +} + div.content-box h1 { padding: 6px; padding-top: 0px; @@ -107,11 +130,11 @@ footer { } footer a { - text-decoration: underline; + text-decoration: underline; } td.poster { - text-align: center; + text-align: center; } .forum td.poster a { @@ -126,7 +149,7 @@ td.poster { } .reddit td.poster a { - color: #f33f3f; + color: #f33f3f; text-decoration: none; font-weight: bold; } @@ -137,100 +160,124 @@ td.poster { } .reddit td { - background-color: #07192c; + background-color: #07192c; } td.toggle { - padding: 0px; - width: 34px; - height: 34px; + padding: 0px; + width: 34px; + height: 34px; } td.time { - width: 200px; - text-align: center; + width: 200px; + text-align: center; } td.forum { - text-overflow: ellipsis; - overflow: hidden; - white-space: nowrap; - max-width: 140px; - text-align: center; + text-overflow: ellipsis; + overflow: hidden; + white-space: nowrap; + max-width: 140px; + text-align: center; } .reddit .forum a::before { - content: '/r/'; + content: '/r/'; } td.toggle { - position: relative; - padding: 0px; + position: relative; + padding: 0px; } td.toggle img { - cursor: pointer; - width: 18px; - height: 18px; - position: absolute; - top: 50%; - left: 50%; - transform: translate(-50%, -50%); - filter: opacity(70%); + cursor: pointer; + width: 18px; + height: 18px; + position: absolute; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); + filter: opacity(70%); } .expander:hover, .collapser:hover { - filter: opacity(100%); + filter: opacity(100%); } td.icon { - position: relative; - padding: 0px; - width: 35px; + position: relative; + padding: 0px; + width: 35px; } td.icon img { - position: absolute; - height: 24px; - top: 50%; - left: 50%; - transform: translate(-50%, -50%); + position: absolute; + height: 24px; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); +} + +td.title { + max-width: 450px; + overflow: hidden; } td.body { - background-color: #262626; - padding: 10px; + background-color: #262626; + padding: 10px; } td.body blockquote { - background: #323232; - border-left: 5px solid #1a1a1a; - margin: 1.5em 10px; - padding: 0.5em 10px; + background: #323232; + border-left: 5px solid #1a1a1a; + margin: 1.5em 10px; + padding: 0.5em 10px; +} + +td.body img { + max-width: 100%; } .spoilerHidden .spoilerContent { - display: none; + display: none; } .spoilerTitle input[type="button"] { - margin-left: 8px; - color: #c9ba8a; - background-color: #0f0f0f; - border: 0; - cursor: pointer; - text-decoration: none; + margin-left: 8px; + color: #c9ba8a; + background-color: #0f0f0f; + border: 0; + cursor: pointer; + text-decoration: none; } .spoilerTitle input[type="button"]:hover { - text-decoration: underline; + text-decoration: underline; } .spoiler { - border: 1px solid #1a1a1a; - background-color: #323232; + border: 1px solid #1a1a1a; + background-color: #323232; } .spoilerTitle, .spoilerContent { - padding: 8px; + padding: 8px; +} + +#locale-selection { + float: right; + list-style: none; + margin: 0px; +} + +#locale-selection li { + display: inline; + opacity: 0.5; +} + +#locale-selection li.selected-locale { + opacity: 1.0; } diff --git a/server/testdata/forum-maintenance.html b/server/testdata/forum-maintenance.html new file mode 100644 index 0000000..6e1f145 --- /dev/null +++ b/server/testdata/forum-maintenance.html @@ -0,0 +1,74 @@ + + + + Path of Exile + + + + + + + + + + + + + + + +
+
+
+
+ + + + + + + + + +
+ + Path of Exile + +
+
+
+
+
+
+

Down For Maintenance

+
+

+ pathofexile.com is currently down for maintenance. Please try again later.

+Thanks for your patience! +

+
+
+
+
+ + +
+
+
+
+
+
+
+
+
+
+ © 2010 - 2019 Grinding Gear Games + Contact Support
+ +
+
+
+
+
+ + diff --git a/server/testdata/forum-posts-logged-out.html b/server/testdata/forum-posts-logged-out.html new file mode 100644 index 0000000..c530980 --- /dev/null +++ b/server/testdata/forum-posts-logged-out.html @@ -0,0 +1,508 @@ + + + + Path of Exile + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+ +
+ + + English + + + + + Português Brasileiro + + + + + Русский + + + + + ไทย + + + + + Deutsch + + + + + Français + + + + + Español + + + + + + + + 한국어 + + +
+ + Path of Exile + + +
+ +
+
+
+
+
+

Login Required

+
+
+

You must log in to visit this area.

+
+ +
+
+
+ +
+
+
+
+
+
+
+
+
+

+ Home +

+ +
+
+

+ Game +

+ +
+
+

+ Forum +

+ +
+
+
+

+ Events +

+ +
+
+

+ Trade +

+ +
+
+

+ Shop +

+ +
+
+
+ + + + © 2010 - 2024 + Grinding Gear Games + + + + Terms of Use, Privacy Notice and Cookies Notice + + + - + + Contact Support + + + - + + Developer API + + + 5e876bd1d0b1304cf3ea631a1f5c6916 +
+
+
+
+
+
+
+ + + + + diff --git a/server/testdata/forum-posts.html b/server/testdata/forum-posts.html index 34b7a1f..9d23698 100644 --- a/server/testdata/forum-posts.html +++ b/server/testdata/forum-posts.html @@ -69,38 +69,38 @@
View Profile»Recent posts by Chris
we had a great time too!
Lead Developer. Follow us on: Twitter | YouTube | Facebook | Contact Support if you need help!
-maps3
View post
Avatar -
Posted by
on Feb 5, 2017 2:06:05 PM
Grinding Gear Games
Yesterday, we attended the San Francisco fan meetup and had a great time! It was our busiest one yet, and we were able to stay for three hours answering questions and meeting fans. Today's news post has some photos of the event! Read More. +
Posted by
on Feb 5, 2017, 2:06:05 PM
Grinding Gear Games
Yesterday, we attended the San Francisco fan meetup and had a great time! It was our busiest one yet, and we were able to stay for three hours answering questions and meeting fans. Today's news post has some photos of the event! Read More.
Lead Developer. Follow us on: Twitter | YouTube | Facebook | Contact Support if you need help!
-maps3
Last edited by Chris on Feb 6, 2017 8:31:55 AM
View post
+maps3
Last edited by Chris on Feb 6, 2017 8:31:55 AM
View post
Avatar -
Posted by
on Feb 5, 2017 1:48:29 PM
Grinding Gear Games
Chris, Jonathan and Nick are going to be in San Francisco next week to hold a press tour for various upcoming announcements. If you're interested in meeting with them, they'll be hosting a fan meet-up on Saturday February 4th at 4pm (PST). If you're in or around the area, you should come on by! Read More. +
Posted by
on Feb 5, 2017, 1:48:29 PM
Grinding Gear Games
Chris, Jonathan and Nick are going to be in San Francisco next week to hold a press tour for various upcoming announcements. If you're interested in meeting with them, they'll be hosting a fan meet-up on Saturday February 4th at 4pm (PST). If you're in or around the area, you should come on by! Read More.
Lead Developer. Follow us on: Twitter | YouTube | Facebook | Contact Support if you need help!
maps3
View post
Avatar -
Posted by
on Jan 29, 2017 2:17:19 PM
Grinding Gear Games
Last year at the 2016 New Zealand Game Developers Conference, two of our designers gave a presentation called "The Labyrinth: Building Pathways for Players". It is a really detailed look at how Path of Exile's Labyrinth was designed and some of the challenges they faced when creating it. We have made the presentation (complete with their speaker notes to explain it) available here as a google document. +
Posted by
on Jan 29, 2017, 2:17:19 PM
Grinding Gear Games
Last year at the 2016 New Zealand Game Developers Conference, two of our designers gave a presentation called "The Labyrinth: Building Pathways for Players". It is a really detailed look at how Path of Exile's Labyrinth was designed and some of the challenges they faced when creating it. We have made the presentation (complete with their speaker notes to explain it) available here as a google document. Read More.
Lead Developer. Follow us on: Twitter | YouTube | Facebook | Contact Support if you need help!
maps3
View post
Avatar -
Posted by
on Jan 24, 2017 4:30:34 PM
Grinding Gear Games
"
Completed 9 ChallengesArcis wrote:
afaik, the main reason people want a SSF league is so the game doesn't have to be balanced around the fact that trading will always be much more efficient than soloing. if there are no additional benefits to playing SSF, i think this has missed the point.

+
Posted by
on Jan 24, 2017, 4:30:34 PM
Grinding Gear Games
"
Completed 9 ChallengesArcis wrote:
afaik, the main reason people want a SSF league is so the game doesn't have to be balanced around the fact that trading will always be much more efficient than soloing. if there are no additional benefits to playing SSF, i think this has missed the point.


Getting a fair playing field is really important to people who don't want to engage in trade.

I'm sure the two leagues will play out at massively different paces, and that's totally fine with us.
Lead Developer. Follow us on: Twitter | YouTube | Facebook | Contact Support if you need help!
maps3
View post
Avatar -
Posted by
on Jan 22, 2017 7:37:48 PM
Grinding Gear Games
Self-enforced Solo Self-Found play is an increasingly popular choice for players who want to have a more challenging experience in Path of Exile. In March's Content Update 2.6.0, we plan to add formal support for this mode so that players who enjoy SSF can prove that they reached certain levels without outside assistance. We have written a Development Manifesto post about our current plans. +
Posted by
on Jan 22, 2017, 7:37:48 PM
Grinding Gear Games
Self-enforced Solo Self-Found play is an increasingly popular choice for players who want to have a more challenging experience in Path of Exile. In March's Content Update 2.6.0, we plan to add formal support for this mode so that players who enjoy SSF can prove that they reached certain levels without outside assistance. We have written a Development Manifesto post about our current plans. Read More.
Lead Developer. Follow us on: Twitter | YouTube | Facebook | Contact Support if you need help!
maps3
View post
Avatar -
Posted by
on Jan 22, 2017 7:25:56 PM
Grinding Gear Games
We plan to add a system to officially record the progress of Solo Self-Found players in Content Update 2.6.0. This manifesto post describes our current plan. We welcome your feedback.
+
Posted by
on Jan 22, 2017, 7:25:56 PM
Grinding Gear Games
We plan to add a system to officially record the progress of Solo Self-Found players in Content Update 2.6.0. This manifesto post describes our current plan. We welcome your feedback.

What is Solo Self-Found?

@@ -136,7 +136,7 @@ Please let us know if you have any feedback on this system!
Lead Developer. Follow us on: Twitter | YouTube | Facebook | Contact Support if you need help!
maps3
View post
Avatar -
Posted by
on Jan 22, 2017 7:25:15 PM
Grinding Gear Games
When we started our Xbox One project in 2015, we looked at how other online game communities reacted to the announcement of a console version of those games. While many players were excited, some were disappointed and had a few fears:
+
Posted by
on Jan 22, 2017, 7:25:15 PM
Grinding Gear Games
When we started our Xbox One project in 2015, we looked at how other online game communities reacted to the announcement of a console version of those games. While many players were excited, some were disappointed and had a few fears:
  • The fear that development resources would be spent on a version of the game that didn't directly benefit that player.
  • The fear that the PC version would be modified to become more similar to the new console version.

After we saw other communities reacting like this, we internally pledged to make sure that our development of the console version of Path of Exile took these concerns into account.

@@ -149,17 +149,17 @@ I hope you enjoyed the new trailer and spotted a few of the easter eggs that hint at the massive amount of content we'll be announcing next month!
Lead Developer. Follow us on: Twitter | YouTube | Facebook | Contact Support if you need help!
maps3
Last edited by Chris on Jan 18, 2017 4:54:03 PM
View post
Avatar -
Posted by
on Jan 18, 2017 4:50:53 PM
Grinding Gear Games
Our programmers have completed another wave of game client performance improvements. We plan to release them next week in a minor update called 2.5.2. Today's news post explains how these changes improve Path of Exile's performance. Read More. +
Posted by
on Jan 18, 2017, 4:50:53 PM
Grinding Gear Games
Our programmers have completed another wave of game client performance improvements. We plan to release them next week in a minor update called 2.5.2. Today's news post explains how these changes improve Path of Exile's performance. Read More.
Lead Developer. Follow us on: Twitter | YouTube | Facebook | Contact Support if you need help!
maps3
View post
Avatar -
Posted by
on Jan 10, 2017 2:12:19 PM
Grinding Gear Games
Most of our team are now back at work after the New Zealand holiday period. We have a busy year ahead! The purpose of today's news post is to highlight the rough plans of what happens when the Breach leagues end in seven weeks. Read More. +
Posted by
on Jan 10, 2017, 2:12:19 PM
Grinding Gear Games
Most of our team are now back at work after the New Zealand holiday period. We have a busy year ahead! The purpose of today's news post is to highlight the rough plans of what happens when the Breach leagues end in seven weeks. Read More.
Lead Developer. Follow us on: Twitter | YouTube | Facebook | Contact Support if you need help!
maps3
View post
Avatar -
Posted by
on Jan 8, 2017 7:11:07 PM
Grinding Gear Games
+
Posted by
on Jan 8, 2017, 7:11:07 PM
Grinding Gear Games
diff --git a/server/testdata/forum-preferences.html b/server/testdata/forum-preferences.html index 3ce51ab..a2ec6c2 100644 --- a/server/testdata/forum-preferences.html +++ b/server/testdata/forum-preferences.html @@ -70,131 +70,117 @@
My Account»Preferences

Account Preferences

Looking to change your Supporter Titles? You can now find them here.

-
-
- - - - - - - - - - - -
-
-
- -
Check this box to receive a Message when someone in the forum quotes you.
- -
Check this box to receive the Path of Exile newsletter via email
- -
Check this box to receive account status related email.
-
-
+
+
+
+
+
Check this box to receive a Message when someone in the forum quotes you.
+
Check this box to receive the Path of Exile newsletter via email.
+
Check this box to receive account status related email.
+
+
+