diff --git a/.gitignore b/.gitignore index f744723..4edd60f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.db gggtracker +/server/bindata.go /vendor diff --git a/Dockerfile b/Dockerfile index 37ab1cc..4ab23ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,7 @@ RUN wget -O - https://raw.githubusercontent.com/golang/dep/master/install.sh | s ADD . . RUN dep ensure +RUN go generate ./... RUN go vet . && go test -v ./... RUN go build . diff --git a/Gopkg.lock b/Gopkg.lock index 2c713f1..a190ce6 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -13,6 +13,46 @@ revision = "901648c87902174f774fac311d7f176f8647bdaa" version = "v1.0.0" +[[projects]] + name = "github.com/aws/aws-lambda-go" + packages = [ + "events", + "lambda", + "lambda/messages", + "lambdacontext" + ] + revision = "fb8f88d824489a878aee1e0badde7d8e129f8767" + version = "v1.7.0" + +[[projects]] + name = "github.com/aws/aws-sdk-go-v2" + packages = [ + "aws", + "aws/awserr", + "aws/defaults", + "aws/ec2metadata", + "aws/ec2rolecreds", + "aws/endpointcreds", + "aws/endpoints", + "aws/external", + "aws/signer/v4", + "aws/stscreds", + "internal/awsutil", + "internal/ini", + "internal/sdk", + "private/protocol", + "private/protocol/json/jsonutil", + "private/protocol/jsonrpc", + "private/protocol/query", + "private/protocol/query/queryutil", + "private/protocol/rest", + "private/protocol/xml/xmlutil", + "service/dynamodb", + "service/sts" + ] + revision = "d52522b5f4b95591ff6528d7c54923951aadf099" + version = "v2.0.0-preview.5" + [[projects]] name = "github.com/boltdb/bolt" packages = ["."] @@ -53,6 +93,17 @@ ] revision = "23c074d0eceb2b8a5bfdbb271ab780cde70f05a8" +[[projects]] + name = "github.com/jmespath/go-jmespath" + packages = ["."] + revision = "0b12d6b5" + +[[projects]] + name = "github.com/json-iterator/go" + packages = ["."] + revision = "1624edc4454b8682399def8740d46db5e4362ba4" + version = "v1.1.5" + [[projects]] name = "github.com/labstack/echo" packages = [ @@ -97,12 +148,30 @@ packages = ["."] revision = "00c29f56e2386353d58c599509e8dc3801b0d716" +[[projects]] + name = "github.com/modern-go/concurrent" + packages = ["."] + revision = "bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94" + version = "1.0.3" + +[[projects]] + name = "github.com/modern-go/reflect2" + packages = ["."] + revision = "4b7aa43c6742a2c18fdef89dd197aaae7dac7ccd" + version = "1.0.1" + [[projects]] name = "github.com/pelletier/go-toml" packages = ["."] revision = "acdc4509485b587f5e675510c4f2c63e90ff68a8" version = "v1.1.0" +[[projects]] + name = "github.com/pkg/errors" + packages = ["."] + revision = "645ef00459ed84a119197bfb8d8205042c6df63d" + version = "v0.8.0" + [[projects]] name = "github.com/pmezard/go-difflib" packages = ["difflib"] @@ -219,6 +288,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "fb3f71f8b31a4ef69f8c12df7c17800632ad541fb57db8b99989ec33570b63fd" + inputs-digest = "6ff140fe2ee3cc2d088f6f049eac879676eaa9f3ddc7a5b4ca6c58e0b1f5e79f" solver-name = "gps-cdcl" solver-version = 1 diff --git a/README.md b/README.md index 992b553..6f92932 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This is the repository for [gggtracker.com](https://gggtracker.com). If there's ### Development -If you have Go installed, you can develop and run the server in the usual way: `go get ./... && go run main.go` +If you have Go installed, you can develop and run the server in the usual way: `go get ./... && go generate ./... && go run main.go` ![Development](development.gif) diff --git a/aws-sam/.gitignore b/aws-sam/.gitignore new file mode 100644 index 0000000..f6f05fb --- /dev/null +++ b/aws-sam/.gitignore @@ -0,0 +1,2 @@ +/build +/dist.zip diff --git a/aws-sam/Makefile b/aws-sam/Makefile new file mode 100644 index 0000000..6ee2e6c --- /dev/null +++ b/aws-sam/Makefile @@ -0,0 +1,18 @@ +.PHONY: build build-environment-build lambda-environment-build + +dist.zip: build + zip -j dist.zip build/main + +build: + dep ensure + go generate ../... + docker-compose run --rm build-environment make build-environment-build + docker-compose run --rm lambda-environment make lambda-environment-build + +build-environment-build: + rm -rf ./build + go build -o ./build/main . + ./build/main --help + +lambda-environment-build: + ./build/main --help || (ldd ./build/main && exit 1) diff --git a/aws-sam/docker-compose.yaml b/aws-sam/docker-compose.yaml new file mode 100644 index 0000000..bf9424b --- /dev/null +++ b/aws-sam/docker-compose.yaml @@ -0,0 +1,15 @@ +version: '3' +services: + build-environment: + image: golang:1.11 + volumes: + - ../:/go/src/github.com/ccbrown/gggtracker + working_dir: /go/src/github.com/ccbrown/gggtracker/aws-sam + lambda-environment: + entrypoint: [] + environment: + LD_LIBRARY_PATH: '' + image: lambci/lambda:go1.x + volumes: + - ../:/go/src/github.com/ccbrown/gggtracker + working_dir: /go/src/github.com/ccbrown/gggtracker/aws-sam diff --git a/aws-sam/gggtracker.cfn.yaml b/aws-sam/gggtracker.cfn.yaml index 814247e..a5807e7 100644 --- a/aws-sam/gggtracker.cfn.yaml +++ b/aws-sam/gggtracker.cfn.yaml @@ -1,6 +1,203 @@ AWSTemplateFormatVersion: '2010-09-09' Description: github.com/ccbrown/gggtracker +Parameters: + BinaryMediaType: + Type: String + AllowedValues: + - '*~1*' + - '*/*' + Description: See https://forums.aws.amazon.com/thread.jspa?messageID=797934 + CodeS3Bucket: + Type: String + Description: The bucket that contains the code to deploy. + CodeS3Key: + Type: String + Description: The key for the code to deploy. + CertificateARN: + Type: String + Description: A certificate corresponding to DomainName. + DomainName: + Type: String + Description: The desired domain name. It'll be up to you to actually change your DNS after the stack is deployed. + GoogleAnalytics: + Type: String + Default: '' Resources: + API: + Type: AWS::ApiGateway::RestApi + Properties: + BinaryMediaTypes: + - !Ref BinaryMediaType + Name: !Ref AWS::StackName + APIAccount: + Type: AWS::ApiGateway::Account + Properties: + CloudWatchRoleArn: !GetAtt APICloudWatchRole.Arn + APIBasePathMapping: + Type: AWS::ApiGateway::BasePathMapping + DependsOn: APIDomainName + Properties: + DomainName: !Ref DomainName + RestApiId: !Ref API + Stage: !Ref APIStage + APIBasePathMappingSubdomain1: + Type: AWS::ApiGateway::BasePathMapping + DependsOn: APISubdomainName1 + Properties: + DomainName: !Sub br.${DomainName} + RestApiId: !Ref API + Stage: !Ref APIStage + APIBasePathMappingSubdomain2: + Type: AWS::ApiGateway::BasePathMapping + DependsOn: APISubdomainName2 + Properties: + DomainName: !Sub ru.${DomainName} + RestApiId: !Ref API + Stage: !Ref APIStage + APIBasePathMappingSubdomain3: + Type: AWS::ApiGateway::BasePathMapping + DependsOn: APISubdomainName3 + Properties: + DomainName: !Sub th.${DomainName} + RestApiId: !Ref API + Stage: !Ref APIStage + APIBasePathMappingSubdomain4: + Type: AWS::ApiGateway::BasePathMapping + DependsOn: APISubdomainName4 + Properties: + DomainName: !Sub de.${DomainName} + RestApiId: !Ref API + Stage: !Ref APIStage + APIBasePathMappingSubdomain5: + Type: AWS::ApiGateway::BasePathMapping + DependsOn: APISubdomainName5 + Properties: + DomainName: !Sub fr.${DomainName} + RestApiId: !Ref API + Stage: !Ref APIStage + APIBasePathMappingSubdomain6: + Type: AWS::ApiGateway::BasePathMapping + DependsOn: APISubdomainName6 + Properties: + DomainName: !Sub es.${DomainName} + RestApiId: !Ref API + Stage: !Ref APIStage + APICloudWatchRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: apigateway.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AmazonAPIGatewayPushToCloudWatchLogs + APIDomainName: + Type: AWS::ApiGateway::DomainName + Properties: + CertificateArn: !Ref CertificateARN + DomainName: !Ref DomainName + EndpointConfiguration: + Types: + - EDGE + APISubdomainName1: + Type: AWS::ApiGateway::DomainName + Properties: + CertificateArn: !Ref CertificateARN + DomainName: !Sub br.${DomainName} + EndpointConfiguration: + Types: + - EDGE + APISubdomainName2: + Type: AWS::ApiGateway::DomainName + Properties: + CertificateArn: !Ref CertificateARN + DomainName: !Sub ru.${DomainName} + EndpointConfiguration: + Types: + - EDGE + APISubdomainName3: + Type: AWS::ApiGateway::DomainName + Properties: + CertificateArn: !Ref CertificateARN + DomainName: !Sub th.${DomainName} + EndpointConfiguration: + Types: + - EDGE + APISubdomainName4: + Type: AWS::ApiGateway::DomainName + Properties: + CertificateArn: !Ref CertificateARN + DomainName: !Sub de.${DomainName} + EndpointConfiguration: + Types: + - EDGE + APISubdomainName5: + Type: AWS::ApiGateway::DomainName + Properties: + CertificateArn: !Ref CertificateARN + DomainName: !Sub fr.${DomainName} + EndpointConfiguration: + Types: + - EDGE + APISubdomainName6: + Type: AWS::ApiGateway::DomainName + Properties: + CertificateArn: !Ref CertificateARN + DomainName: !Sub es.${DomainName} + EndpointConfiguration: + Types: + - EDGE + APIProxyResource: + Type: AWS::ApiGateway::Resource + Properties: + ParentId: !GetAtt API.RootResourceId + PathPart: '{proxy+}' + RestApiId: !Ref API + APIRootMethod: + Type: AWS::ApiGateway::Method + Properties: + AuthorizationType: NONE + HttpMethod: ANY + Integration: + Type: AWS_PROXY + IntegrationHttpMethod: POST + Uri: !Sub arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${APIFunction.Arn}/invocations + ResourceId: !GetAtt API.RootResourceId + RestApiId: !Ref API + APIProxyMethod: + Type: AWS::ApiGateway::Method + Properties: + AuthorizationType: NONE + HttpMethod: ANY + Integration: + Type: AWS_PROXY + IntegrationHttpMethod: POST + Uri: !Sub arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${APIFunction.Arn}/invocations + ResourceId: !Ref APIProxyResource + RestApiId: !Ref API + APIDeployment: + Type: AWS::ApiGateway::Deployment + DependsOn: + - APIRootMethod + - APIProxyMethod + Properties: + RestApiId: !Ref API + APIStage: + Type: AWS::ApiGateway::Stage + DependsOn: + - APIAccount + Properties: + DeploymentId: !Ref APIDeployment + MethodSettings: + - HttpMethod: '*' + LoggingLevel: ERROR + MetricsEnabled: true + ResourcePath: /* + RestApiId: !Ref API + StageName: stage DynamoDBTable: Type: AWS::DynamoDB::Table Properties: @@ -16,4 +213,48 @@ Resources: KeyType: RANGE ProvisionedThroughput: ReadCapacityUnits: 25 - WriteCapacityUnits: 25 + WriteCapacityUnits: 50 + APIFunction: + Type: AWS::Lambda::Function + Properties: + Code: + S3Bucket: !Ref CodeS3Bucket + S3Key: !Ref CodeS3Key + Environment: + Variables: + GGGTRACKER_DYNAMODB_TABLE: !Ref DynamoDBTable + GGGTRACKER_GA: !Ref GoogleAnalytics + Handler: main + MemorySize: 128 + Role: !GetAtt AppFunctionRole.Arn + Runtime: go1.x + Timeout: 60 + APIFunctionAPIPermission: + Type: AWS::Lambda::Permission + Properties: + Action: lambda:InvokeFunction + FunctionName: !GetAtt APIFunction.Arn + Principal: apigateway.amazonaws.com + SourceArn: !Sub arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}:${API}/*/*/* + AppFunctionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: lambda.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole + Policies: + - PolicyName: gggtracker + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: dynamodb:* + Resource: + - !Sub arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${DynamoDBTable} + - !Sub arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${DynamoDBTable}/* diff --git a/aws-sam/main.go b/aws-sam/main.go new file mode 100644 index 0000000..f9ca730 --- /dev/null +++ b/aws-sam/main.go @@ -0,0 +1,160 @@ +package main + +import ( + "bufio" + "bytes" + "encoding/base64" + "flag" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "os" + "strings" + + "github.com/aws/aws-lambda-go/events" + "github.com/aws/aws-lambda-go/lambda" + "github.com/aws/aws-sdk-go-v2/aws/external" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/crypto/ssh/terminal" + "golang.org/x/sys/unix" + + "github.com/ccbrown/gggtracker/server" +) + +type ResponseWriter struct { + header http.Header + statusCode int + body *bytes.Buffer +} + +func (w *ResponseWriter) Header() http.Header { + if w.header == nil { + w.header = http.Header{} + } + return w.header +} + +func (w *ResponseWriter) Write(b []byte) (int, error) { + if w.body == nil { + w.body = &bytes.Buffer{} + } + return w.body.Write(b) +} + +func (w *ResponseWriter) WriteHeader(statusCode int) { + w.statusCode = statusCode +} + +func (w *ResponseWriter) APIGatewayProxyResponse() (*events.APIGatewayProxyResponse, error) { + ret := &events.APIGatewayProxyResponse{} + if w.statusCode != 0 { + ret.StatusCode = w.statusCode + } else { + ret.StatusCode = http.StatusOK + } + for key, values := range w.header { + if len(values) > 1 { + return nil, fmt.Errorf("header has multiple values: " + key) + } else if len(values) == 1 { + if ret.Headers == nil { + ret.Headers = map[string]string{} + } + ret.Headers[key] = values[0] + } + } + if w.body != nil { + ret.Body = base64.StdEncoding.EncodeToString(w.body.Bytes()) + ret.IsBase64Encoded = true + } + return ret, nil +} + +func NewRequest(request *events.APIGatewayProxyRequest) (*http.Request, error) { + resource, err := url.ParseRequestURI(request.Path) + if err != nil { + return nil, errors.Wrap(err, "unable to parse request URI") + } + if len(request.QueryStringParameters) > 0 { + query := url.Values{} + for key, value := range request.QueryStringParameters { + query.Set(key, value) + } + resource.RawQuery = query.Encode() + } + req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(request.HTTPMethod + " " + resource.RequestURI() + " HTTP/1.0\r\n\r\n"))) + if err != nil { + return nil, errors.Wrap(err, "unable to create request") + } + + req.Proto = "HTTP/1.1" + req.ProtoMinor = 1 + + if request.Body != "" { + var body []byte + if request.IsBase64Encoded { + body, err = base64.StdEncoding.DecodeString(request.Body) + if err != nil { + return nil, errors.Wrap(err, "unable to decode base64 body") + } + } else { + body = []byte(request.Body) + } + req.ContentLength = int64(len(body)) + req.Body = ioutil.NopCloser(bytes.NewReader(body)) + } + + for key, value := range request.Headers { + req.Header.Set(key, value) + } + req.Host = req.Header.Get("Host") + req.RemoteAddr = request.RequestContext.Identity.SourceIP + + return req, nil +} + +func Handler(handler http.Handler) func(*events.APIGatewayProxyRequest) (*events.APIGatewayProxyResponse, error) { + return func(request *events.APIGatewayProxyRequest) (*events.APIGatewayProxyResponse, error) { + req, err := NewRequest(request) + if err != nil { + return nil, err + } + resp := &ResponseWriter{} + handler.ServeHTTP(resp, req) + return resp.APIGatewayProxyResponse() + } +} + +func StartHTTPHandler() { + config, err := external.LoadDefaultAWSConfig() + if err != nil { + logrus.Fatal(err) + } + db, err := server.NewDynamoDBDatabase(dynamodb.New(config), os.Getenv("GGGTRACKER_DYNAMODB_TABLE")) + if err != nil { + logrus.Fatal(err) + } + defer db.Close() + + e := server.New(db, os.Getenv("GGGTRACKER_GA")) + lambda.Start(Handler(e)) +} + +func main() { + if !terminal.IsTerminal(unix.Stdout) { + logrus.SetFormatter(&logrus.JSONFormatter{}) + } + + flags := flag.NewFlagSet(os.Args[0], flag.ContinueOnError) + if err := flags.Parse(os.Args[1:]); err != nil { + if err == flag.ErrHelp { + // Exit with no error if --help was given. This is used to test the build. + os.Exit(0) + } + logrus.Fatal(err) + } + + StartHTTPHandler() +} diff --git a/main.go b/main.go index 11b2da1..1df23f8 100644 --- a/main.go +++ b/main.go @@ -2,14 +2,10 @@ package main import ( "fmt" - "net/http" - "path" "strings" "github.com/aws/aws-sdk-go-v2/aws/external" "github.com/aws/aws-sdk-go-v2/service/dynamodb" - "github.com/labstack/echo" - "github.com/labstack/echo/middleware" log "github.com/sirupsen/logrus" "github.com/spf13/pflag" "github.com/spf13/viper" @@ -19,7 +15,7 @@ import ( func main() { pflag.IntP("port", "p", 8080, "the port to listen on") - pflag.String("staticdir", "./server/static", "the static files to serve") + pflag.String("staticdir", "", "this argument is ignored and will be removed") pflag.String("ga", "", "a google analytics account") pflag.String("db", "./gggtracker.db", "the database file path") pflag.String("dynamodb-table", "", "if given, DynamoDB will be used instead of a database file") @@ -31,8 +27,6 @@ func main() { viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) viper.AutomaticEnv() - e := echo.New() - var db server.Database var err error if tableName := viper.GetString("dynamodb-table"); tableName != "" { @@ -68,18 +62,6 @@ func main() { defer forumIndexer.Close() } - e.Use(middleware.Recover()) - - e.GET("/", server.IndexHandler(server.IndexConfiguration{ - GoogleAnalytics: viper.GetString("ga"), - })) - e.GET("/activity.json", server.ActivityHandler(db)) - e.GET("/rss", server.RSSHandler(db)) - e.GET("/rss.php", func(c echo.Context) error { - return c.Redirect(http.StatusMovedPermanently, server.AbsoluteURL(c, "/rss")) - }) - e.File("/favicon.ico", path.Join(viper.GetString("staticdir"), "favicon.ico")) - e.Static("/static", viper.GetString("staticdir")) - + e := server.New(db, viper.GetString("ga")) log.Fatal(e.Start(fmt.Sprintf(":%v", viper.GetInt("port")))) } diff --git a/server/index_handler.go b/server/index_handler.go index 3c420d6..ee7bc72 100644 --- a/server/index_handler.go +++ b/server/index_handler.go @@ -88,6 +88,7 @@ func init() { func IndexHandler(configuration IndexConfiguration) echo.HandlerFunc { return func(c echo.Context) error { + c.Response().Header().Set("Content-Type", "text/html; charset=utf-8") locale := LocaleForRequest(c.Request()) return indexTemplate.Execute(c.Response(), struct { Configuration IndexConfiguration diff --git a/server/server.go b/server/server.go new file mode 100644 index 0000000..d2ae0d3 --- /dev/null +++ b/server/server.go @@ -0,0 +1,50 @@ +//go:generate sh -c "go get -u github.com/kevinburke/go-bindata/... && `go env GOPATH`/bin/go-bindata -pkg server -ignore '(^|/)\\..*' static/..." +package server + +import ( + "bytes" + "net/http" + "net/url" + "path" + "path/filepath" + "time" + + "github.com/labstack/echo" + "github.com/labstack/echo/middleware" +) + +func serveAsset(c echo.Context, path string) error { + b, err := Asset(path) + if err != nil { + http.NotFound(c.Response(), c.Request()) + return nil + } + http.ServeContent(c.Response(), c.Request(), path, time.Time{}, bytes.NewReader(b)) + return nil +} + +func New(db Database, ga string) *echo.Echo { + e := echo.New() + e.Use(middleware.Recover()) + + e.GET("/", IndexHandler(IndexConfiguration{ + GoogleAnalytics: ga, + })) + e.GET("/activity.json", ActivityHandler(db)) + e.GET("/rss", RSSHandler(db)) + e.GET("/rss.php", func(c echo.Context) error { + return c.Redirect(http.StatusMovedPermanently, AbsoluteURL(c, "/rss")) + }) + e.GET("/favicon.ico", func(c echo.Context) error { + return serveAsset(c, "static/favicon.ico") + }) + e.GET("/static/*", func(c echo.Context) error { + p, err := url.PathUnescape(c.Param("*")) + if err != nil { + return err + } + return serveAsset(c, filepath.Join("static", path.Clean("/"+p))) + }) + + return e +}