From 80a6f56a960fb9e6e2ed76c0fb42e5a435b141e0 Mon Sep 17 00:00:00 2001 From: Christopher Brown Date: Thu, 8 Jul 2021 22:08:35 -0400 Subject: [PATCH 1/2] set user agent when scraping forums --- server/forum_indexer.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/server/forum_indexer.go b/server/forum_indexer.go index 3647c05..1df85a7 100644 --- a/server/forum_indexer.go +++ b/server/forum_indexer.go @@ -16,6 +16,8 @@ import ( log "github.com/sirupsen/logrus" ) +const UserAgent = "gggtracker.com (github.com/ccbrown/gggtracker)" + type ForumIndexer struct { configuration ForumIndexerConfiguration closeSignal chan struct{} @@ -118,7 +120,14 @@ func (indexer *ForumIndexer) requestDocument(resource string) (*goquery.Document Jar: jar, Timeout: time.Second * 10, } - resp, err := client.Get(urlString) + + req, err := http.NewRequest("GET", urlString, nil) + req.Header.Set("User-Agent", UserAgent) + if err != nil { + return nil, err + } + + resp, err := client.Do(req) if err != nil { return nil, err } From 5d5895bbeb7768260438951ac3792ed0e76a9d2d Mon Sep 17 00:00:00 2001 From: Christopher Brown Date: Thu, 8 Jul 2021 22:12:46 -0400 Subject: [PATCH 2/2] travis -> github actions --- .github/workflows/push.yml | 12 ++++++++++++ .travis.yml | 6 ------ 2 files changed, 12 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/push.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml new file mode 100644 index 0000000..f033944 --- /dev/null +++ b/.github/workflows/push.yml @@ -0,0 +1,12 @@ +name: Push +on: [push] +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Build + run: | + make docker-image diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index bf46dbe..0000000 --- a/.travis.yml +++ /dev/null @@ -1,6 +0,0 @@ -language: go -sudo: required -services: - - docker -script: - - make docker-image