Skip to content

Commit 6c29ce7

Browse files
authored
set user agent when scraping forums (ccbrown#82)
* set user agent when scraping forums * travis -> github actions
1 parent b3e4e44 commit 6c29ce7

File tree

3 files changed

+22
-7
lines changed

3 files changed

+22
-7
lines changed

.github/workflows/push.yml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
name: Push
2+
on: [push]
3+
jobs:
4+
build:
5+
name: Build
6+
runs-on: ubuntu-latest
7+
steps:
8+
- name: Checkout
9+
uses: actions/checkout@v2
10+
- name: Build
11+
run: |
12+
make docker-image

.travis.yml

Lines changed: 0 additions & 6 deletions
This file was deleted.

server/forum_indexer.go

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ import (
1616
log "github.com/sirupsen/logrus"
1717
)
1818

19+
const UserAgent = "gggtracker.com (github.com/ccbrown/gggtracker)"
20+
1921
type ForumIndexer struct {
2022
configuration ForumIndexerConfiguration
2123
closeSignal chan struct{}
@@ -118,7 +120,14 @@ func (indexer *ForumIndexer) requestDocument(resource string) (*goquery.Document
118120
Jar: jar,
119121
Timeout: time.Second * 10,
120122
}
121-
resp, err := client.Get(urlString)
123+
124+
req, err := http.NewRequest("GET", urlString, nil)
125+
req.Header.Set("User-Agent", UserAgent)
126+
if err != nil {
127+
return nil, err
128+
}
129+
130+
resp, err := client.Do(req)
122131
if err != nil {
123132
return nil, err
124133
}

0 commit comments

Comments
 (0)