hide headless mode

Author: Hongze Xia

go.mod

@@ -4,7 +4,7 @@ go 1.12
 
 require (
 	cloud.google.com/go v0.30.0
-	github.com/chromedp/cdproto v0.0.0-20191114225735-6626966fbae4 // indirect
+	github.com/chromedp/cdproto v0.0.0-20191114225735-6626966fbae4
 	github.com/chromedp/chromedp v0.5.1
 	github.com/google/go-cmp v0.3.1 // indirect
 	github.com/googleapis/gax-go v2.0.0+incompatible // indirect

gutils/tasks.go

@@ -23,7 +23,7 @@ func processEntity(ent *models.Entity, pushClient *pushover.Client) {
 	defer func() {
 		ctx, cancel := context.WithTimeout(context.Background(), time.Duration(CancelWaitTime))
 		if err := ent.Save(ctx, EntityType, DsClient, true); err != nil {
-			log.Println("ERROR: failed to save entity:", err, ". Entity: ", ent)
+			log.Printf("ERROR: failed to save entity [%s] with %v", ent.Name, err)
 		}
 		cancel()
 	}()
@@ -91,6 +91,7 @@ func Refresh(pushClient *pushover.Client, fetchLimit int) {
 	log.Println("INFO: Refresh started")
 	entities := FetchData(fetchLimit)
 	for _, ent := range entities {
+		log.Printf("INFO: processing [%s] XPATH (%s) at %s", ent.Name, ent.XPATH, ent.URL)
 		processEntity(&ent, pushClient)
 	}
 	log.Println("INFO: Refresh ended")

main.go

@@ -49,7 +49,8 @@ func main() {
 	flag.StringVar(&userToken, "userToken", "", "pushover user token")
 	flag.StringVar(&port, "port", "8080", "server port")
 	flag.IntVar(&schdlFreq, "schdlFreq", 2, "schedule frequency in minutes")
-	flag.IntVar(&fetchLimit, "fetchLimit", 2, "fetch limit from google datastore")
+	flag.IntVar(&fetchLimit, "fetchLimit", 10, "fetch limit from google datastore")
+	flag.Parse()
 	if appToken == "" || userToken == "" {
 		log.Fatalln("appToken and userToken must be given")
 	}

models/requests.go

@@ -5,6 +5,7 @@ import "cloud.google.com/go/datastore"
 type (
 	// Options is the options for an entry
 	Options struct {
+		Email      string  `json:"email"`
 		CheckFreq  int16   `json:"checkFreq"` // in minutes
 		AlertType  string  `json:"alertType"`
 		Threshold  float32 `json:"threshold"`

tests/chromedp_test.go

@@ -0,0 +1,19 @@
+package main
+
+import (
+	"log"
+	"testing"
+
+	"github.com/xiahongze/pricetracker/trackers"
+)
+
+func TestChromedp(t *testing.T) {
+	url := "https://shop.coles.com.au/a/a-nsw-metro-westmead/product/goldn-canola-canola-oil"
+	xpath := `//span/strong[@class="product-price"]`
+	price, ok := trackers.ChromeTracker(&url, &xpath)
+	if !ok {
+		t.Errorf("can't fetch price from %s with %s", url, xpath)
+		return
+	}
+	log.Printf("price: %s", price)
+}

trackers/chrome_tracker.go

@@ -8,17 +8,19 @@ import (
 	"strings"
 	"time"
 
+	"github.com/chromedp/cdproto/page"
 	"github.com/chromedp/chromedp"
 )
 
 var (
-	chromeTimeout = time.Second * 60
+	chromeTimeout = time.Second * 10
 	chromeOpts    = []chromedp.ExecAllocatorOption{
 		chromedp.NoFirstRun,
 		chromedp.NoDefaultBrowserCheck,
 		chromedp.Headless,
 		chromedp.DisableGPU,
 		chromedp.Flag("enable-automation", false),
+		chromedp.UserAgent(`Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36`),
 		chromedp.Flag("disable-background-networking", true),
 		chromedp.Flag("disable-background-timer-throttling", true),
 		chromedp.Flag("disable-backgrounding-occluded-windows", true),
@@ -35,6 +37,23 @@ var (
 		chromedp.Flag("disable-renderer-backgrounding", true),
 		chromedp.Flag("disable-sync", true),
 	}
+	/**
+	 * Hard research to find out how to hide the automation mode
+	 * https://github.com/chromedp/chromedp/issues/396
+	 * https://intoli.com/blog/not-possible-to-block-chrome-headless/
+	 **/
+	hiddenScript = `
+	Object.defineProperty(navigator, 'webdriver', {
+		get: () => false,
+	  });`
+	hide = chromedp.ActionFunc(func(ctx context.Context) error {
+		_, err := page.AddScriptToEvaluateOnNewDocument(hiddenScript).Do(ctx)
+		if err != nil {
+			return err
+		}
+		// log.Println("identifier: ", identifier.String())
+		return nil
+	})
 )
 
 func init() {
@@ -59,10 +78,15 @@ func ChromeTracker(url, xpath *string) (string, bool) {
 	defer cancel()
 
 	var res string
-	err := chromedp.Run(ctx, chromedp.Navigate(*url), chromedp.Text(*xpath, &res, chromedp.NodeVisible, chromedp.BySearch))
+
+	err := chromedp.Run(ctx,
+		hide,
+		chromedp.Navigate(*url),
+		chromedp.Text(*xpath, &res, chromedp.NodeVisible, chromedp.BySearch),
+	)
 
 	if err != nil {
-		log.Fatal(err)
+		log.Printf("WARN: failed to fetch with chromedp with %v", err)
 	}
 
 	return strings.TrimSpace(res), true