Automate top-picks API fetching

Author: Jack Li

.gitignore

@@ -1,4 +1,4 @@
 /**/__pycache__
 /**/js/node_modules
 /**/js/antibot-simulation.js
-
+/**/tmp

ticketscraping/constants.py

@@ -1,6 +1,57 @@
+from uuid import uuid4
+from urllib.parse import quote_plus
+
 ANTIBOT_JS_CODE_URL = "https://epsf.ticketmaster.com/eps-d"
+TOKEN_INTERROGATION_URL = "https://epsf.ticketmaster.com/eps-d?d=www.ticketmaster.com"
+
+
+def get_top_picks_url(
+    eventId): return f"https://offeradapter.ticketmaster.com/api/ismds/event/{eventId}/quickpicks"
+
+
+EVENT_ID = "0C005B5587A017CF"
+BASIC_REQ_HEADER = {"origin": "https://www.ticketmaster.com",
+                    "referer": "https://www.ticketmaster.com/"}
+def get_top_picks_header(): return {
+    **BASIC_REQ_HEADER,
+    "tmps-correlation-id": str(uuid4())
+}
+
+def get_top_picks_query_params_str(qty=2, priceInterval=(
+    0, 100)):
+    def get_top_picks_query_params(qty, priceInterval): return {
+        'show': 'places maxQuantity sections',
+        'mode': 'primary:ppsectionrow resale:ga_areas platinum:all',
+        'qty': qty,
+        'q': f"and(not(\'accessible\'),any(listprices,$and(gte(@,{priceInterval[0]}),lte(@,{priceInterval[1]}))))",
+        'includeStandard': 'true',
+        'includeResale': 'true',
+        'includePlatinumInventoryType': 'false',
+        'embed': ['area', 'offer', 'description'],
+        'apikey': 'b462oi7fic6pehcdkzony5bxhe',
+        'apisecret': 'pquzpfrfz7zd2ylvtz3w5dtyse',
+        'limit': 100,
+        'offset': 0,
+        'sort': '-quality',
+    }
+    def encodeURI(s):
+        return quote_plus(s, safe="():=,$@'")
+    def formulateURI(params):
+        items = []
+        for p in params:
+            if isinstance(params[p], list):
+                for i in params[p]:
+                    items.append(encodeURI(f"{p}={i}"))
+            else:
+                items.append(encodeURI(f"{p}={params[p]}"))
+        return '?' + '&'.join(items)
+    return formulateURI(get_top_picks_query_params(qty, priceInterval))
 
 FN_MATCHING_REGEX = r"\(function\(\){.*}\)\(\)"
+TOKEN_RENEW_SEC_OFFSET = 3
+TOKEN_RENEW_PRIORITY = 3
+TICKET_SCRAPING_PRIORITY = 1
+TICKET_SCRAPING_INTERVAL = 60
 
 INJECTOR_LOCATION = "js/injector.js"
 INJECTOR_HEADER_LOCATION = "js/injector-header.js"

ticketscraping/prepare_reese84token.py

@@ -0,0 +1,54 @@
+import os
+import re
+import json
+import requests
+import subprocess
+from . import constants
+
+
+def getReese84Token():
+    def readFileContentToString(filename):
+        f = open(filename, 'r')
+        content = f.read()
+        f.close()
+        return content
+
+    # fetch the javascript that generates the reese84
+    antibot_js_code_full = requests.get(constants.ANTIBOT_JS_CODE_URL).text
+
+    # trim the code to the function that is only used
+    match_obj = re.search(constants.FN_MATCHING_REGEX, antibot_js_code_full)
+    if not match_obj:
+        return None
+    start, end = match_obj.span()
+    antibot_js_code_trim = antibot_js_code_full[start:end]
+
+    # inject the code to the javascript
+    injector_js_code_loc = os.path.join(
+        os.path.dirname(__file__), constants.INJECTOR_LOCATION)
+    injector_header_js_code_loc = os.path.join(os.path.dirname(
+        __file__), constants.INJECTOR_HEADER_LOCATION)
+    injector_js_code, injector_header_js_code = readFileContentToString(
+        injector_js_code_loc), readFileContentToString(injector_header_js_code_loc)
+    runnable_js_code = injector_header_js_code + \
+        antibot_js_code_trim + injector_js_code
+
+    # save the runnable js code
+    runnable_file_loc = os.path.join(os.path.dirname(
+        __file__), constants.RENNABLE_FILENAME)
+    runnable_file = open(runnable_file_loc, "w")
+    runnable_file.write(runnable_js_code)
+    runnable_file.close()
+
+    # run the js code using local node.js
+    res = subprocess.run(
+        ["node", runnable_file_loc], capture_output=True)
+    token_str = res.stdout
+
+    # produce the reese84 object
+    token = json.loads(token_str)
+
+    # invoke the get token api to get the reese84 token
+    token_json_res = requests.post(
+        constants.TOKEN_INTERROGATION_URL, headers=constants.BASIC_REQ_HEADER, json=token)
+    return token_json_res.json()

ticketscraping/scraping.py

@@ -1,57 +1,43 @@
 import os
-import re
+import sched
+import time
 import json
 import requests
 import subprocess
 from . import constants
-
-
-def readFileContentToString(filename):
-    f = open(filename, 'r')
-    content = f.read()
-    f.close()
-    return content
-
-def getReese84Token():
-    # fetch the javascript that generates the reese84
-    antibot_js_code_full = requests.get(constants.ANTIBOT_JS_CODE_URL).text
-
-    # trim the code to the function that is only used
-    match_obj = re.search(constants.FN_MATCHING_REGEX, antibot_js_code_full)
-    if not match_obj:
-        return None
-    start, end = match_obj.span()
-    antibot_js_code_trim = antibot_js_code_full[start:end]
-
-    # inject the code to the javascript
-    injector_js_code_loc = os.path.join(
-        os.path.dirname(__file__), constants.INJECTOR_LOCATION)
-    injector_header_js_code_loc = os.path.join(os.path.dirname(
-        __file__), constants.INJECTOR_HEADER_LOCATION)
-    injector_js_code, injector_header_js_code = readFileContentToString(
-        injector_js_code_loc), readFileContentToString(injector_header_js_code_loc)
-    runnable_js_code = injector_header_js_code + \
-        antibot_js_code_trim + injector_js_code
-
-    # save the runnable js code
-    runnable_file_loc = os.path.join(os.path.dirname(
-        __file__), constants.RENNABLE_FILENAME)
-    runnable_file = open(runnable_file_loc, "w")
-    runnable_file.write(runnable_js_code)
-    runnable_file.close()
-
-    # run the js code using local node.js
-    res = subprocess.run(
-        ["node", runnable_file_loc], capture_output=True)
-    token_str = res.stdout
-
-    # produce the reese84 object
-    token = json.loads(token_str)
-    print(token)
-
-    # invoke the get token api to get the reese84 token
-    # TO-DO
-
-
-# invoke the top-picks api to get the tickets
-# TO-DO
+from threading import Semaphore
+from .prepare_reese84token import getReese84Token
+
+s = sched.scheduler(time.time, time.sleep)
+token_semaphore = Semaphore(1)
+reese84_token = {}
+    
+
+def obtainReese84Token(sch):
+    global reese84_token, token_semaphore
+    token_semaphore.acquire()
+    reese84_token = getReese84Token()
+    token_semaphore.release()
+    print(reese84_token)
+    sch.enter(reese84_token['renewInSec'] -
+              constants.TOKEN_RENEW_SEC_OFFSET, constants.TOKEN_RENEW_PRIORITY, obtainReese84Token, (sch,))
+
+
+def ticket_scraping(sch):
+    global reese84_token, token_semaphore
+    token_semaphore.acquire()
+    top_picks_url = constants.get_top_picks_url(constants.EVENT_ID)
+    top_picks_q_params_str = constants.get_top_picks_query_params_str(2, (0, 200))
+    top_picks_url = top_picks_url + top_picks_q_params_str
+    top_picks_header = constants.get_top_picks_header()
+    res = requests.get(top_picks_url, headers=top_picks_header,
+                       cookies=dict(reese84=reese84_token['token']))
+    token_semaphore.release()
+    print(res.json())
+    sch.enter(constants.TICKET_SCRAPING_INTERVAL,
+              constants.TICKET_SCRAPING_PRIORITY, ticket_scraping, (sch,))
+
+def start():
+    obtainReese84Token(s)
+    ticket_scraping(s)
+    s.run()

tmtracker/urls.py

@@ -15,12 +15,15 @@
 """
 from django.contrib import admin
 from django.urls import path
-from ticketscraping.scraping import getReese84Token
+from ticketscraping.scraping import start
 from utils import get_db_handle
+from datetime import datetime
+import requests
 
 get_db_handle('tickets')
 print("=== database connection is established ===")
-getReese84Token()
+print(f"server started at {datetime.now().strftime('%d/%m/%Y %H:%M:%S')}")
+start()
 
 urlpatterns = [
     path('admin/', admin.site.urls),