abstract logic of ticket scraping into classes

Author: Jack Li

ticketscraping/constants.py

@@ -1,5 +1,4 @@
 from uuid import uuid4
-from urllib.parse import quote_plus
 
 ANTIBOT_JS_CODE_URL = "https://epsf.ticketmaster.com/eps-d"
 TOKEN_INTERROGATION_URL = "https://epsf.ticketmaster.com/eps-d?d=www.ticketmaster.com"
@@ -17,41 +16,28 @@ def get_top_picks_header(): return {
     "tmps-correlation-id": str(uuid4())
 }
 
-def get_top_picks_query_params_str(qty=2, priceInterval=(
-    0, 100)):
-    def get_top_picks_query_params(qty, priceInterval): return {
-        'show': 'places maxQuantity sections',
-        'mode': 'primary:ppsectionrow resale:ga_areas platinum:all',
-        'qty': qty,
-        'q': f"and(not(\'accessible\'),any(listprices,$and(gte(@,{priceInterval[0]}),lte(@,{priceInterval[1]}))))",
-        'includeStandard': 'true',
-        'includeResale': 'true',
-        'includePlatinumInventoryType': 'false',
-        'embed': ['area', 'offer', 'description'],
-        'apikey': 'b462oi7fic6pehcdkzony5bxhe',
-        'apisecret': 'pquzpfrfz7zd2ylvtz3w5dtyse',
-        'limit': 100,
-        'offset': 0,
-        'sort': '-quality',
-    }
-    def encodeURI(s):
-        return quote_plus(s, safe="():=,$@'")
-    def formulateURI(params):
-        items = []
-        for p in params:
-            if isinstance(params[p], list):
-                for i in params[p]:
-                    items.append(encodeURI(f"{p}={i}"))
-            else:
-                items.append(encodeURI(f"{p}={params[p]}"))
-        return '?' + '&'.join(items)
-    return formulateURI(get_top_picks_query_params(qty, priceInterval))
+def get_top_picks_query_params(qty, priceInterval): return {
+    'show': 'places maxQuantity sections',
+    'mode': 'primary:ppsectionrow resale:ga_areas platinum:all',
+    'qty': qty,
+    'q': f"and(not(\'accessible\'),any(listprices,$and(gte(@,{priceInterval[0]}),lte(@,{priceInterval[1]}))))",
+    'includeStandard': 'true',
+    'includeResale': 'true',
+    'includePlatinumInventoryType': 'false',
+    'embed': ['area', 'offer', 'description'],
+    'apikey': 'b462oi7fic6pehcdkzony5bxhe',
+    'apisecret': 'pquzpfrfz7zd2ylvtz3w5dtyse',
+    'limit': 100,
+    'offset': 0,
+    'sort': '-quality',
+}
 
 FN_MATCHING_REGEX = r"\(function\(\){.*}\)\(\)"
 TOKEN_RENEW_SEC_OFFSET = 3
-TOKEN_RENEW_PRIORITY = 3
-TICKET_SCRAPING_PRIORITY = 1
+TOKEN_RENEW_PRIORITY = 1
+TICKET_SCRAPING_PRIORITY = 3
 TICKET_SCRAPING_INTERVAL = 60
+TICKET_SCRAPING_TOKEN_AWAIT_MAX_INTERVAL = 5
 
 INJECTOR_LOCATION = "js/injector.js"
 INJECTOR_HEADER_LOCATION = "js/injector-header.js"

ticketscraping/scraping.py

@@ -8,36 +8,77 @@
 from threading import Semaphore
 from .prepare_reese84token import getReese84Token
 
-s = sched.scheduler(time.time, time.sleep)
-token_semaphore = Semaphore(1)
-reese84_token = {}
+class Reese84TokenUpdating():
+    def __init__(self):
+        self.is_running = False
+        self.reese84_token = {}
+        self.token_semaphore = Semaphore(0)
+        self.scheduler = sched.scheduler(time.time, time.sleep)
+    
+    def initialize_reese84_token(self):
+        """
+        This method should not be called directly.
+        """
+        self.reese84_token = getReese84Token()
+        self.token_semaphore.release() # produce a new token
+        self.scheduler.enter(self.reese84_token['renewInSec'] -
+                             constants.TOKEN_RENEW_SEC_OFFSET, constants.TOKEN_RENEW_PRIORITY, self.renew_reese84_token)
+
+    def renew_reese84_token(self):
+        """
+        This method should not be called directly.
+        """
+        self.token_semaphore.acquire() # invalidate a token
+        self.reese84_token = getReese84Token()
+        self.token_semaphore.release()
+        self.scheduler.enter(self.reese84_token['renewInSec'] -
+                             constants.TOKEN_RENEW_SEC_OFFSET, constants.TOKEN_RENEW_PRIORITY, self.renew_reese84_token)
+    
+    def start(self):
+        # if the scheduler is already started - do nothing
+        if self.is_running: return
+        self.is_running = True
+        self.initialize_reese84_token()
+        self.scheduler.run(False)
 
 
-def obtainReese84Token(sch):
-    global reese84_token, token_semaphore
-    token_semaphore.acquire()
-    reese84_token = getReese84Token()
-    token_semaphore.release()
-    print(reese84_token)
-    sch.enter(reese84_token['renewInSec'] -
-              constants.TOKEN_RENEW_SEC_OFFSET, constants.TOKEN_RENEW_PRIORITY, obtainReese84Token, (sch,))
 
+class TicketScraping():
+    def __init__(self, token_generator: Reese84TokenUpdating, event_id = constants.EVENT_ID, num_seats=2, price_range=(0, 200)):
+        self.event_id = event_id
+        self.num_seats = num_seats
+        self.price_range = price_range
+        self.token_gen = token_generator
+        self.scheduler = sched.scheduler(time.time, time.sleep)
+        self.is_running = False
+
+    def ticket_scraping(self):
+        if self.token_gen.token_semaphore._value <= 0:
+            # retry after a delay
+            self.scheduler.enter(constants.TICKET_SCRAPING_TOKEN_AWAIT_MAX_INTERVAL,
+                                 constants.TICKET_SCRAPING_PRIORITY, self.ticket_scraping)
+            return
+        top_picks_url = constants.get_top_picks_url(self.event_id)
+        top_picks_q_params = constants.get_top_picks_query_params(
+            self.num_seats, self.price_range)
+        top_picks_header = constants.get_top_picks_header()
+        res = requests.get(top_picks_url, headers=top_picks_header, params=top_picks_q_params,
+                           cookies=dict(reese84=self.token_gen.reese84_token['token']))
+        print(res.json())
+        self.scheduler.enter(constants.TICKET_SCRAPING_INTERVAL,
+                  constants.TICKET_SCRAPING_PRIORITY, self.ticket_scraping)
+    
+    def start(self):
+        # if the scheduler is already started - do nothing
+        if self.is_running:
+            return
+        self.is_running = True
+        self.ticket_scraping()
+        self.scheduler.run(False)
 
-def ticket_scraping(sch):
-    global reese84_token, token_semaphore
-    token_semaphore.acquire()
-    top_picks_url = constants.get_top_picks_url(constants.EVENT_ID)
-    top_picks_q_params_str = constants.get_top_picks_query_params_str(2, (0, 200))
-    top_picks_url = top_picks_url + top_picks_q_params_str
-    top_picks_header = constants.get_top_picks_header()
-    res = requests.get(top_picks_url, headers=top_picks_header,
-                       cookies=dict(reese84=reese84_token['token']))
-    token_semaphore.release()
-    print(res.json())
-    sch.enter(constants.TICKET_SCRAPING_INTERVAL,
-              constants.TICKET_SCRAPING_PRIORITY, ticket_scraping, (sch,))
 
 def start():
-    obtainReese84Token(s)
-    ticket_scraping(s)
-    s.run()
+    reese_token_gen = Reese84TokenUpdating()
+    ticket_scraping = TicketScraping(reese_token_gen)
+    reese_token_gen.start()
+    ticket_scraping.start()

tmtracker/__init__.py

@@ -0,0 +1 @@
+default_app_config = 'tmtracker.app.MyAppConfig'

tmtracker/app.py

@@ -0,0 +1,14 @@
+from django.apps import AppConfig
+from utils import get_db_handle
+from ticketscraping.scraping import start
+from datetime import datetime
+
+class MyAppConfig(AppConfig):
+   name = "tmtracker"
+   verbose_name = "start tmtracker"
+   
+   def ready(self):
+      print(f"server started at {datetime.now().strftime('%d/%m/%Y %H:%M:%S')}")
+      get_db_handle('tickets')
+      print("=== database connection is established ===")
+      start()

tmtracker/settings.py

@@ -36,6 +36,7 @@
     'django.contrib.contenttypes',
     'django.contrib.messages',
     'django.contrib.staticfiles',
+    "tmtracker"
 ]
 
 MIDDLEWARE = [

tmtracker/urls.py

@@ -15,15 +15,6 @@
 """
 from django.contrib import admin
 from django.urls import path
-from ticketscraping.scraping import start
-from utils import get_db_handle
-from datetime import datetime
-import requests
-
-get_db_handle('tickets')
-print("=== database connection is established ===")
-print(f"server started at {datetime.now().strftime('%d/%m/%Y %H:%M:%S')}")
-start()
 
 urlpatterns = [
     path('admin/', admin.site.urls),