Addperiodic task

Author: Jackiebibili

apps/storage/storage.py

@@ -10,7 +10,9 @@ def insert_one(collection_name, doc: dict, db_name="tickets"):
    return insert_one__(coll, doc)
 
 # insert many
-def insert_many(collection_name, docs: list, db_name="tickets"):
+def insert_many(collection_name, docs: list[dict], db_name="tickets"):
+   if len(docs) == 0:
+      return True
    db = get_db_handle(db_name)
    coll = db[collection_name]
    # additional attributes

apps/ticketscraping/constants.py

@@ -13,7 +13,9 @@ def get_top_picks_url(
                     "referer": "https://www.ticketmaster.com/"}
 DATABASE = {
     "EVENTS": "events",
-    "TOP_PICKS": "top-picks"
+    "TOP_PICKS": "top-picks",
+    "BEST_AVAILABLE_SEATS": "best-available-seats",
+    "BEST_HISTORY_SEATS": "best-history-seats"
 }
 def get_top_picks_header(): return {
     **BASIC_REQ_HEADER,
@@ -31,7 +33,7 @@ def get_top_picks_query_params(qty, priceInterval): return {
     'embed': ['area', 'offer', 'description'],
     'apikey': 'b462oi7fic6pehcdkzony5bxhe',
     'apisecret': 'pquzpfrfz7zd2ylvtz3w5dtyse',
-    'limit': 25,
+    'limit': 100,
     'offset': 0,
     'sort': '-quality',
 }

apps/ticketscraping/models/pick.py

@@ -0,0 +1,32 @@
+class Pick():
+   def __init__(self, type, selection, quality, section, row, area, maxQuantity, offer, seat_columns, _id=None, scraping_id=None):
+      self._id = _id
+      self.scraping_id = scraping_id
+      self.type = type
+      self.selection = selection
+      self.quality = quality
+      self.section = section
+      self.row = row
+      self.area = area
+      self.maxQuantity = maxQuantity
+      self.offer = offer
+      self.price = offer.get('listPrice')
+      self.seat_columns = seat_columns
+   
+   def setScrapingId(self, scraping_id: str):
+      self.scraping_id = scraping_id
+
+   def __eq__(self, other):
+      return (self.section == other.section and self.row == other.row and
+              ((type(self.seat_columns) is list and len(
+                  self.seat_columns) > 0 and type(other.seat_columns) is list and len(
+                  other.seat_columns) > 0 and self.seat_columns[0] == other.seat_columns[0]) or 
+                  (self.seat_columns is None and other.seat_columns is None)) and
+               self.price == other.price)
+
+   def __hash__(self):
+      return hash((self.section,
+                   self.row,
+                   self.seat_columns[0] if type(self.seat_columns) is list and len(
+                       self.seat_columns) > 0 else None,
+                   self.price))

apps/ticketscraping/scraping.py

@@ -8,7 +8,8 @@
 from threading import Semaphore
 from .prepare_reese84token import getReese84Token
 from ..storage.storage import *
-from .seat_analysis import store_seats
+from .seat_analysis import format_seats
+from .tasks.periodic import run_periodic_task
 
 class Reese84TokenUpdating():
     def __init__(self):
@@ -75,19 +76,25 @@ def flag_for_termination(self):
 
     def ticket_scraping(self):
         if self.token_gen.token_semaphore._value <= 0:
-            # retry after a delay
+            # phase: retry after a delay
             self.scheduler.enter(constants.TICKET_SCRAPING_TOKEN_AWAIT_MAX_INTERVAL,
                                  constants.TICKET_SCRAPING_PRIORITY, self.ticket_scraping)
             return
+        # scrape the top-picks from ticketmaster
         top_picks_url = constants.get_top_picks_url(self.event_id)
         top_picks_q_params = constants.get_top_picks_query_params(
             self.num_seats, self.price_range)
         top_picks_header = constants.get_top_picks_header()
         res = requests.get(top_picks_url, headers=top_picks_header, params=top_picks_q_params,
                            cookies=dict(reese84=self.token_gen.reese84_token['token']))
         # print(res.json())
-        res_obj = res.json()
-        store_seats(res_obj, self.subscribe_id)
+
+        # prune and format the received picks
+        picks_obj = format_seats(res.json(), self.subscribe_id)
+
+        # periodic task: update collections best_available_seats and best_history_seats
+        run_periodic_task(picks_obj, self.subscribe_id)
+
         print("Got the ticket info from TM. /", res.status_code)
         self.scheduler.enter(constants.TICKET_SCRAPING_INTERVAL,
                              constants.TICKET_SCRAPING_PRIORITY, self.ticket_scraping)

apps/ticketscraping/seat_analysis.py

@@ -1,9 +1,9 @@
 from dateutil import parser
-from ..storage.storage import insert_one
-from ..ticketscraping import constants
+# from ..storage.storage import insert_one
+# from ..ticketscraping import constants
 
 
-def store_seats(data, subscriber_id):
+def format_seats(data, subscriber_id):
     # prune top-picks data structure
     pruned_picks = prune_pick_attributes(data)
 
@@ -14,10 +14,10 @@ def store_seats(data, subscriber_id):
         remove_embedded_field
     ], pruned_picks, subscriber_id)
 
-    # store in db
-    # print(res)
-    insert_one(constants.DATABASE['TOP_PICKS'], res)
-    pass
+    # # store in db
+    # # print(res)
+    # insert_one(constants.DATABASE['TOP_PICKS'], res)
+    return res
 
 def pipe(fns: list, *args):
     out = args

apps/ticketscraping/tasks/periodic.py

@@ -0,0 +1,71 @@
+from ...storage.storage import find_many, insert_many, delete_many
+from ...ticketscraping import constants
+from ..models.pick import Pick
+
+def generate_picks_set_from_picks(picks):
+   def __helper(pick: dict):
+      return Pick(_id=pick.get('_id'),
+                  scraping_id=pick.get('scraping_id'),
+                  type=pick['type'],
+                  selection=pick['selection'],
+                  quality=pick['quality'],
+                  section=pick['section'],
+                  row=pick['row'],
+                  area=pick['area'],
+                  maxQuantity=pick['maxQuantity'],
+                  offer=pick['offer'],
+                  seat_columns=pick['seat_columns'])
+
+   if type(picks) is dict:
+      return set(map(__helper, picks['picks']))
+   elif type(picks) is list:
+      return set(map(__helper, picks))
+   else:
+      raise Exception('argument type error')
+
+def get_current_best_available(scraping_id: str):
+   return find_many(constants.DATABASE['BEST_AVAILABLE_SEATS'], {"scraping_id": scraping_id})
+def remove_best_seats(seats: set[Pick]):
+   ids = []
+   for seat in seats:
+      ids.append(seat._id)
+   return delete_many(constants.DATABASE['BEST_AVAILABLE_SEATS'], {"_id" : {"$in": ids}})
+def insert_best_seats(seats: set[Pick], scraping_id: str):
+   for seat in seats:
+      seat.setScrapingId(scraping_id)
+   return insert_many(constants.DATABASE['BEST_AVAILABLE_SEATS'], list(map(lambda seat: vars(seat), seats)))
+def insert_history_seats(seats: set[Pick]):
+   return insert_many(constants.DATABASE['BEST_HISTORY_SEATS'], list(map(lambda seat: vars(seat), seats)))
+
+
+
+def run_periodic_task(picks: dict, scraping_id: str):
+   # B the list of new best available seats
+   new_best_avail = generate_picks_set_from_picks(picks)
+   # A be the list of current best available seats
+   cur_best_avail = generate_picks_set_from_picks(get_current_best_available(scraping_id))
+
+   # Compute C := A-B which is the seats
+   overwritten_seats = cur_best_avail - new_best_avail
+
+   # Compute D := B-A which is the new seats
+   new_seats = new_best_avail - cur_best_avail
+
+   print(f"size of B is {len(new_best_avail)}")
+   print(f"size of A is {len(cur_best_avail)}")
+   print(f"size of C is {len(overwritten_seats)}")
+   print(f"size of D is {len(new_seats)}")
+   
+   # Remove C from best_available_seats
+   remove_best_seats(overwritten_seats)
+
+   # Insert D to best_available_seats
+   insert_best_seats(new_seats, scraping_id)
+
+   # Save C to best_history_seats.
+   insert_history_seats(overwritten_seats)
+
+   # TODO
+   # Use D to invoke a handler to analyze them against the best_history_seats asynchronously.
+   
+   pass