fetch scraping jobs from db during startup and fix startup error

Author: Jack Li

startupApp/__init__.py

@@ -0,0 +1 @@
+default_app_config = 'startupApp.apps.MyAppConfig'

startupApp/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

tmtracker/app.py startupApp/apps.pytmtracker/app.py renamed to startupApp/apps.py

@@ -1,13 +1,14 @@
 from django.apps import AppConfig
-from utils import get_db_handle
 from ticketscraping.scraping import start
 from datetime import datetime
 
+
 class MyAppConfig(AppConfig):
-   name = "tmtracker"
+   name = "startupApp"
    verbose_name = "start tmtracker"
-   
+
    def ready(self):
-      print(f"server started at {datetime.now().strftime('%d/%m/%Y %H:%M:%S')}")
+      print(
+          f"server started at {datetime.now().strftime('%d/%m/%Y %H:%M:%S')}")
       print("=== database connection is established ===")
       start()

startupApp/migrations/__init__.py

@@ -0,0 +1,3 @@
+from django.db import models
+
+# Create your models here.

startupApp/models.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

startupApp/tests.py

@@ -0,0 +1,3 @@
+from django.shortcuts import render
+
+# Create your views here.

startupApp/views.py

@@ -1,25 +1,25 @@
 from pymongo import collection
 
-def __insert_one__(coll: collection.Collection, doc: dict):
+def insert_one__(coll: collection.Collection, doc: dict):
    return coll.insert_one(doc)
 
-def __insert_many__(coll: collection.Collection, docs: list):
+def insert_many__(coll: collection.Collection, docs: list):
    return coll.insert_many(docs)
 
-def __delete_one__(coll: collection.Collection, filter: dict):
+def delete_one__(coll: collection.Collection, filter: dict):
    return coll.delete_one(filter)
 
-def __delete_many__(coll: collection.Collection, filter: dict):
+def delete_many__(coll: collection.Collection, filter: dict):
    return coll.delete_many(filter)
 
-def __find_one_and_replace__(coll: collection.Collection, filter: dict, new_doc: dict):
+def find_one_and_replace__(coll: collection.Collection, filter: dict, new_doc: dict):
    return coll.find_one_and_replace(filter, new_doc)
 
-def __find_one_and_delete__(coll: collection.Collection, filter: dict):
+def find_one_and_delete__(coll: collection.Collection, filter: dict):
    return coll.find_one_and_delete(filter)
 
-def __find_one__(coll: collection.Collection, filter: dict, projection):
+def find_one__(coll: collection.Collection, filter: dict, projection):
    return coll.find_one(filter=filter, projection=projection)
 
-def __find_many__(coll: collection.Collection, filter: dict, projection, **kwargs):
+def find_many__(coll: collection.Collection, filter: dict, projection, **kwargs):
    return coll.find(filter=filter, projection=projection, **kwargs)

storage/base.py

@@ -1,4 +1,4 @@
-from ..utils import get_db_handle
+from utils import get_db_handle
 from .base import *
 from uuid import uuid4
 
@@ -7,48 +7,48 @@ def insert_one(collection_name, doc: dict, db_name="tickets"):
    db = get_db_handle(db_name)
    doc["_id"] = str(uuid4())
    coll = db[collection_name]
-   return __insert_one__(coll, doc)
+   return insert_one__(coll, doc)
 
 # insert many
 def insert_many(collection_name, docs: list, db_name="tickets"):
    db = get_db_handle(db_name)
    for doc in docs:
       doc["_id"] = str(uuid4())
    coll = db[collection_name]
-   return __insert_many__(coll, docs)
+   return insert_many__(coll, docs)
 
 # delete one
 def delete_one(collection_name, filter: dict, db_name="tickets"):
    db = get_db_handle(db_name)
    coll = db[collection_name]
-   return __delete_one__(coll, filter)
+   return delete_one__(coll, filter)
 
 # delete many
 def delete_many(collection_name, filter: dict, db_name="tickets"):
    db = get_db_handle(db_name)
    coll = db[collection_name]
-   return __delete_many__(coll, filter)
+   return delete_many__(coll, filter)
 
 # find one and replace
 def find_one_and_replace(collection_name, filter: dict, new_doc: dict, db_name="tickets"):
    db = get_db_handle(db_name)
    coll = db[collection_name]
-   return __find_one_and_replace__(coll, filter, new_doc)
+   return find_one_and_replace__(coll, filter, new_doc)
 
 # find one and delete
 def find_one_and_delete(collection_name, filter: dict, db_name="tickets"):
    db = get_db_handle(db_name)
    coll = db[collection_name]
-   return __find_one_and_delete__(coll, filter)
+   return find_one_and_delete__(coll, filter)
 
 # find one
 def find_one(collection_name, filter: dict, projection=None, db_name="tickets"):
    db = get_db_handle(db_name)
    coll = db[collection_name]
-   return __find_one__(coll, filter, projection)
+   return find_one__(coll, filter, projection)
 
 # find many
 def find_many(collection_name, filter: dict, projection=None, db_name="tickets", **kwargs):
    db = get_db_handle(db_name)
    coll = db[collection_name]
-   return __find_many__(coll, filter, projection, **kwargs)
+   return list(find_many__(coll, filter, projection, **kwargs))

storage/storage.py

@@ -11,6 +11,10 @@ def get_top_picks_url(
 EVENT_ID = "0C005B5587A017CF"
 BASIC_REQ_HEADER = {"origin": "https://www.ticketmaster.com",
                     "referer": "https://www.ticketmaster.com/"}
+DATABASE = {
+    "EVENTS": "events",
+    "TOP_PICKS": "top-picks"
+}
 def get_top_picks_header(): return {
     **BASIC_REQ_HEADER,
     "tmps-correlation-id": str(uuid4())