Adds CachedSC2Factories.

These factories perform basic caching of remote resources with minimal
configuration. There are currently three kinds:

* DictCachedSC2Factory - Caches results in memory with a dict. Uses an
    optional argument `cache_max_size` which dumbly limits the number
    of resources to cache in memory. Dict size is maintained by deleting
    the least active entry at insert time when full. Resource size can
    vary wildly at times; this factory does nothing to adjust for that.

* FileCachedSC2Factory - Cache results on disk in files. Requires your
    configuration with a pre-existing `cache_dir` for writing files.

* DoubleCachedSC2Factory - Uses both of the above caching layers to
    reduce IO time for commonly used resources. Order of operations:

	1. try to load from DictCache and return.
	2. try to load from FileCache, write to DictCache, and return.
	3. Load remote resource, write to both caches and return.

    Requires `cache_dir` and accepts an optional `cache_max_size` to
    limit the DictCache size.

These caching schemes can be enabled painlessly by calling:

 * sc2reader.useDictCache(cache_max_size=0, **SC2Factory_options)
 * sc2reader.useFileCache(cache_dir, **SC2Factory_options)
 * sc2reader.useDoubleCache(cache_dir, cache_max_size=0, **SC2Factory_options)

Thanks ggtracker for letting me push these utilties to the public repo.

Author: GraylinKim

sc2reader/__init__.py

@@ -1,5 +1,7 @@
 from __future__ import absolute_import
 
+import sys
+
 # import submodules
 from sc2reader import plugins, data, scripts
 
@@ -11,21 +13,35 @@
 # For backwards compatibility
 SC2Reader = factories.SC2Factory
 
-# Expose a nice module level interface
-__defaultSC2Reader = factories.SC2Factory()
+def setFactory(factory):
+    # Expose a nice module level interface
+    module = sys.modules[__name__]
+    module.load_replays = factory.load_replays
+    module.load_replay = factory.load_replay
+    module.load_maps = factory.load_maps
+    module.load_map = factory.load_map
+    module.load_game_summaries = factory.load_game_summaries
+    module.load_game_summary = factory.load_game_summary
+    module.load_map_infos = factory.load_map_infos
+    module.load_map_info = factory.load_map_info
+    module.load_map_histories = factory.load_map_headers
+    module.load_map_history = factory.load_map_header
+
+    module.configure = factory.configure
+    module.reset = factory.reset
+
+    module.register_plugin = factory.register_plugin
+    module._defaultFactory = factory
+
+def useFileCache(cache_dir, **options):
+    setFactory(factories.FileCachedSC2Factory(cache_dir, **options))
+
+def useDictCache(cache_max_size=0, **options):
+    setFactory(factories.DictCachedSC2Factory(cache_max_size, **options))
+
+def useDoubleCache(cache_dir, cache_max_size=0, **options):
+    setFactory(factories.DoubleCachedSC2Factory(cache_dir, cache_max_size, **options))
 
-load_replays = __defaultSC2Reader.load_replays
-load_replay = __defaultSC2Reader.load_replay
-load_maps = __defaultSC2Reader.load_maps
-load_map = __defaultSC2Reader.load_map
-load_game_summaries = __defaultSC2Reader.load_game_summaries
-load_game_summary = __defaultSC2Reader.load_game_summary
-load_map_infos = __defaultSC2Reader.load_map_infos
-load_map_info = __defaultSC2Reader.load_map_info
-load_map_histories = __defaultSC2Reader.load_map_headers
-load_map_history = __defaultSC2Reader.load_map_header
+setFactory(factories.SC2Factory())
 
-configure = __defaultSC2Reader.configure
-reset = __defaultSC2Reader.reset
 
-register_plugin = __defaultSC2Reader.register_plugin

sc2reader/factories.py

@@ -219,28 +219,108 @@ def _load_resource(self, resource, options=None, **new_options):
         return (resource, resource_name)
 
 
-class SC2Cache(SC2Factory):
-
-    def __init__(self, **options):
-        super(SC2Cache, self).__init__(self, **options)
-        self.cache = IntitializeCache(**options)
-
-    def load_map(self, map_file, options=None, **new_options):
-        options = options or utils.merged_dict(self.options, new_options)
-
-        if self.cache.has(map_file):
-            return self.cache.get(map_file)
+import urlparse, time
+
+class CachedSC2Factory(SC2Factory):
+
+    def get_remote_cache_key(self, remote_resource):
+        # Strip the port and use the domain as the bucket
+        # and use the full path as the key
+        parseresult = urlparse.urlparse(remote_resource)
+        bucket = re.sub(r':.*', '', parseresult.netloc)
+        key = parseresult.path.strip('/')
+        return (bucket, key)
+
+    def load_remote_resource_contents(self, remote_resource, **options):
+        cache_key = self.get_remote_cache_key(remote_resource)
+        if not self.cache_has(cache_key):
+            resource = super(CachedSC2Factory, self).load_remote_resource_contents(remote_resource, **options)
+            self.cache_set(cache_key, resource)
         else:
-            map = super(SC2Cache, self).load_map(map_file, options=options)
-            self.cache.set(map_file, map)
-            return map
-
-    def load_replay(self, replay_file, options=None, **new_options):
-        options = options or utils.merged_dict(self.options, new_options)
-
-        if self.cache.has(replay_file):
-            return self.cache.get(replay_file)
+            resource = self.cache_get(cache_key)
+        return resource
+
+    def cache_has(self, cache_key):
+        raise NotImplemented()
+
+    def cache_get(self, cache_key):
+        raise NotImplemented()
+
+    def cache_set(self, cache_key, value):
+        raise NotImplemented()
+
+class FileCachedSC2Factory(CachedSC2Factory):
+    def __init__(self, cache_dir, **options):
+        super(FileCachedSC2Factory, self).__init__(**options)
+        self.cache_dir = os.path.abspath(cache_dir)
+        if not os.path.isdir(self.cache_dir):
+            raise ValueError("cache_dir ({}) must be an existing directory.".format(self.cache_dir))
+        elif not os.access(self.cache_dir, os.F_OK | os.W_OK | os.R_OK ):
+            raise ValueError("Must have read/write access to {} for local file caching.".format(self.cache_dir))
+
+    def cache_has(self, cache_key):
+        print "file has",cache_key
+        return os.path.exists(self.cache_path(cache_key))
+
+    def cache_get(self, cache_key, **options):
+        print "file get",cache_key
+        return self.load_local_resource_contents(self.cache_path(cache_key),**options)
+
+    def cache_set(self, cache_key, value):
+        print "file set",cache_key
+        cache_path = self.cache_path(cache_key)
+        bucket_dir = os.path.dirname(cache_path)
+        if not os.path.exists(bucket_dir):
+            os.makedirs(bucket_dir)
+
+        with open(cache_path, 'w') as out:
+            out.write(value)
+
+    def cache_path(self, cache_key):
+        return os.path.join(self.cache_dir,*(cache_key))
+
+class DictCachedSC2Factory(CachedSC2Factory):
+    def __init__(self, cache_max_size=0, **options):
+        super(DictCachedSC2Factory, self).__init__(**options)
+        self.cache_dict = dict()
+        self.cache_used = dict()
+        self.cache_max_size = cache_max_size
+
+    def cache_set(self, cache_key, value):
+        print "dict set",cache_key
+        if self.cache_max_size and len(self.cache_dict) >= self.cache_max_size:
+            oldest_cache_key = min(self.cache_used.items(), key=lambda e: e[1])[0]
+            print "dict del", oldest_cache_key
+            del self.cache_used[oldest_cache_key]
+            del self.cache_dict[oldest_cache_key]
+        self.cache_dict[cache_key] = value
+        self.cache_used[cache_key] = time.time()
+
+    def cache_get(self, cache_key):
+        print "dict get",cache_key
+        self.cache_used[cache_key] = time.time()
+        return self.cache_dict[cache_key]
+
+    def cache_has(self, cache_key):
+        print "dict has",cache_key
+        return cache_key in self.cache_dict
+
+class DoubleCachedSC2Factory(DictCachedSC2Factory, FileCachedSC2Factory):
+
+    def __init__(self, cache_dir, cache_max_size=0, **options):
+        super(DoubleCachedSC2Factory, self).__init__(cache_max_size, cache_dir=cache_dir, **options)
+
+    def load_remote_resource_contents(self, remote_resource, **options):
+        cache_key = self.get_remote_cache_key(remote_resource)
+
+        if DictCachedSC2Factory.cache_has(self, cache_key):
+            return DictCachedSC2Factory.cache_get(self, cache_key)
+
+        if not FileCachedSC2Factory.cache_has(self, cache_key):
+            resource = SC2Factory.load_remote_resource_contents(self, remote_resource, **options)
+            FileCachedSC2Factory.cache_set(self, cache_key, resource)
         else:
-            replay = super(SC2Cache, self).load_replay(replay_file, options=options)
-            self.cache.set(replay_file, replay)
-            return replay
+            resource = FileCachedSC2Factory.cache_get(self, cache_key)
+
+        DictCachedSC2Factory.cache_set(self, cache_key, resource)
+        return resource

sc2reader/scripts/sc2parse.py

@@ -5,11 +5,13 @@
 import sc2reader
 import traceback
 
+sc2reader.useFileCache('/home/graylin/projects/sc2reader/local_cache')
+
 def main():
 	for argument in sys.argv[1:]:
 		for path in sc2reader.utils.get_files(argument):
 			try:
-				replay = sc2reader.load_replay(path, debug=True)
+				replay = sc2reader.load_replay(path, debug=True, verbose=True)
 			except sc2reader.exceptions.ReadError as e:
 				print e.replay.filename
 				print '{build} - {real_type} on {map_name} - Played {start_time}'.format(**e.replay.__dict__)
@@ -23,7 +25,12 @@ def main():
 				print path
 				replay = sc2reader.load_replay(path, debug=True, load_level=1)
 				print '{build} - {real_type} on {map_name} - Played {start_time}'.format(**replay.__dict__)
-				print '[ERROR]', e
+				print '[ERROR]', e.message
+				for pid, attributes in replay.attributes.items():
+					print pid, attributes
+				for pid, info in enumerate(replay.raw_data['replay.details'].players):
+					print pid, info
+				print replay.raw_data['replay.initData'].player_names
 				traceback.print_exc()
 				print