Merge branch 'master' of http://github.com/GraylinKim/sc2reader

Author: GraylinKim

MANIFEST.in

@@ -1,3 +1,4 @@
 include LICENSE.txt
 include CONTRIBUTORS.txt
 include README.txt
+recursive-include sc2reader *.csv

sc2reader/__init__.py

@@ -1,5 +1,7 @@
 from __future__ import absolute_import
 
+import sys
+
 # import submodules
 from sc2reader import plugins, data, scripts
 
@@ -11,21 +13,35 @@
 # For backwards compatibility
 SC2Reader = factories.SC2Factory
 
-# Expose a nice module level interface
-__defaultSC2Reader = factories.SC2Factory()
+def setFactory(factory):
+    # Expose a nice module level interface
+    module = sys.modules[__name__]
+    module.load_replays = factory.load_replays
+    module.load_replay = factory.load_replay
+    module.load_maps = factory.load_maps
+    module.load_map = factory.load_map
+    module.load_game_summaries = factory.load_game_summaries
+    module.load_game_summary = factory.load_game_summary
+    module.load_map_infos = factory.load_map_infos
+    module.load_map_info = factory.load_map_info
+    module.load_map_histories = factory.load_map_headers
+    module.load_map_history = factory.load_map_header
+
+    module.configure = factory.configure
+    module.reset = factory.reset
+
+    module.register_plugin = factory.register_plugin
+    module._defaultFactory = factory
+
+def useFileCache(cache_dir, **options):
+    setFactory(factories.FileCachedSC2Factory(cache_dir, **options))
+
+def useDictCache(cache_max_size=0, **options):
+    setFactory(factories.DictCachedSC2Factory(cache_max_size, **options))
+
+def useDoubleCache(cache_dir, cache_max_size=0, **options):
+    setFactory(factories.DoubleCachedSC2Factory(cache_dir, cache_max_size, **options))
 
-load_replays = __defaultSC2Reader.load_replays
-load_replay = __defaultSC2Reader.load_replay
-load_maps = __defaultSC2Reader.load_maps
-load_map = __defaultSC2Reader.load_map
-load_game_summaries = __defaultSC2Reader.load_game_summaries
-load_game_summary = __defaultSC2Reader.load_game_summary
-load_map_infos = __defaultSC2Reader.load_map_infos
-load_map_info = __defaultSC2Reader.load_map_info
-load_map_histories = __defaultSC2Reader.load_map_headers
-load_map_history = __defaultSC2Reader.load_map_header
+setFactory(factories.SC2Factory())
 
-configure = __defaultSC2Reader.configure
-reset = __defaultSC2Reader.reset
 
-register_plugin = __defaultSC2Reader.register_plugin

sc2reader/data/__init__.py

@@ -1,11 +1,6 @@
 from __future__ import absolute_import
 
-#from sc2reader.data.utils import DataObject
-#from sc2reader.data.build16561 import Data_16561
-#from sc2reader.data.build17326 import Data_17326
-#from sc2reader.data.build18317 import Data_18317
-#from sc2reader.data.build19595 import Data_19595
-
+import pkgutil
 
 class Build(object):
     def __init__(self, build_id, units, abilities):
@@ -271,87 +266,88 @@ def __cmp__(self, other):
 class Ability(object):
     pass
 
+
 def create_build(build):
-    units_file = path.join(BASE_PATH, "{}_{}.csv".format(build,"units"))
-    abils_file = path.join(BASE_PATH, "{}_{}.csv".format(build,"abilities"))
-    with open(units_file, 'r') as data_file:
-        units = dict()
-        for row in [UnitRow(*line.strip().split('|')[1:]) for line in data_file]:
-            unit_id = int(row.id, 10) << 8 | 1
-            values = dict(cost=[0,0,0], race='Neutral',is_army=False, is_building=False, is_worker=False)
-            race, minerals, vespene, supply = "Neutral", 0, 0, 0
-            for race in ('Protoss','Terran','Zerg'):
-                if row.type.lower() in unit_lookup[race]:
-                    values.update(unit_lookup[race][row.type.lower()])
-                    values['race']=race
-                    break
-
-            units[unit_id] = type(row.title,(Unit,), dict(
-                type=unit_id,
-                name=row.title,
-                title=row.title,
-                race=values['race'],
-                minerals=values['cost'][0],
-                vespene=values['cost'][1],
-                supply=values['cost'][2],
-                is_building=values['is_building'],
-                is_worker=values['is_worker'],
-                is_army=values['is_army'],
+    units = dict()
+    units_file = "{0}_{1}.csv".format(build,"units")
+    units_data = pkgutil.get_data('sc2reader.data',units_file).split('\n')[:-1]
+    for row in [UnitRow(*line.strip().split('|')[1:]) for line in units_data]:
+        unit_id = int(row.id, 10) << 8 | 1
+        values = dict(cost=[0,0,0], race='Neutral',is_army=False, is_building=False, is_worker=False)
+        race, minerals, vespene, supply = "Neutral", 0, 0, 0
+        for race in ('Protoss','Terran','Zerg'):
+            if row.type.lower() in unit_lookup[race]:
+                values.update(unit_lookup[race][row.type.lower()])
+                values['race']=race
+                break
+
+        units[unit_id] = type(row.title,(Unit,), dict(
+            type=unit_id,
+            name=row.title,
+            title=row.title,
+            race=values['race'],
+            minerals=values['cost'][0],
+            vespene=values['cost'][1],
+            supply=values['cost'][2],
+            is_building=values['is_building'],
+            is_worker=values['is_worker'],
+            is_army=values['is_army'],
+        ))
+
+        if row.title.lower() in ('probe','zealot','stalker','immortal','phoenix','hightemplar','warpprism','archon','colossus','voidray'):
+            units[unit_id+1] = type("Hallucinated"+row.title,(Unit,), dict(
+                type=unit_id+1,
+                name="Hallucinated"+row.title,
+                title="Hallucinated"+row.title,
+                race='Protoss',
+                minerals=0,
+                vespene=0,
+                supply=0,
+                is_building=False,
+                is_army=True,
+                is_worker=False,
             ))
 
-            if row.title.lower() in ('probe','zealot','stalker','immortal','phoenix','hightemplar','warpprism','archon','colossus','voidray'):
-                units[unit_id+1] = type("Hallucinated"+row.title,(Unit,), dict(
-                    type=unit_id+1,
-                    name="Hallucinated"+row.title,
-                    title="Hallucinated"+row.title,
-                    race='Protoss',
-                    minerals=0,
-                    vespene=0,
-                    supply=0,
-                    is_building=False,
-                    is_army=True,
-                    is_worker=False,
-                ))
-
-
-    with open(abils_file, 'r') as data_file:
-        abilities = {0:type('RightClick',(Ability,), dict(type=0, name='RightClick', title='Right Click', is_build=False, build_time=None, build_unit=None))}
-        for row in [line.strip().split('|') for line in data_file]:
-            base = int(row[1],10) << 5
-            if base == 0: continue
-
-            # Temporary Hack here.
-            if base == 0xe80:
-                real_abils = [(0xe80,"QueueCancel0"), (0xe81,"QueueCancel1")]
-            else:
-                real_abils = [(base|i,t) for i,t in enumerate(row[3:]) if t.strip()!='']
-
-            for abil_id, title in real_abils:
-                abilities[abil_id] = type(title,(Ability,), dict(
-                    type=abil_id,
-                    name=title,
-                    title=title,
-                    is_build=False,
-                    build_time=None,
-                    build_unit=None
-                ))
-
-
-            # Some abilities have missing entries..
-            if len(real_abils) == 0:
-                abilities[base] = type(row[2],(Ability,), dict(
-                    type=base,
-                    name=row[2],
-                    title=row[2],
-                    is_build=False,
-                    build_time=None,
-                    build_unit=None
-                ))
-
-            if int(row[1],10) == 249 and build==22612:
-                pass
-                #print row
-                #print abilities[0x1f20], abilities[0x1f21], abilities[0x1f22], abilities[0x1f23]
+
+    abils_file = "{0}_{1}.csv".format(build,"abilities")
+    abils_data = pkgutil.get_data('sc2reader.data',abils_file).split('\n')[:-1]
+    abilities = {0:type('RightClick',(Ability,), dict(type=0, name='RightClick', title='Right Click', is_build=False, build_time=None, build_unit=None))}
+    for row in [line.strip().split('|') for line in abils_data]:
+        base = int(row[1],10) << 5
+        if base == 0: continue
+
+        # Temporary Hack here.
+        if base == 0xe80:
+            real_abils = [(0xe80,"QueueCancel0"), (0xe81,"QueueCancel1")]
+        else:
+            real_abils = [(base|i,t) for i,t in enumerate(row[3:]) if t.strip()!='']
+
+        for abil_id, title in real_abils:
+            abilities[abil_id] = type(title,(Ability,), dict(
+                type=abil_id,
+                name=title,
+                title=title,
+                is_build=False,
+                build_time=None,
+                build_unit=None
+            ))
+
+
+        # Some abilities have missing entries..
+        if len(real_abils) == 0:
+            abilities[base] = type(row[2],(Ability,), dict(
+                type=base,
+                name=row[2],
+                title=row[2],
+                is_build=False,
+                build_time=None,
+                build_unit=None
+            ))
+
+        if int(row[1],10) == 249 and build==22612:
+            pass
+            #print row
+            #print abilities[0x1f20], abilities[0x1f21], abilities[0x1f22], abilities[0x1f23]
 
     data = Build(build, units, abilities)
     for unit in units.values():

sc2reader/factories.py

@@ -219,28 +219,101 @@ def _load_resource(self, resource, options=None, **new_options):
         return (resource, resource_name)
 
 
-class SC2Cache(SC2Factory):
+import urlparse, time
+
+class CachedSC2Factory(SC2Factory):
+
+    def get_remote_cache_key(self, remote_resource):
+        # Strip the port and use the domain as the bucket
+        # and use the full path as the key
+        parseresult = urlparse.urlparse(remote_resource)
+        bucket = re.sub(r':.*', '', parseresult.netloc)
+        key = parseresult.path.strip('/')
+        return (bucket, key)
+
+    def load_remote_resource_contents(self, remote_resource, **options):
+        cache_key = self.get_remote_cache_key(remote_resource)
+        if not self.cache_has(cache_key):
+            resource = super(CachedSC2Factory, self).load_remote_resource_contents(remote_resource, **options)
+            self.cache_set(cache_key, resource)
+        else:
+            resource = self.cache_get(cache_key)
+        return resource
 
-    def __init__(self, **options):
-        super(SC2Cache, self).__init__(self, **options)
-        self.cache = IntitializeCache(**options)
+    def cache_has(self, cache_key):
+        raise NotImplemented()
 
-    def load_map(self, map_file, options=None, **new_options):
-        options = options or utils.merged_dict(self.options, new_options)
+    def cache_get(self, cache_key):
+        raise NotImplemented()
 
-        if self.cache.has(map_file):
-            return self.cache.get(map_file)
-        else:
-            map = super(SC2Cache, self).load_map(map_file, options=options)
-            self.cache.set(map_file, map)
-            return map
+    def cache_set(self, cache_key, value):
+        raise NotImplemented()
+
+class FileCachedSC2Factory(CachedSC2Factory):
+    def __init__(self, cache_dir, **options):
+        super(FileCachedSC2Factory, self).__init__(**options)
+        self.cache_dir = os.path.abspath(cache_dir)
+        if not os.path.isdir(self.cache_dir):
+            raise ValueError("cache_dir ({}) must be an existing directory.".format(self.cache_dir))
+        elif not os.access(self.cache_dir, os.F_OK | os.W_OK | os.R_OK ):
+            raise ValueError("Must have read/write access to {} for local file caching.".format(self.cache_dir))
+
+    def cache_has(self, cache_key):
+        return os.path.exists(self.cache_path(cache_key))
+
+    def cache_get(self, cache_key, **options):
+        return self.load_local_resource_contents(self.cache_path(cache_key),**options)
+
+    def cache_set(self, cache_key, value):
+        cache_path = self.cache_path(cache_key)
+        bucket_dir = os.path.dirname(cache_path)
+        if not os.path.exists(bucket_dir):
+            os.makedirs(bucket_dir)
+
+        with open(cache_path, 'w') as out:
+            out.write(value)
+
+    def cache_path(self, cache_key):
+        return os.path.join(self.cache_dir,*(cache_key))
 
-    def load_replay(self, replay_file, options=None, **new_options):
-        options = options or utils.merged_dict(self.options, new_options)
+class DictCachedSC2Factory(CachedSC2Factory):
+    def __init__(self, cache_max_size=0, **options):
+        super(DictCachedSC2Factory, self).__init__(**options)
+        self.cache_dict = dict()
+        self.cache_used = dict()
+        self.cache_max_size = cache_max_size
 
-        if self.cache.has(replay_file):
-            return self.cache.get(replay_file)
+    def cache_set(self, cache_key, value):
+        if self.cache_max_size and len(self.cache_dict) >= self.cache_max_size:
+            oldest_cache_key = min(self.cache_used.items(), key=lambda e: e[1])[0]
+            del self.cache_used[oldest_cache_key]
+            del self.cache_dict[oldest_cache_key]
+        self.cache_dict[cache_key] = value
+        self.cache_used[cache_key] = time.time()
+
+    def cache_get(self, cache_key):
+        self.cache_used[cache_key] = time.time()
+        return self.cache_dict[cache_key]
+
+    def cache_has(self, cache_key):
+        return cache_key in self.cache_dict
+
+class DoubleCachedSC2Factory(DictCachedSC2Factory, FileCachedSC2Factory):
+
+    def __init__(self, cache_dir, cache_max_size=0, **options):
+        super(DoubleCachedSC2Factory, self).__init__(cache_max_size, cache_dir=cache_dir, **options)
+
+    def load_remote_resource_contents(self, remote_resource, **options):
+        cache_key = self.get_remote_cache_key(remote_resource)
+
+        if DictCachedSC2Factory.cache_has(self, cache_key):
+            return DictCachedSC2Factory.cache_get(self, cache_key)
+
+        if not FileCachedSC2Factory.cache_has(self, cache_key):
+            resource = SC2Factory.load_remote_resource_contents(self, remote_resource, **options)
+            FileCachedSC2Factory.cache_set(self, cache_key, resource)
         else:
-            replay = super(SC2Cache, self).load_replay(replay_file, options=options)
-            self.cache.set(replay_file, replay)
-            return replay
+            resource = FileCachedSC2Factory.cache_get(self, cache_key)
+
+        DictCachedSC2Factory.cache_set(self, cache_key, resource)
+        return resource