Work in Progress: sc2reader rewrite.

This changeset represents a significant restructuring that should make options
easier to handle, improve debugging, and provide a highly literate base of
source code for understanding the replay format and the intricacies of parsing
it into a workable object tree.

TODO: use SC2ReaderError exceptions, comment all files, finish processors.

Author: GraylinKim

sc2reader/__init__.py

@@ -1,70 +1,173 @@
-import os, copy
+# -*- coding: utf-8 -*-
+"""
+    sc2reader
+    ----------
 
-import mpyq
+    A Starcraft II replay parsing library intended to promote innovation in
+    Starcraft tools and communities. Eventually, it will cover all official
+    releases and dump easily to JSON for inter-language portabilty.
 
-import config
-from objects import Replay
-from utils import ReplayBuffer, AttributeDict
+    sc2reader has been carefully written for clarity to serve as a starting
+    point for those who want to write their own parsers, potentially in other
+    languages for native access.
 
-__version__ = "0.3.0"
-__author__ = "Graylin Kim <[email protected]>"
+    Enjoy.
 
+    :copyright: (c) 2011 Graylin Kim
+    :license: MIT, See LICENSE.txt for details
+"""
 
-class SC2Reader(object):
-    def __init__(self, **options):
-        #Set Defaults before configuring with user options
-        self.options = AttributeDict(
-            directory="",
-            processors=[],
-            debug=False,
-            verbose=False,
-            parse_events=True)
-        self.configure(**options)
+__version__ = '0.3-dev'
 
-    def read(self, location):
-        if self.options.directory:
-            location = os.path.join(self.options.directory,location)
+#System imports
+import os
 
-        if self.options.verbose: print "Reading: %s" % location
+#PyPi imports
+import mpyq
 
-        if os.path.isdir(location):
-            #SC2Reader::read each subfile/directory and combine the lists
-            read = lambda file: self.read(os.path.join(location,file))
-            tolist = lambda x: [x] if isinstance(x,Replay) else x
-            return sum(map(tolist,(read(x) for x in os.listdir(location))),[])
+#Package imports
+import config
+import objects
+import utils
 
-        with open(location) as replay_file:
-            replay = Replay(replay_file,**self.options.copy())
-            archive = mpyq.MPQArchive(location,listfile=False)
 
-            for file in self.files:
-                buffer = ReplayBuffer(archive.read_file(file))
-                read = config.readers[replay.build][file]
-                read(buffer,replay)
+class Reader(object):
+    """ The SC2Reader class acts as a factory class for replay objects. The
+        class accepts a key=value list of options to override defaults (see
+        config.py) and exposes a very simple read/configure interface and
+        orchestrates the replay build process.
+    """
 
-            #Handle user processors after internal processors
-            for process in self.processors+self.options.processors:
-                replay = process(replay)
 
-            return replay
+    def __init__(self, **user_options):
+        """ The constructor makes a copy of the default_options to make sure the
+            option configuration doesn't propogate back to the default_options.
+            It should support any arbitrary number of different Reader objects.
+        """
+        self.options = config.default_options.copy()
+        self.configure(**user_options)
 
     def configure(self,**options):
         self.options.update(options)
 
-        #Update system configuration
-        myconfig = config.full if self.options.parse_events else config.partial
-        self.files = myconfig.files
-        self.processors = myconfig.processors
+        # Depending on the options choosen, the system needs to update related
+        # options and setting in order to get the reading right.
+        self.sys = config.full if self.options.parse_events else config.partial
+
+    def read(self, location, **user_options):
+        """ Read indicated file or recursively read matching files from the
+            specified directory. Returns a replay or a list of replays depending
+            on the context.
+        """
+
+        # Base the options off a copy to leave the Reader options uneffected.
+        options = self.options.copy()
+        options.update(user_options)
+
+        # The directory option allows users to specify file locations relative
+        # to a location other than the present working directory by joining the
+        # location with the directory of their choice.
+        if options.directory:
+            location = os.path.join(options.directory,location)
+
+        # When passed a directory as the location, the Reader recursively builds
+        # a list of replays to return using the utils.get_files function. This
+        # function respects the following arguments:
+        #   * depth: The maximum depth to traverse. Defaults to unlimited (-1)
+        #   * follow_symlinks: Boolean for following symlinks. Defaults to True
+        #   * exclude_dirs: A list of directory names to skip while recursing
+        #   * incldue_regex: A regular expression rule which all returned file
+        #       names must match. Defaults to None
+        #
+        replays, files = list(), utils.get_files(location,**options)
+
+        # If no files are found, it could be for a variety of reasons
+        # raise a NoMatchingFilesError to alert them to the situation
+        if not files:
+            raise NoMatchingFilesError()
+
+        for location in files:
+            if options.verbose: print "Reading: %s" % location
+
+            with open(location) as replay_file:
+                # The Replay constructor scans the header of the replay file for
+                # the build number and stores the options for later use. The
+                # options are copied so subsequent option changes are isolated.
+                replay = objects.Replay(replay_file,**options.copy())
+
+                # .SC2Replay files are written in Blizzard's MPQ Archive format.
+                # The format stores a header which contains a block table that
+                # specifies the location of each encrypted file.
+                #
+                # Unfortunately, some replay sites modify the replay contents to
+                # add messages promoting their sites without updating the header
+                # correctly. The listfile option(hack) lets us bypass this issue
+                # by specifying the files we want instead of generating a list.
+                archive = mpyq.MPQArchive(location,listfile=False)
+
+                # These files are configured for either full or partial parsing
+                for file in self.sys.files:
+
+                    # For each file, we build a smart buffer object from the
+                    # utf-8 encoded bitstream that mpyq extracts.
+                    buffer = utils.ReplayBuffer(archive.read_file(file))
+
+                    # Each version of Starcraft slightly modifies some portions
+                    # of the format for some files. To work with this, the
+                    # config file has a nested lookup structure of
+                    # [build][file]=>reader which returns the appropriate reader
+                    #
+                    # TODO: Different versions also have different data mappings
+                    #       sc2reader doesn't yet handle this difficulty.
+                    #
+                    # Readers use the type agnostic __call__ interface so that
+                    # they can be implemented as functions or classes as needed.
+                    #
+                    # Readers store the extracted information into the replay
+                    # object for post processing because correct interpretation
+                    # of the information often requires data from other files.
+                    config.readers[replay.build][file].__call__(buffer,replay)
+
+                # Now that the replay has been loaded with the "raw" data from
+                # the archive files we run the system level post processors to
+                # organize the data into a cross referenced data structure.
+                #
+                # After system level processors have run, call each of the post
+                # processors provided by the user. This would be a good place to
+                # convert the object to a serialized json string for cross
+                # language processes or add custom attributes.
+                #
+                # TODO: Maybe we should switch this to a hook based architecture
+                #       Needs to be able to load "contrib" type processors..
+                for process in self.sys.processors+self.options.processors:
+                    replay = process(replay)
+
+                replays.append(replay)
+
+        return replays
+
+    def read_file(file,**options):
+        replays = self.read(file,**options)
+
+        # While normal usage would suggest passing in only filenames, it is
+        # possible that directories could be passed in. Don't fail silently!
+        if len(replays) > 1:
+            raise MultipleMatchError(replays)
+
+        # Propogate the replay in a singular context
+        return replays[0] if len(replays) > 0 else None
+
+"""sc2reader uses a default SC2Reader class instance to provide a package level
+interface to its functionality. The package level interface presents the same
+functional interface, it just saves the hassel of creating the class object.
+"""
+__defaultReader = Reader()
+
+def read(location, **user_options):
+    return __defaultReader.read(location,**user_options)
 
-
-'''Package Level Interface'''
-__defaultSC2Reader = SC2Reader()
-
-#Allow options on the package level read for one off reads.
-def read(location, **options):
-    reader = SC2Reader(**options) if options else __defaultSC2Reader
-    return reader.read(location)
-
-#Allow package level configuration for lazy people
 def configure(**options):
-    __defaultSC2Reader.configure(**options)
+    config.default_options.update(options)
+
+def reset():
+    __defaultReader = Reader()

sc2reader/config.py

@@ -1,7 +1,19 @@
-from utils import AttributeDict
+from .processors import *
+from .readers import *
+from .utils import AttributeDict
 
-from processors import *
-from readers import *
+default_options = AttributeDict(
+    directory="",
+    processors=[],
+    debug=False,
+    verbose=False,
+    parse_events=True,
+    include_regex=None,
+    exclude_dirs=[],
+    recursive=True,
+    depth=-1,
+    follow_symlinks=True
+)
 
 full = AttributeDict(
     files = [
@@ -11,14 +23,16 @@
         'replay.message.events',
         'replay.game.events'],
     processors = [
-        PeopleProcessor,
-        AttributeProcessor,
-        TeamsProcessor,
-        MessageProcessor,
-        RecorderProcessor,
-        EventProcessor,
-        ApmProcessor,
-        ResultsProcessor],
+        FullProcessor,
+        #PeopleProcessor,
+        #AttributeProcessor,
+        #TeamsProcessor,
+        #MessageProcessor,
+        #RecorderProcessor,
+        #EventProcessor,
+        #ApmProcessor,
+        #ResultsProcessor
+    ],
 )
 
 partial = AttributeDict(