Fixes several minor issues, reorganizes some data, and adds realm support.

There we a couple minor syntax/naming issues from the switch back to standard
python coding conventions which were fixed.

The replay class had its players attribute restructured to have two access
points: replay.players which is an unsorted list of players, and
replay.player which is a dictionary mapping players to their pids.

Basic replay.initData parasing was implemented to gain support for realms.
This addition made it possible to fix the player urls to work across realms!

A small shell.py script was added for debugging purposes, to use it invoke
the interactive prompt with `python -i shell.py` which will execute the setup
code and bring you to the python shell for further work.

Author: GraylinKim

.gitignore

@@ -2,4 +2,5 @@
 *.pyc
 dist
 build
-sc2reader.egg-info
+sc2reader.egg-info
+PKG-INFO.txt

sc2reader/objects.py

@@ -139,12 +139,13 @@ def __repr__(self):
 
 class Player(object):
 
-    def __init__(self, pid,  data):
+    def __init__(self, pid, data, realm="us"):
         self.pid = pid
+        self.realm = realm
         self.name = data[0].decode("hex")
         self.uid = data[1][4]
         self.uidIndex = data[1][2]
-        self.url = "http://us.battle.net/sc2/en/profile/%s/%s/%s/" % (self.uid, self.uidIndex, self.name)
+        self.url = "http://%s.battle.net/sc2/en/profile/%s/%s/%s/" % (self.realm, self.uid, self.uidIndex, self.name)
         self.race = data[2].decode("hex")
         self.rgba = dict([
                 ['r', data[3][1]], 
@@ -154,6 +155,8 @@ def __init__(self, pid,  data):
             ])
         self.recorder = True
         self.handicap = data[6]
+        self.team = None # A number to be supplied later
+        self.type = "" # Human or Computer
 
     def __str__(self):
         return "Player %s - %s (%s)" % (self.pid, self.name, self.race)

sc2reader/parsers.py

@@ -1,8 +1,8 @@
 from time import ctime
 from collections import defaultdict
 
-from objects import attribute, Message, Player, Event
-from EventParsers import *
+from objects import Attribute, Message, Player, Event
+from eventparsers import *
 from utils import ByteStream
 from exceptions import ParseError
 
@@ -14,7 +14,7 @@
 #################################################
 def get_detail_parser(build):
     #This file format appears to have never changed
-    return detail_parser()
+    return DetailParser()
 
 def get_attribute_parser(build):
     if build >= 17326:
@@ -36,6 +36,29 @@ def get_message_parser(build):
     #format appears to have not changed
     return MessageParser()
 
+def get_initdata_parser(build):
+    return InitdataParser()
+    
+class InitdataParser(object):
+    def load(self,replay,filecontents):
+        bytes = ByteStream(filecontents)
+        num_players = bytes.get_big_int(1)
+        for p in range(0,num_players):
+            name_length = bytes.get_big_int(1)
+            name = bytes.get_string(name_length)
+            bytes.skip(5)
+        
+        bytes.skip(5) # Unknown
+        bytes.get_string(4) # Always Dflt
+        bytes.skip(15) #Unknown
+        id_length = bytes.get_big_int(1)
+        sc_account_id = bytes.get_string(id_length)
+        bytes.skip(684) # Fixed Length data for unknown purpose
+        while( bytes.get_string(4).lower() == 's2ma' ):
+            bytes.skip(2)
+            replay.realm = bytes.get_string(2).lower()
+            unknown_map_hash = bytes.get_big(32)
+            
 #################################################
 # replay.attributes.events Parsing classes
 #################################################
@@ -54,7 +77,7 @@ def load_attribute(self, replay, bytes):
             ]
 
         #Complete the decoding in the attribute object
-        return attribute(attr_data)
+        return Attribute(attr_data)
 
     def load(self, replay, filecontents):
         bytes = ByteStream(filecontents)
@@ -80,18 +103,17 @@ def load(self, replay, filecontents):
         #Set player attributes as available,  requires already populated player list
         for pid, attributes in data.iteritems():
             if pid == 16: continue
-            player = replay.players[pid]
+            player = replay.player[pid]
             player.color = attributes['Color']
             player.team = attributes['Teams'+replay.type]
             player.race2 = attributes['Race']
             player.difficulty = attributes['Difficulty']
-
             #Computer players can't record games
             player.type = attributes['Player Type']
             if player.type == "Computer":
                 player.recorder = False
 
-class attribute_parser_17326(attribute_parser):
+class AttributeParser_17326(AttributeParser):
 	def load_header(self, replay, bytes):
 		bytes.skip(5, byte_code=True)              #Always start with 4 nulls
 		self.count = bytes.get_little_int(4)       #get total attribute count
@@ -103,13 +125,14 @@ class DetailParser(object):
     def load(self, replay, filecontents):
         data =  ByteStream(filecontents).parse_serialized_data()
 
-        replay.players = [None] #Pad the front for proper IDs
         for pid, pdata in enumerate(data[0]):
-            replay.players.append(Player(pid+1, pdata)) #shift the id to start @ 1
+            replay.add_player(Player(pid+1, pdata, replay.realm)) #shift the id to start @ 1
 
         replay.map = data[1].decode("hex")
         replay.file_time = data[5]
         replay.date = ctime( (data[5]-116444735995904000)/10000000 )
+        
+        replay.details_data = data
 
 ##################################################
 # replay.message.events parsing classes
@@ -134,7 +157,7 @@ def load(self, replay, filecontents):
                 elif flags & 0x0F == 0:
                     bytes.skip(4)
                     if player_id < len(replay.players):
-                        replay.players[player_id].recorder = False
+                        replay.player[player_id].recorder = False
                     else:
                         pass #This "player" is an observer or something
 

sc2reader/replay.py

@@ -10,17 +10,19 @@
 class Replay(object):
 
     def __init__(self, replay, partial_parse=True, full_parse=True):
-        self.file = ""
+        self.filename = replay
         self.speed = ""
         self.release_string = ""
         self.build = ""
         self.type = ""
         self.category = ""
         self.map = ""
+        self.realm = ""
         self.events = list()
         self.results = dict()
-        self.teams = dict()
-        self.players = list()
+        self.teams = defaultdict(list)
+        self.players = list() #Unordered list of Player
+        self.player = dict() #Maps pid to Player
         self.events_by_type = dict()
         self.attributes = list()
         self.length = None # (minutes, seconds) tuple
@@ -29,44 +31,54 @@ def __init__(self, replay, partial_parse=True, full_parse=True):
         self.versions = None # (number,number,number,number) tuple
         self.recorder = None # Player object
         self.frames = None # Integer representing FPS
+
+        # Set in parsers.DetailParser.load, should we hide this?
+        self.file_time = None # Probably number milliseconds since EPOCH
+        # Marked as private in case people want raw file access
+        self._files = dict() # Files extracted from mpyq
+
+         #Used internally to ensure parse ordering
+        self.__parsed = dict(details=False, attributes=False, messages=False, events=False, initdata=False)
 
         # TODO: Change to something better
         self.date = "" # Date when the game was played
-        # TODO: hide this?
-        self.files = dict() # From the mpq lib
-        # TODO: investigate where this gets set
-        self.file_time # Probably number of seconds or milliseconds since EPOCH (the 1970 date)
-        # TODO: This is used for internal purposes, better hide it!
-        self.parsed = dict()
-        
-        #Make sure the file exists and is readable
-        if not os.access(replay, os.F_OK):
-            raise ValueError("File at '%s' cannot be found" % replay)
-        elif not os.access(replay, os.R_OK):
-            raise ValueError("File at '%s' cannot be read" % replay)
-        
-        self.file = replay
+        
+        
+        #Make sure the file exists and is readable, first and foremost
+        if not os.access(self.filename, os.F_OK):
+            raise ValueError("File at '%s' cannot be found" % self.filename)
+        elif not os.access(self.filename, os.R_OK):
+            raise ValueError("File at '%s' cannot be read" % self.filename)
+            
+        #Always parse the header first, the extract the files
         self._parse_header()
-        self.files = MPQArchive(replay).extract()
-        self.parsed = dict(details=False, attributes=False, messages=False, events=False)
+        #Extract the available file from the MPQArchive
+        self._files = MPQArchive(replay).extract()
 
         #These are quickly parsed files that contain most of the game information
         #The order is important, I need some way to reinforce it in the future
         if partial_parse or full_parse:
+            self._parse_initdata()
             self._parse_details()
             self._parse_attributes()
             self._parse_messages()
 
+        #Parsing events takes forever, so only do this on request
         if full_parse:
             self._parse_events()
 
+    
+    def add_player(self,player):
+        self.players.append(player)
+        self.player[player.pid] = player
+        
     def _parse_header(self):
         #Open up a ByteStream for its contents
-        source = ByteStream(open(self.file).read())
+        source = ByteStream(open(self.filename).read())
 
         #Check the file type for the MPQ header bytes
-        if source.getBig(4) != "4D50511B":
-            raise TypeError("File '%s' is not an MPQ file" % self.file)
+        if source.get_big(4) != "4D50511B":
+            raise TypeError("File '%s' is not an MPQ file" % self.filename)
 
         #Extract replay header data
         max_data_size = source.get_little_int(4) #possibly data max size
@@ -83,39 +95,45 @@ def _parse_header(self):
         self.frames,self.seconds = (data[3], data[3]/16)
         self.length = (self.seconds/60, self.seconds%60)
 
+    def _parse_initdata(self):
+        parsers.get_initdata_parser(self.build).load(self, self._files['replay.initData'])
+        self.__parsed['initdata'] = True
+        
     def _parse_details(self):
+        if not self.__parsed['initdata']:
+            raise ValueError("The replay initdata must be parsed before parsing details")
+            
         #Get player and map information
-        parsers.get_detail_parser(self.build).load(self, self.files['replay.details'])
-        self.parsed['details'] = True
+        parsers.get_detail_parser(self.build).load(self, self._files['replay.details'])
+        self.__parsed['details'] = True
 
     def _parse_attributes(self):
         #The details file data is required for parsing
-        if not self.parsed['details']:
+        if not self.__parsed['details']:
             raise ValueError("The replay details must be parsed before parsing attributes")
 
-        parsers.get_attribute_parser(self.build).load(self, self.files['replay.attributes.events'])
-        self.parsed['attributes'] = True
+        parsers.get_attribute_parser(self.build).load(self, self._files['replay.attributes.events'])
+        self.__parsed['attributes'] = True
 
         #We can now create teams
-        self.teams = defaultdict(list)
-        for player in self.players[1:]: #Skip the 'None' player 0
+        for player in self.players: #Skip the 'None' player 0
             self.teams[player.team].append(player)
 
     def _parse_messages(self):
         #The details file data is required for parsing
-        if not self.parsed['details']:
+        if not self.__parsed['details']:
             raise ValueError("The replay details must be parsed before parsing messages")
 
-        parsers.get_message_parser(self.build).load(self, self.files['replay.message.events'])
-        self.parsed['messages'] = True
+        parsers.get_message_parser(self.build).load(self, self._files['replay.message.events'])
+        self.__parsed['messages'] = True
 
     def _parse_events(self):
         #The details file data is required for parsing
-        if not self.parsed['details']:
+        if not self.__parsed['details']:
             raise ValueError("The replay details must be parsed before parsing events")
 
-        parsers.get_event_parser(self.build).load(self, self.files['replay.game.events'])
-        self.parsed['events'] = True
+        parsers.get_event_parser(self.build).load(self, self._files['replay.game.events'])
+        self.__parsed['events'] = True
 
         #We can now sort events by type and get results
         self.events_by_type = defaultdict(list)
@@ -126,15 +144,15 @@ def _parse_events(self):
 
     def _process_results(self):
         #The details,attributes, and events are required
-        if not (self.parsed['details'] and self.parsed['attributes'] and self.parsed['events']):
+        if not (self.__parsed['details'] and self.__parsed['attributes'] and self.__parsed['events']):
             raise ValueError("The replay details must be parsed before parsing attributes")
 
         #Remove players from the teams as they drop out of the game   
         self.results = dict([team, len(players)] for team, players in self.teams.iteritems())
         for event in self.events_by_type['leave']:
             #Some spectator actions seem to be recorded, they aren't on teams anyway
             if event.player < len(self.players):
-                team = self.players[event.player].team
+                team = self.player[event.player].team
                 self.results[team] -= 1 
 
         #mark all teams with no players left as losing, save the rest of the teams
@@ -165,7 +183,7 @@ def _process_results(self):
             if len(remaining) == 1:
                 self.results[remaining.pop()] = "Won"
 
-        for player in self.players[1:]:
+        for player in self.players:
             player.result = self.results[player.team]
 
 if __name__ == '__main__':

sc2reader/utils.py

@@ -47,7 +47,7 @@ def get_little(self, number, byte_code=False):
             return result, byte_string 
         return result
 
-    def getString(self, length, byte_code=False):
+    def get_string(self, length, byte_code=False):
         string, bytes = self.get_big(length, byte_code=True)
         if byte_code:
             return string.decode("hex"), bytes
@@ -72,7 +72,7 @@ def get_count(self, byte_code=False):
             return num/2, byte_string
         return num/2
 
-    def getTimestamp(self, byte_code=False):
+    def get_timestamp(self, byte_code=False):
         #Get the first byte
         byte, byte_string = self.get_big_int(1, byte_code=True)
 

scripts/sc2printer

@@ -1,6 +1,7 @@
 #!c:\\Python27\python.exe
 
 import os, sys
+sys.path.append(r'C:\Users\graylinkim\sc2reader')
 from sc2reader import Replay
 from sc2reader.exceptions import ParseError
 

shell.py

@@ -0,0 +1,2 @@
+from sc2reader import Replay
+from mpyq import MPQArchive

test_replays/test_all.py

@@ -3,9 +3,10 @@
 
 # TODO:
 #   - Performance tests to measure the effect of optimizations
-import os
+import os,sys
 import pytest
 
+sys.path.append(r"C:\Users\graylinkim\sc2reader")
 from sc2reader import Replay
 from sc2reader.exceptions import ParseError
 
@@ -44,7 +45,9 @@ def test_1():
     assert replay.type == "1v1"
     assert replay.category == "Ladder"
 
-#    assert len(replay.players) == 2
+    assert len(replay.players) == 2
+    assert replay.player[1].name == "Emperor"
+    assert replay.player[2].name == "Boom"
     emperor = find(lambda player: player.name == "Emperor", replay.players)
     assert emperor.team == 1
     assert emperor.race == "Protoss"
@@ -55,15 +58,15 @@ def test_1():
     assert boom.race == "Terran"
     assert boom.recorder == True
 
-    # for player in replay.players:
-    #     assert player.type == "Human"
+    for player in replay.players:
+        assert player.type == "Human"
 
     # Because it is a 1v1 and the recording player quit, we should know the winner.
     assert emperor.result == "Won"
     assert boom.result == "Lost"
 
-    # assert emperor.url == "http://eu.battle.net/sc2/en/profile/520049/1/Emperor/"
-    # assert boom.url == "http://eu.battle.net/sc2/en/profile/1694745/1/Boom/"
+    assert emperor.url == "http://eu.battle.net/sc2/en/profile/520049/1/Emperor/"
+    assert boom.url == "http://eu.battle.net/sc2/en/profile/1694745/1/Boom/"
 
     assert len(replay.messages) == 12
     assert find(lambda player: player.pid == replay.messages[0].player, replay.players).name == "Emperor"