Commit a game events debugging script and experimental fixes.

Author: GraylinKim

game_events_debug.py

@@ -0,0 +1,80 @@
+""" Game Event Parsing Debugger
+
+    Recursively searches for unique replays specified by command line argument
+    and attempts a parse in order to catch the ReadError. The ReadError will
+    contain all the context information we need to inspect the error.
+
+    Outputs the following information in the following format:
+
+        ggtracker/ggtracker_replay_202597.SC2Replay
+          ....... 00:04 P2 AbilityEvent        - 00 22 0b 01 08 34 07 00 31 00 00 01 1f 0d 21 f8 00 8c c0 80 00 05 ff 00
+          ....... 00:05 P1 SelectionEvent      - 38 21 ac 00 06 00 01 05 15 01 03 03 05 18 00 01 05 20 00 01 05 28 00 01
+          0x000F5 00:05 P2 SelectionEvent      - 00 22 ac 00 02 f4 00 00  | 00 0c a1 89 00 10 00 00 00 21 0b 01 00 01 5c 01 00 0c 22 0b
+                                                 01 08 34 07 00 23 80 00 01 1f 0d 41 18 00 8a c0 80 00 05 ff 00 08 21 ac 00 02 11 01 02
+
+    The first line is the full path to the file followed by an indented list
+    of the last 2 events to be parsed with their associated bytes.
+
+    The 3rd line for the event the parser failed on is in the following format:
+
+        cursor event_type - event_bytes | following bytes
+                            and the next bytes after that
+
+    With a pipe character delimiting where the parser ended the event.
+
+    The general workflow is something like this:
+
+        1) Run the debugger over a set of replays
+        2) Figure out where the failed event boundry should have been
+        3) Alter the reader code to make it happen
+        4) Repeat
+
+    To figure out how to alter the reader code you'll probably need to compile
+    lists of failed events with the correct bytes and look for patterns that
+    would help you read them the right way. Sometimes it also helps to think
+    about what that event represents and what would make its content length
+    vary. Cross checking against a replay can also be useful.
+
+    When altering the reader code, NEVER change code in an older game events
+    reader unless you really really know what you are doing. Instead copy the
+    function you'd like to change into the GameEventsReader_22612 class and
+    make your changes there.
+"""
+import sys
+import sc2reader
+from sc2reader.exceptions import ReadError
+from sc2reader.utils import get_files
+
+BYTES_WIDTH = 45
+
+def format_bytes(bytes):
+    """I find it easier to read hex when spaced by byte"""
+    bytes = bytes.encode("hex")
+    ret = ""
+    for i in range(len(bytes)):
+        if i%2:
+            ret += " "
+        else:
+            ret += bytes[i:i+2]
+    return ret
+
+def get_name(obj):
+    return obj.__class__.__name__
+
+for filename in set(sum((list(get_files(arg)) for arg in sys.argv[1:]),list())):
+    try:
+        replay = sc2reader.load_replay(filename, debug=True)
+    except ReadError as e:
+        print filename
+        if len(e.game_events):
+            for event in e.game_events[-3:-1]:
+                print "  ....... {:0>2}:{:0>2} P{} {: <19} - {}".format(event.second/60, event.second%60, event.pid, get_name(event), format_bytes(event.bytes))
+            last_event = e.game_events[-1]
+            line1_end = e.location+BYTES_WIDTH-len(last_event.bytes)-1
+            line1 = e.buffer.read_range(e.location, line1_end)
+            line2 = e.buffer.read_range(line1_end,line1_end+BYTES_WIDTH)
+            print "  0x{:0>5X} {:0>2}:{:0>2} P{} {: <19} - {} | {}".format(e.location, last_event.second/60, last_event.second%60, last_event.pid, get_name(last_event), format_bytes(last_event.bytes), format_bytes(line1))
+            print " "*41+format_bytes(line2)
+        else:
+            print "  0x{:0>5X} {: <19} - {}".format(0, "None", format_bytes(e.buffer.read_range(0,BYTES_WIDTH+1)))
+        print

sc2reader/readers.py

@@ -203,7 +203,8 @@ def __call__(self, buffer, replay):
             0x01: self.get_action_parser,
             0x02: self.get_unknown2_parser,
             0x03: self.get_camera_parser,
-            0x04: self.get_unknown4_parser
+            0x04: self.get_unknown4_parser,
+            0x05: self.get_unknown5_parser
         }
 
         game_events, frames = list(), 0
@@ -275,7 +276,7 @@ def get_setup_parser(self, code):
         # the game starting immediately afterwords. On occasion, for unknown
         # reasons, other events (particularly camera events) will register
         # before the game has actually started. Weird.
-        if   code in (0x0B, 0x0C, 0x2C): return self.parse_join_event
+        if   code in (0x0B, 0x0C, 0x1C, 0x2C): return self.parse_join_event
         elif code in (0x05,): return self.parse_start_event
         else:
             raise ParseError("Unknown Setup Parser Code {0}".format(code))
@@ -329,6 +330,9 @@ def get_unknown4_parser(self, code):
         else:
             raise ParseError("Unknown Unknown4 Parser Code {0}".format(code))
 
+    def get_unknown5_parser(self, code):
+        return self.parse_05XX_event
+
     def parse_join_event(self, buffer, frames, type, code, pid):
         return PlayerJoinEvent(frames, pid, type, code)
 
@@ -742,3 +746,109 @@ def right_click_target(self, buffer, frames, type, code, pid, flag, atype):
         target = (buffer.read_int(BIG_ENDIAN), buffer.read_short(BIG_ENDIAN))
         buffer.skip(10)
         return TargetAbilityEvent(frames, pid, type, code, ability, target)
+
+class GameEventsReader_22612(GameEventsReader_19595):
+    def parse_join_event(self, buffer, frames, type, code, pid):
+        buffer.read_byte()
+        return PlayerJoinEvent(frames, pid, type, code)
+
+    def cancel(self, buffer, frames, type, code, pid, flag, atype):
+        ability = buffer.read_short(endian=BIG_ENDIAN)
+        ability = ability << 8 | buffer.read_byte()
+
+        # creation autoid number / object id
+        created_id = buffer.read_object_id()
+        # TODO : expose the id
+        buffer.read_byte()
+        return AbilityEvent(frames, pid, type, code, ability)
+
+    def parse_ability_event(self, buffer, frames, type, code, pid):
+        """Moves the right click move to the top level"""
+        flag = buffer.read_byte()
+        atype = buffer.read_byte()
+
+        if atype & 0x20: # cancels only now
+            return self.cancel(buffer, frames, type, code, pid, flag, atype)
+        elif atype & 0x40: # all command card abilities?
+            return self.command_card(buffer, frames, type, code, pid, flag, atype)
+        elif atype & 0x80: # right-click on target?
+            return self.right_click_target(buffer, frames, type, code, pid, flag, atype)
+        elif atype < 0x10: #new to patch 1.3.3, location now??
+            return self.right_click_move(buffer, frames, type, code, pid, flag, atype)
+
+        raise ParseError()
+
+    def right_click_move(self, buffer, frames, type, code, pid, flag, atype):
+        # This may port back to previous versions. Haven't checked
+        # Can differ by up to (+/-1, +/-8) from sc2gears readings
+        # See command_card implementation for details
+        if atype in (0x00,0x02):
+            buffer.skip(3)
+            while buffer.read_byte() & 0xF0 != 0:
+                buffer.skip(8)
+            return AbilityEvent(frames,pid,type,code,0x3601)
+
+
+        else: # atype == 0x08:
+            #!?!?!?!?!?
+            buffer.skip(6)
+            if buffer.read_byte() == 0x00:
+                buffer.skip(3)
+            else:
+                buffer.skip(12)
+            """
+            buffer.skip(8)
+            if buffer.read_byte() == 0x0D:
+                buffer.skip(9)
+            buffer.read_byte()
+            """
+            return AbilityEvent(frames,pid,type,code,0x3601)
+
+
+        """
+        x = buffer.read_short(BIG_ENDIAN)/256.0
+        buffer.shift(5) # what is this for, why 5 bits instead of 4?
+        y = buffer.read_short(BIG_ENDIAN)/256.0
+        buffer.read(bits=5) # I'll just assume we should do it again
+        """
+
+        #buffer.skip(19)
+
+        return AbilityEvent(frames, pid, type, code, 0x3601)
+
+    def parse_05XX_event(self, buffer, frames, type, code, pid):
+        buffer.skip(4)
+        return UnknownEvent(frames, pid, type, code)
+
+    def parse_overlay(self, buffer, mode):
+        mode = mode >> 1
+        if mode == 0x01: # deselect overlay mask
+            data = buffer.read_bitmask()
+        elif mode == 0x02: # deselect mask
+            data = [buffer.read_byte() for i in range(buffer.read_byte())]
+        elif mode == 0x03: # replace mask
+            data = [buffer.read_byte() for i in range(buffer.read_byte())]
+        else:
+            data=None
+
+        buffer.align()
+
+        return mode, data
+
+    def parse_selection_event(self, buffer, frames, type, code, pid):
+
+        bank = code >> 4
+        first = buffer.read_byte() # TODO ?
+
+        deselect = self.parse_overlay(buffer, buffer.shift(4))
+
+        # <count> (<type_id>, <count>,)*
+        object_types = [ (buffer.read_object_type(read_modifier=True), buffer.read_byte(), ) for i in range(buffer.read_byte()) ]
+        # <count> (<object_id>,)*
+        object_ids = [ buffer.read_object_id() for i in range(buffer.read_byte()) ]
+
+        # repeat types count times
+        object_types = chain(*[[object_type,]*count for (object_type, count) in object_types])
+        objects = zip(object_ids, object_types)
+
+        return SelectionEvent(frames, pid, type, code, bank, objects, deselect)

sc2reader/resources.py

@@ -512,7 +512,8 @@ def register_default_readers(self):
         self.register_reader('replay.game.events', readers.GameEventsReader_Base(), lambda r: r.build <  16561)
         self.register_reader('replay.game.events', readers.GameEventsReader_16561(), lambda r: 16561 <= r.build < 18574)
         self.register_reader('replay.game.events', readers.GameEventsReader_18574(), lambda r: 18574 <= r.build < 19595)
-        self.register_reader('replay.game.events', readers.GameEventsReader_19595(), lambda r: 19595 <= r.build)
+        self.register_reader('replay.game.events', readers.GameEventsReader_19595(), lambda r: 19595 <= r.build < 22612)
+        self.register_reader('replay.game.events', readers.GameEventsReader_22612(), lambda r: 22612 <= r.build)
 
     def register_default_datapacks(self):
         """Registers factory default datapacks."""