first pass cleaning up Exception messaging before saving to Redis

thanks to @brianmario for the code to do the UTF8 sanitization.

Author: hone

lib/resque.rb

@@ -13,6 +13,8 @@
 require 'resque/worker'
 require 'resque/plugin'
 
+require 'resque/vendor/utf8_util'
+
 module Resque
   include Helpers
   extend self

lib/resque/failure/redis.rb

@@ -8,7 +8,7 @@ def save
           :failed_at => Time.now.strftime("%Y/%m/%d %H:%M:%S %Z"),
           :payload   => payload,
           :exception => exception.class.to_s,
-          :error     => exception.to_s,
+          :error     => UTF8Util.clean(exception.to_s),
           :backtrace => filter_backtrace(Array(exception.backtrace)),
           :worker    => worker.to_s,
           :queue     => queue

lib/resque/vendor/utf8_util.rb

@@ -0,0 +1,26 @@
+module UTF8Util
+  # use '?' intsead of the unicode replace char, since that is 3 bytes
+  # and can increase the string size if it's done a lot
+  REPLACEMENT_CHAR = "?"
+
+  # Replace invalid UTF-8 character sequences with a replacement character
+  #
+  # Returns self as valid UTF-8.
+  def self.clean!(str)
+    raise NotImplementedError
+  end
+
+  # Replace invalid UTF-8 character sequences with a replacement character
+  #
+  # Returns a copy of this String as valid UTF-8.
+  def self.clean(str)
+    clean!(str.dup)
+  end
+
+end
+
+if RUBY_VERSION <= '1.9'
+  require 'resque/vendor/utf8_util/utf8_util_18'
+else
+  require 'resque/vendor/utf8_util/utf8_util_19'
+end

lib/resque/vendor/utf8_util/utf8_util_18.rb

@@ -0,0 +1,91 @@
+require 'strscan'
+
+module UTF8Util
+  HIGH_BIT_RANGE = /[\x80-\xff]/
+
+  # Check if this String is valid UTF-8
+  #
+  # Returns true or false.
+  def self.valid?(str)
+    sc = StringScanner.new(str)
+
+    while sc.skip_until(HIGH_BIT_RANGE)
+      sc.pos -= 1
+
+      if !sequence_length(sc)
+        return false
+      end
+    end
+
+    true
+  end
+
+  # Replace invalid UTF-8 character sequences with a replacement character
+  #
+  # Returns self as valid UTF-8.
+  def self.clean!(str)
+    sc = StringScanner.new(str)
+    while sc.skip_until(HIGH_BIT_RANGE)
+      pos = sc.pos = sc.pos-1
+
+      if !sequence_length(sc)
+        str[pos] = REPLACEMENT_CHAR
+      end
+    end
+
+    str
+  end
+
+  # Validate the UTF-8 sequence at the current scanner position.
+  #
+  # scanner - StringScanner instance so we can advance the pointer as we verify.
+  #
+  # Returns The length in bytes of this UTF-8 sequence, false if invalid.
+  def self.sequence_length(scanner)
+    leader = scanner.get_byte[0]
+
+    if (leader >> 5) == 0x6
+      if check_next_sequence(scanner)
+        return 2
+      else
+        scanner.pos -= 1
+      end
+    elsif (leader >> 4) == 0x0e
+      if check_next_sequence(scanner)
+        if check_next_sequence(scanner)
+          return 3
+        else
+          scanner.pos -= 2
+        end
+      else
+        scanner.pos -= 1
+      end
+    elsif (leader >> 3) == 0x1e
+      if check_next_sequence(scanner)
+        if check_next_sequence(scanner)
+          if check_next_sequence(scanner)
+            return 4
+          else
+            scanner.pos -= 3
+          end
+        else
+          scanner.pos -= 2
+        end
+      else
+        scanner.pos -= 1
+      end
+    end
+
+    false
+  end
+
+  private
+
+  # Read another byte off the scanner oving the scan position forward one place
+  #
+  # Returns nothing.
+  def self.check_next_sequence(scanner)
+    byte = scanner.get_byte[0]
+    (byte >> 6) == 0x2
+  end
+end

lib/resque/vendor/utf8_util/utf8_util_19.rb

@@ -0,0 +1,5 @@
+module UTF8Util
+  def self.clean!(str)
+    str.force_encoding("binary").encode("UTF-8", :invalid => :replace, :undef => :replace, :replace => REPLACEMENT_CHAR)
+  end
+end