Improve test case for resque#638 using redis' blocking list operations,

ensure job is hard-killed when it doesn't die gracefully in the specified time

Author: yaauie

test/worker_test.rb

@@ -438,64 +438,71 @@ def self.perform
     assert_not_equal original_connection, Resque.redis.client.connection.instance_variable_get("@sock")
   end
 
-
-  test 'SIGTERM' do
-    begin
-      class LongRunningJob
-        @queue = :long_running_job
-        def self.perform(time)
-          $child_pid_writer.write Process.pid
-          $child_pid_writer.close
-          sleep(time)
-        rescue SignalException => e
-          $child_worker_message_writer.write %Q(SignalException caught! #{e.inspect})
-          $child_worker_message_writer.close
+  {
+    'cleanup occurs in allotted time' => nil,
+    'cleanup takes too long' => 2
+  }.each do |scenario,rescue_time|
+    test "SIGTERM when #{scenario}" do
+      begin
+        class LongRunningJob
+          @queue = :long_running_job
+          def self.perform( run_time, rescue_time=nil )
+            Resque.redis.client.reconnect # get its own connection
+            Resque.redis.rpush( 'sigterm-test:start', Process.pid )
+            sleep run_time
+            Resque.redis.rpush( 'sigterm-test:result', 'Finished Normally' )
+          rescue SignalException => e
+            Resque.redis.rpush( 'sigterm-test:result', %Q(Caught SignalException: #{e.inspect}))
+            sleep rescue_time unless rescue_time.nil?
+          ensure
+            Resque.redis.rpush( 'sigterm-test:final', 'exiting.' )
+          end
         end
-      end
 
-      $child_pid_reader, $child_pid_writer = IO.pipe
-      $child_worker_message_reader, $child_worker_message_writer = IO.pipe
+        Resque.enqueue( LongRunningJob, 5, rescue_time )
 
-      Resque.enqueue(LongRunningJob, 2)
+        worker_pid = Kernel.fork do
+          # ensure we actually fork
+          $TESTING = false 
+          # reconnect since we just forked
+          Resque.redis.client.reconnect
 
-      worker_pid = Kernel.fork do
-        # don't hold up the read-end of the pipe
-        $child_pid_reader.close
-        # ensure we actually fork
-        $TESTING = false
-        # reconnect since we just forked
-        Resque.redis.client.reconnect
+          worker = Resque::Worker.new(:long_running_job)
+          worker.term_timeout = 1
 
-        worker = Resque::Worker.new(:long_running_job)
-        worker.term_timeout = 1
+          worker.work(0)
+          exit!
+        end
 
-        worker.work(0)
-        exit!
+        # ensure the worker is started
+        start_status = Resque.redis.blpop( 'sigterm-test:start', 5 )
+        assert_not_nil start_status
+        child_pid = start_status[1].to_i
+        assert_operator child_pid, :>, 0
+
+        # send signal to abort the worker
+        Process.kill('TERM', worker_pid)
+        Process.waitpid(worker_pid)
+
+        # wait to see how it all came down
+        result = Resque.redis.blpop( 'sigterm-test:result', 5 )
+        assert_not_nil result
+        assert !result[1].start_with?('Finished Normally'), 'Job Finished normally. Sleep not long enough?'
+        assert result[1].start_with? 'Caught SignalException', 'Signal exception not raised in child.'
+
+        # ensure that the child pid is no longer running
+        child_still_running = !(`ps -p #{child_pid.to_s} -o pid=`).empty?
+        assert !child_still_running
+
+        # see if post-cleanup occurred. This should happen IFF the rescue_time is less than the term_timeout
+        post_cleanup_occurred = Resque.redis.lpop( 'sigterm-test:final' )
+        assert post_cleanup_occurred, 'post cleanup did not occur. SIGKILL sent too early?' if rescue_time.nil?
+        assert !post_cleanup_occurred, 'post cleanup occurred. SIGKILL sent too late?' unless rescue_time.nil?
+
+      ensure
+        remaining_keys = Resque.redis.keys('sigterm-test:*') || []
+        Resque.redis.del(*remaining_keys) unless remaining_keys.empty?
       end
-
-      # close writers in parent now that we've forked
-      $child_pid_writer.close
-      $child_worker_message_writer.close
-
-      # give it a moment to start up the worker
-      sleep(0.2)
-
-      # send signal to abort the worker
-      Process.kill('TERM', worker_pid)
-      Process.waitpid(worker_pid)
-
-      # try to get the child pid
-      child_pid = Timeout.timeout(1.0) { $child_pid_reader.read.to_i }
-      assert child_pid > 0
-
-      child_still_running = !(`ps -p #{child_pid.to_s} -o pid=`).empty?
-      assert !child_still_running
-
-      child_exception = $child_worker_message_reader.read
-      assert_not_equal '', child_exception, 'child did not raise SignalException'
-    ensure
-      $child_pid_reader, $child_pid_writer, $child_worker_message_reader, $child_worker_message_writer = nil, nil, nil, nil
     end
   end
-
 end