Replace open-uri with Faraday

Author: cadwallion

bnet_scraper.gemspec

@@ -16,6 +16,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   s.add_runtime_dependency 'nokogiri'
+  s.add_runtime_dependency 'faraday'
   s.add_development_dependency 'rake'
   s.add_development_dependency 'rspec'
   s.add_development_dependency 'fakeweb'

lib/bnet_scraper.rb

@@ -1,6 +1,5 @@
 require 'bnet_scraper/starcraft2'
-require 'net/http'
-require 'open-uri'
+require 'faraday'
 require 'nokogiri'
 
 module BnetScraper

lib/bnet_scraper/starcraft2/achievement_scraper.rb

@@ -43,7 +43,8 @@ def scrape
 
       # retrieves the account's achievements overview page HTML for scraping
       def get_response
-        @response = Nokogiri::HTML(open(profile_url+"achievements/")) 
+        response = Faraday.get "#{profile_url}achievements/"
+        @response = Nokogiri::HTML(response.body) 
       end
 
       # scrapes the recent achievements from the account's achievements overview page

lib/bnet_scraper/starcraft2/league_scraper.rb

@@ -30,15 +30,18 @@ def initialize options = {}
       end
 
       def scrape
-        @response = Nokogiri::HTML(open(@url))
-        value = @response.css(".data-title .data-label h3").inner_text().strip 
-        header_regex = /Season (\d{1}) - \s+(\dv\d)( Random)? (\w+)\s+Division (.+)/
-        header_values = value.match(header_regex).to_a
-        header_values.shift()
-        @season, @size, @random, @division, @name = header_values
-        
-        @random = !@random.nil?
-        output
+        @response = Faraday.get @url
+        if @response.success?
+          @response = Nokogiri::HTML(@response.body)
+          value = @response.css(".data-title .data-label h3").inner_text().strip 
+          header_regex = /Season (\d{1}) - \s+(\dv\d)( Random)? (\w+)\s+Division (.+)/
+          header_values = value.match(header_regex).to_a
+          header_values.shift()
+          @season, @size, @random, @division, @name = header_values
+          
+          @random = !@random.nil?
+          output
+        end
       end
 
       def output

lib/bnet_scraper/starcraft2/match_history_scraper.rb

@@ -25,7 +25,11 @@ def match_url
 
       # retrieves the match history HTML for scraping
       def get_response
-        @response = Nokogiri::HTML(open(match_url))
+        @response = Faraday.get match_url
+        
+        if @response.success?
+          @response = Nokogiri::HTML(@response.body)
+        end
       end
 
       def scrape

lib/bnet_scraper/starcraft2/profile_scraper.rb

@@ -30,23 +30,30 @@ def scrape
 
       # scrapes the profile page for basic account information
       def get_profile_data
-        response = Nokogiri::HTML(open(profile_url))
+        response = Faraday.get profile_url
 
-        @race = response.css("#season-snapshot .module-footer a").first().inner_html()
-        @wins = response.css("#career-stats h2").inner_html()
-        @achievement_points = response.css("#profile-header h3").inner_html()
+        if response.success?
+          html = Nokogiri::HTML(response.body)
+
+          @race = html.css("#season-snapshot .module-footer a").first().inner_html()
+          @wins = html.css("#career-stats h2").inner_html()
+          @achievement_points = html.css("#profile-header h3").inner_html()
+        end
       end
 
       # scrapes the league list from account's league page and sets an array of URLs
       def get_league_list
-        response = Nokogiri::HTML(open(profile_url + "ladder/leagues"))
+        response = Faraday.get profile_url + "ladder/leagues"
+        if response.success?
+          html = Nokogiri::HTML(response.body)
 
-        @leagues = response.css("a[href*='#current-rank']").map do |league|
-          {
-            name: league.inner_html().strip,
-            id: league.attr('href').sub('#current-rank',''),
-            href: "#{profile_url}ladder/#{league.attr('href')}"
-          }
+          @leagues = html.css("a[href*='#current-rank']").map do |league|
+            {
+              name: league.inner_html().strip,
+              id: league.attr('href').sub('#current-rank',''),
+              href: "#{profile_url}ladder/#{league.attr('href')}"
+            }
+          end
         end
       end
 

lib/bnet_scraper/starcraft2/status_scraper.rb

@@ -17,7 +17,8 @@ class Status
       class << self
 
         def fetch
-          Nokogiri::HTML(open(SOURCE))
+          response = Faraday.get SOURCE
+          Nokogiri::HTML(response.body)
             .css('.forumPost').first.css('span').to_a
             .each_slice(2).map { |i| { :region => i.first.text, :status => i.last.text } }
         end