Class: UHaul::Crawler

Inherits:
Object
  • Object
show all
Defined in:
lib/uhaul/crawler.rb

Overview

Used to fetch and parse either HTML or XML via a URL.

Constant Summary collapse

HOST = 
'https://www.uhaul.com'

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Instance Attribute Details

#url ⇒ Hash (readonly)

Returns:

  • (Hash)

Raises:



11
12
13
 
# File 'lib/uhaul/crawler.rb', line 11

def self.json(url:)
  new.json(url:)
end

Class Method Details

.html(url:) ⇒ Nokogiri::HTML::Document

Parameters:

  • url (String)

Returns:

  • (Nokogiri::HTML::Document)

Raises:



18
19
20
 
# File 'lib/uhaul/crawler.rb', line 18

def self.html(url:)
  new.html(url:)
end

.json(url:) ⇒ Hash

Returns:

  • (Hash)

Raises:



11
12
13
 
# File 'lib/uhaul/crawler.rb', line 11

def self.json(url:)
  new.json(url:)
end

.xml(url:) ⇒ Nokogiri::XML::Document

Parameters:

  • url (String)

Returns:

  • (Nokogiri::XML::Document)

Raises:



25
26
27
 
# File 'lib/uhaul/crawler.rb', line 25

def self.xml(url:)
  new.xml(url:)
end

Instance Method Details

#connection ⇒ HTTP::Client

Returns:

  • (HTTP::Client) 


30
31
32
33
34
35
36
37
38
39
40
41
 
# File 'lib/uhaul/crawler.rb', line 30

def connection
  @connection ||= begin
    config = UHaul.config

    connection = HTTP.use(:auto_deflate).use(:auto_inflate).persistent(HOST)
    connection = connection.headers(config.headers) if config.headers?
    connection = connection.timeout(config.timeout) if config.timeout?
    connection = connection.via(*config.via) if config.proxy?

    connection
  end
end

#fetch(url:) ⇒ HTTP::Response

Parameters:

  • url (String)

Returns:

  • (HTTP::Response)

Raises:



45
46
47
48
49
50
 
# File 'lib/uhaul/crawler.rb', line 45

def fetch(url:)
  response = connection.follow.get(url)
  raise FetchError.new(url:, response: response.flush) unless response.status.ok?

  response
end

#html(url:) ⇒ Nokogiri::XML::Document

Parameters:

  • url (String)

Returns:

  • (Nokogiri::XML::Document)

Raises:



62
63
64
 
# File 'lib/uhaul/crawler.rb', line 62

def html(url:)
  Nokogiri::HTML(String(fetch(url:).body))
end

#json(url:) ⇒ Hash

Parameters:

  • url (String)

Returns:

  • (Hash)

Raises:



55
56
57
 
# File 'lib/uhaul/crawler.rb', line 55

def json(url:)
  JSON.parse(String(fetch(url:).body))
end

#xml(url:) ⇒ Nokogiri::XML::Document

Parameters:

  • url (String)

Returns:

  • (Nokogiri::XML::Document)

Raises:



69
70
71
 
# File 'lib/uhaul/crawler.rb', line 69

def xml(url:)
  Nokogiri::XML(String(fetch(url:).body))
end