This server allows you to test various mechanize behavior against other HTTP clients. Some endpoints may require headers be set to have a reasonable function, or may respond diffently to POST vs GET requests. Please see the servlet implementation and mechanize tests for further details.
Here are the servlet endpoints available:
-
#{servlets}
-
#{query.join "\n"}
#{req.query}
BODY
end
def do_POST(req, res)
res.content_type = 'text/html'
query = []
req.query.each_key { |k|
key = WEBrick::HTTPUtils.unescape k
req.query[k].each_data { |data|
value = WEBrick::HTTPUtils.unescape data
query << "-
#{query.join "\n"}
#{req.body}
BODY
end
end
mechanize-2.10.1/lib/mechanize/test_case/ntlm_servlet.rb 0000644 0000041 0000041 00000001577 14645745627 023352 0 ustar www-data www-data # frozen_string_literal: true
class NTLMServlet < WEBrick::HTTPServlet::AbstractServlet
def do_GET(req, res)
if req['Authorization'] =~ /^NTLM (.*)/ then
authorization = $1.unpack('m*').first
if authorization =~ /^NTLMSSP\000\001/ then
type_2 = 'TlRMTVNTUAACAAAADAAMADAAAAABAoEAASNFZ4mr' \
'ze8AAAAAAAAAAGIAYgA8AAAARABPAE0AQQBJAE4A' \
'AgAMAEQATwBNAEEASQBOAAEADABTAEUAUgBWAEUA' \
'UgAEABQAZABvAG0AYQBpAG4ALgBjAG8AbQADACIA' \
'cwBlAHIAdgBlAHIALgBkAG8AbQBhAGkAbgAuAGMA' \
'bwBtAAAAAAA='
res['WWW-Authenticate'] = "NTLM #{type_2}"
res.status = 401
elsif authorization =~ /^NTLMSSP\000\003/ then
res.body = 'ok'
else
res['WWW-Authenticate'] = 'NTLM'
res.status = 401
end
else
res['WWW-Authenticate'] = 'NTLM'
res.status = 401
end
end
end
mechanize-2.10.1/lib/mechanize/test_case/refresh_without_url.rb 0000644 0000041 0000041 00000000541 14645745627 024725 0 ustar www-data www-data # frozen_string_literal: true
class RefreshWithoutUrl < WEBrick::HTTPServlet::AbstractServlet
@@count = 0
def do_GET(req, res)
address = "#{req.host}:#{req.port}"
res['Content-Type'] = "text/html"
@@count += 1
if @@count > 1
res['Refresh'] = "0; url=http://#{address}/";
else
res['Refresh'] = "0";
end
end
end
mechanize-2.10.1/lib/mechanize/test_case/refresh_with_empty_url.rb 0000644 0000041 0000041 00000000547 14645745627 025421 0 ustar www-data www-data # frozen_string_literal: true
class RefreshWithEmptyUrl < WEBrick::HTTPServlet::AbstractServlet
@@count = 0
def do_GET(req, res)
address = "#{req.host}:#{req.port}"
res.content_type = "text/html"
@@count += 1
if @@count > 1
res['Refresh'] = "0; url=http://#{address}/";
else
res['Refresh'] = "0; url=";
end
end
end
mechanize-2.10.1/lib/mechanize/test_case/redirect_servlet.rb 0000644 0000041 0000041 00000000624 14645745627 024171 0 ustar www-data www-data # frozen_string_literal: true
class RedirectServlet < WEBrick::HTTPServlet::AbstractServlet
def do_GET(req, res)
res['Content-Type'] = req.query['ct'] || 'text/html'
res.status = req.query['code'] ? req.query['code'].to_i : '302'
res['Location'] = req['X-Location'] || '/verb'
end
alias :do_POST :do_GET
alias :do_HEAD :do_GET
alias :do_PUT :do_GET
alias :do_DELETE :do_GET
end
mechanize-2.10.1/lib/mechanize/test_case/one_cookie_no_spaces_servlet.rb 0000644 0000041 0000041 00000000570 14645745627 026534 0 ustar www-data www-data # frozen_string_literal: true
class OneCookieNoSpacesServlet < WEBrick::HTTPServlet::AbstractServlet
def do_GET(req, res)
cookie = WEBrick::Cookie.new("foo", "bar")
cookie.path = "/"
cookie.expires = Time.now + 86400
res.cookies << cookie.to_s.gsub(/; /, ';')
res['Content-Type'] = "text/html"
res.body = "hello"
end
end
mechanize-2.10.1/lib/mechanize/test_case/basic_auth_servlet.rb 0000644 0000041 0000041 00000001165 14645745627 024473 0 ustar www-data www-data # frozen_string_literal: true
class BasicAuthServlet < WEBrick::HTTPServlet::AbstractServlet
def do_GET(req,res)
htpd = nil
Tempfile.open 'dot.htpasswd' do |io|
htpd = WEBrick::HTTPAuth::Htpasswd.new(io.path)
htpd.set_passwd('Blah', 'user', 'pass')
end
authenticator = WEBrick::HTTPAuth::BasicAuth.new({
:UserDB => htpd,
:Realm => 'Blah',
:Logger => Logger.new(nil)
})
begin
authenticator.authenticate(req,res)
res.body = 'You are authenticated'
rescue WEBrick::HTTPStatus::Unauthorized
res.status = 401
end
end
alias :do_POST :do_GET
end
mechanize-2.10.1/lib/mechanize/test_case/infinite_redirect_servlet.rb 0000644 0000041 0000041 00000000615 14645745627 026056 0 ustar www-data www-data # frozen_string_literal: true
class InfiniteRedirectServlet < WEBrick::HTTPServlet::AbstractServlet
def do_GET(req, res)
res['Content-Type'] = req.query['ct'] || "text/html"
res.status = req.query['code'] ? req.query['code'].to_i : '302'
number = req.query['q'] ? req.query['q'].to_i : 0
res['Location'] = "/infinite_redirect?q=#{number + 1}"
end
alias :do_POST :do_GET
end
mechanize-2.10.1/lib/mechanize/test_case/bad_chunking_servlet.rb 0000644 0000041 0000041 00000000433 14645745627 025002 0 ustar www-data www-data # frozen_string_literal: true
class BadChunkingServlet < WEBrick::HTTPServlet::AbstractServlet
def do_GET req, res
res.keep_alive = false if res.respond_to? :keep_alive=
res['Transfer-Encoding'] = 'chunked'
res.body = <<-BODY
a\r
0123456789\r
0\r
BODY
end
end
mechanize-2.10.1/lib/mechanize/test_case/file_upload_servlet.rb 0000644 0000041 0000041 00000000661 14645745627 024654 0 ustar www-data www-data # frozen_string_literal: true
class FileUploadServlet < WEBrick::HTTPServlet::AbstractServlet
def do_POST req, res
res.body = req.body
end
def do_GET req, res
res.content_type = 'text/html'
res.body = <<-BODY
You can POST anything to this endpoint, though
BODY end end mechanize-2.10.1/lib/mechanize/test_case/send_cookies_servlet.rb 0000644 0000041 0000041 00000000563 14645745627 025037 0 ustar www-data www-data # frozen_string_literal: true class SendCookiesServlet < WEBrick::HTTPServlet::AbstractServlet def do_GET(req, res) res.content_type = 'text/html' cookies = req.cookies.map do |c| "-
#{cookies}
require 'addressable/uri'
class Mechanize::Page::Link
attr_reader :node
attr_reader :href
attr_reader :attributes
attr_reader :page
alias :referer :page
def initialize(node, mech, page)
@node = node
@attributes = node
@href = node['href']
@mech = mech
@page = page
@text = nil
@uri = nil
end
# Click on this link
def click
@mech.click self
end
# This method is a shorthand to get link's DOM id.
# Common usage:
# page.link_with(:dom_id => "links_exact_id")
def dom_id
node['id']
end
# This method is a shorthand to get a link's DOM class
# Common usage:
# page.link_with(:dom_class => "links_exact_class")
def dom_class
node['class']
end
def pretty_print(q) # :nodoc:
q.object_group(self) {
q.breakable; q.pp text
q.breakable; q.pp href
}
end
alias inspect pretty_inspect # :nodoc:
# A list of words in the rel attribute, all lower-cased.
def rel
@rel ||= (val = attributes['rel']) ? val.downcase.split(' ') : []
end
# Test if the rel attribute includes +kind+.
def rel? kind
rel.include? kind
end
# Test if this link should not be traced.
def noreferrer?
rel?('noreferrer')
end
# The text content of this link
def text
return @text if @text
@text = @node.inner_text
# If there is no text, try to find an image and use it's alt text
if (@text.nil? or @text.empty?) and imgs = @node.search('img') then
@text = imgs.map do |e|
e['alt']
end.join
end
@text
end
alias :to_s :text
# A URI for the #href for this link. The link is first parsed as a raw
# link. If that fails parsing an escaped link is attepmted.
def uri
@uri ||= if @href then
begin
URI.parse @href
rescue URI::InvalidURIError
begin
URI.parse(Addressable::URI.escape(@href))
rescue Addressable::URI::InvalidURIError
raise URI::InvalidURIError
end
end
end
end
# A fully resolved URI for the #href for this link.
def resolved_uri
@mech.resolve uri
end
end
mechanize-2.10.1/lib/mechanize/page/meta_refresh.rb 0000644 0000041 0000041 00000004054 14645745627 022233 0 ustar www-data www-data # frozen_string_literal: true
##
# This class encapsulates a meta element with a refresh http-equiv. Mechanize
# treats meta refresh elements just like 'a' tags. MetaRefresh objects will
# contain links, but most likely will have no text.
class Mechanize::Page::MetaRefresh < Mechanize::Page::Link
##
# Time to wait before next refresh
attr_reader :delay
##
# This MetaRefresh links did not contain a url= in the content attribute and
# links to itself.
attr_reader :link_self
##
# Matches the content attribute of a meta refresh element. After the match:
#
# $1:: delay
# $3:: url
CONTENT_REGEXP = /^\s*(\d+\.?\d*)\s*(?:;(?:\s*url\s*=\s*(['"]?)(\S*)\2)?\s*)?$/i
##
# Regexp of unsafe URI characters that excludes % for Issue #177
UNSAFE = /[^\-_.!~*'()a-zA-Z\d;\/?:@&%=+$,\[\]]/
##
# Parses the delay and url from the content attribute of a meta
# refresh element.
#
# Returns an array of [delay, url, link_self], where the first two
# are strings containing the respective parts of the refresh value,
# and link_self is a boolean value that indicates whether the url
# part is missing or empty. If base_uri, the URI of the current
# page is given, the value of url becomes an absolute URI.
def self.parse content, base_uri = nil
m = CONTENT_REGEXP.match(content) or return
delay, url = m[1], m[3]
url &&= url.empty? ? nil : Mechanize::Util.uri_escape(url, UNSAFE)
link_self = url.nil?
if base_uri
url = url ? base_uri + url : base_uri
end
return delay, url, link_self
end
def self.from_node node, page, uri = nil
http_equiv = node['http-equiv'] and
/\ARefresh\z/i =~ http_equiv or return
delay, uri, link_self = parse node['content'], uri
return unless delay
new node, page, delay, uri, link_self
end
def initialize node, page, delay, href, link_self = false
super node, page.mech, page
@delay = delay.include?(?.) ? delay.to_f : delay.to_i
@href = href
@link_self = link_self
end
def noreferrer?
true
end
end
mechanize-2.10.1/lib/mechanize/page/image.rb 0000644 0000041 0000041 00000006443 14645745627 020655 0 ustar www-data www-data # frozen_string_literal: true
##
# An image element on an HTML page
class Mechanize::Page::Image
attr_reader :node
attr_accessor :page
attr_accessor :mech
##
# Creates a new Mechanize::Page::Image from an image +node+ and source
# +page+.
def initialize node, page
@node = node
@page = page
@mech = page.mech
end
##
# The alt attribute of the image
def alt
node['alt']
end
##
# The caption of the image. In order of preference, the #title, #alt, or
# empty string "".
def caption
title || alt || ''
end
alias :text :caption
##
# The class attribute of the image
def dom_class
node['class']
end
##
# The id attribute of the image
def dom_id
node['id']
end
##
# The suffix of the #url. The dot is a part of suffix, not a delimiter.
#
# p image.url # => "http://example/test.jpg"
# p image.extname # => ".jpg"
#
# Returns an empty string if #url has no suffix:
#
# p image.url # => "http://example/sampleimage"
# p image.extname # => ""
def extname
return nil unless src
File.extname url.path
end
##
# Downloads the image.
#
# agent.page.image_with(:src => /logo/).fetch.save
#
# The referer is:
#
# #page("parent") ::
# all images on http html, relative #src images on https html
# (no referer) ::
# absolute #src images on https html
# user specified ::
# img.fetch(nil, my_referer_uri_or_page)
def fetch parameters = [], referer = nil, headers = {}
mech.get src, parameters, referer || image_referer, headers
end
##
# The height attribute of the image
def height
node['height']
end
def image_referer # :nodoc:
http_page = page.uri && page.uri.scheme == 'http'
https_page = page.uri && page.uri.scheme == 'https'
case
when http_page then page
when https_page && relative? then page
else
Mechanize::File.new(nil, { 'content-type' => 'text/plain' }, '', 200)
end
end
##
# MIME type guessed from the image url suffix
#
# p image.extname # => ".jpg"
# p image.mime_type # => "image/jpeg"
# page.images_with(:mime_type => /gif|jpeg|png/).each do ...
#
# Returns nil if url has no (well-known) suffix:
#
# p image.url # => "http://example/sampleimage"
# p image.mime_type # => nil
def mime_type
suffix_without_dot = extname ? extname.sub(/\A\./){''}.downcase : nil
Mechanize::Util::DefaultMimeTypes[suffix_without_dot]
end
def pretty_print(q) # :nodoc:
q.object_group(self) {
q.breakable; q.pp url
q.breakable; q.pp caption
}
end
alias inspect pretty_inspect # :nodoc:
def relative? # :nodoc:
%r{^https?://} !~ src
end
##
# The src attribute of the image
def src
node['src']
end
##
# The title attribute of the image
def title
node['title']
end
##
# The URL string of this image
def to_s
url.to_s
end
##
# URI for this image
def url
if relative? then
if page.bases[0] then
page.bases[0].href + src.to_s
else
page.uri + Mechanize::Util.uri_escape(src.to_s)
end
else
URI Mechanize::Util.uri_escape(src)
end
end
alias uri url
##
# The width attribute of the image
def width
node['width']
end
end
mechanize-2.10.1/lib/mechanize/page/base.rb 0000644 0000041 0000041 00000000364 14645745627 020501 0 ustar www-data www-data # frozen_string_literal: true
##
# A base element on an HTML page. Mechanize treats base tags just like 'a'
# tags. Base objects will contain links, but most likely will have no text.
class Mechanize::Page::Base < Mechanize::Page::Link
end
mechanize-2.10.1/lib/mechanize/page/label.rb 0000644 0000041 0000041 00000000536 14645745627 020647 0 ustar www-data www-data # frozen_string_literal: true
##
# A form label on an HTML page
class Mechanize::Page::Label
attr_reader :node
attr_reader :text
attr_reader :page
alias :to_s :text
def initialize(node, page)
@node = node
@text = node.inner_text
@page = page
end
def for
(id = @node['for']) && page.search("##{id}") || nil
end
end
mechanize-2.10.1/lib/mechanize/page/frame.rb 0000644 0000041 0000041 00000001314 14645745627 020655 0 ustar www-data www-data # frozen_string_literal: true
# A Frame object wraps a frame HTML element. Frame objects can be treated
# just like Link objects. They contain #src, the #link they refer to and a
# #name, the name of the frame they refer to. #src and #name are aliased to
# #href and #text respectively so that a Frame object can be treated just like
# a Link.
class Mechanize::Page::Frame < Mechanize::Page::Link
alias :src :href
attr_reader :text
alias :name :text
attr_reader :node
def initialize(node, mech, referer)
super(node, mech, referer)
@node = node
@text = node['name']
@href = node['src']
@content = nil
end
def content
@content ||= @mech.get @href, [], page
end
end
mechanize-2.10.1/lib/mechanize/file_request.rb 0000644 0000041 0000041 00000000564 14645745627 021344 0 ustar www-data www-data # frozen_string_literal: true
##
# A wrapper for a file URI that makes a request that works like a
# Net::HTTPRequest
class Mechanize::FileRequest
attr_accessor :uri
def initialize uri
@uri = uri
end
def add_field *a
end
alias []= add_field
def path
@uri.path
end
def each_header
end
def response_body_permitted?
true
end
end
mechanize-2.10.1/lib/mechanize/test_case.rb 0000644 0000041 0000041 00000017224 14645745627 020630 0 ustar www-data www-data # frozen_string_literal: true
require 'mechanize'
require 'logger'
require 'tempfile'
require 'tmpdir'
require 'webrick'
require 'zlib'
require 'rubygems'
begin
gem 'minitest'
rescue Gem::LoadError
end
require 'minitest/autorun'
begin
require 'minitest/pride'
rescue LoadError
end
##
# A generic test case for testing mechanize. Using a subclass of
# Mechanize::TestCase for your tests will create an isolated mechanize
# instance that won't pollute your filesystem or other tests.
#
# Once Mechanize::TestCase is loaded no HTTP requests will be made outside
# mechanize itself. All requests are handled via WEBrick servlets.
#
# Mechanize uses WEBrick servlets to test some functionality. You can run
# other HTTP clients against the servlets using:
#
# ruby -rmechanize/test_case/server -e0
#
# Which will launch a test server at http://localhost:8000
class Mechanize::TestCase < Minitest::Test
TEST_DIR = File.expand_path '../../../test', __FILE__
REQUESTS = []
##
# Creates a clean mechanize instance +@mech+ for use in tests.
def setup
super
REQUESTS.clear
@mech = Mechanize.new
@ssl_private_key = nil
@ssl_certificate = nil
end
##
# Creates a fake page with URI http://fake.example and an empty, submittable
# form.
def fake_page agent = @mech
uri = URI 'http://fake.example/'
html = String.new(<<~END)
END
Mechanize::Page.new uri, nil, html, 200, agent
end
##
# Is the Encoding constant defined?
def have_encoding?
Object.const_defined? :Encoding
end
##
# Creates a Mechanize::Page with the given +body+
def html_page body
uri = URI 'http://example/'
Mechanize::Page.new uri, nil, body, 200, @mech
end
##
# Creates a Mechanize::CookieJar by parsing the given +str+
def cookie_jar str, uri = URI('http://example')
Mechanize::CookieJar.new.tap do |jar|
jar.parse str, uri
end
end
##
# Runs the block inside a temporary directory
def in_tmpdir
Dir.mktmpdir do |dir|
Dir.chdir dir do
yield
end
end
end
##
# Creates a Nokogiri Node +element+ with the given +attributes+
def node element, attributes = {}
Nokogiri::XML::Node.new(element, Nokogiri::HTML::Document.new).tap do |node|
attributes.each do |name, value|
node[name] = value
end
end
end
##
# Creates a Mechanize::Page for the given +uri+ with the given
# +content_type+, response +body+ and HTTP status +code+
def page uri, content_type = 'text/html', body = String.new, code = 200
uri = URI uri unless URI::Generic === uri
Mechanize::Page.new(uri, { 'content-type' => content_type }, body, code,
@mech)
end
##
# Requests made during this tests
def requests
REQUESTS
end
##
# An SSL private key. This key is the same across all test runs
def ssl_private_key
@ssl_private_key ||= OpenSSL::PKey::RSA.new <<-KEY
-----BEGIN RSA PRIVATE KEY-----
MIG7AgEAAkEA8pmEfmP0Ibir91x6pbts4JmmsVZd3xvD5p347EFvBCbhBW1nv1Gs
bCBEFlSiT1q2qvxGb5IlbrfdhdgyqdTXUQIBAQIBAQIhAPumXslvf6YasXa1hni3
p80joKOug2UUgqOLD2GUSO//AiEA9ssY6AFxjHWuwo/+/rkLmkfO2s1Lz3OeUEWq
6DiHOK8CAQECAQECIQDt8bc4vS6wh9VXApNSKIpVygtxSFe/IwLeX26n77j6Qg==
-----END RSA PRIVATE KEY-----
KEY
end
##
# An X509 certificate. This certificate is the same across all test runs
def ssl_certificate
@ssl_certificate ||= OpenSSL::X509::Certificate.new <<-CERT
-----BEGIN CERTIFICATE-----
MIIBQjCB7aADAgECAgEAMA0GCSqGSIb3DQEBBQUAMCoxDzANBgNVBAMMBm5vYm9k
eTEXMBUGCgmSJomT8ixkARkWB2V4YW1wbGUwIBcNMTExMTAzMjEwODU5WhgPOTk5
OTEyMzExMjU5NTlaMCoxDzANBgNVBAMMBm5vYm9keTEXMBUGCgmSJomT8ixkARkW
B2V4YW1wbGUwWjANBgkqhkiG9w0BAQEFAANJADBGAkEA8pmEfmP0Ibir91x6pbts
4JmmsVZd3xvD5p347EFvBCbhBW1nv1GsbCBEFlSiT1q2qvxGb5IlbrfdhdgyqdTX
UQIBATANBgkqhkiG9w0BAQUFAANBAAAB////////////////////////////////
//8AMCEwCQYFKw4DAhoFAAQUePiv+QrJxyjtEJNnH5pB9OTWIqA=
-----END CERTIFICATE-----
CERT
end
##
# Creates a Tempfile with +content+ that is immediately unlinked
def tempfile content
Tempfile.new(@NAME).tap do |body_io|
body_io.unlink
body_io.write content
body_io.flush
body_io.rewind
end
end
##
# Returns true if the current platform is a Windows platform
def windows?
::RUBY_PLATFORM =~ /mingw|mswin/
end
##
# Return the contents of the file without Windows carriage returns
def file_contents_without_cr(path)
File.read(path).gsub(/\r\n/, "\n")
end
end
require 'mechanize/test_case/servlets'
module Net # :nodoc:
end
class Net::HTTP # :nodoc:
alias :old_do_start :do_start
def do_start
@started = true
end
PAGE_CACHE = {}
alias :old_request :request
def request(req, *data, &block)
url = URI.parse(req.path)
path = WEBrick::HTTPUtils.unescape(url.path)
path = '/index.html' if path == '/'
res = ::Response.new
res.query_params = url.query
req.query = if 'POST' != req.method && url.query then
WEBrick::HTTPUtils.parse_query url.query
elsif req['content-type'] =~ /www-form-urlencoded/ then
WEBrick::HTTPUtils.parse_query req.body
elsif req['content-type'] =~ /boundary=(.+)/ then
boundary = WEBrick::HTTPUtils.dequote $1
WEBrick::HTTPUtils.parse_form_data req.body, boundary
else
{}
end
req.cookies = WEBrick::Cookie.parse(req['Cookie'])
Mechanize::TestCase::REQUESTS << req
if servlet_klass = MECHANIZE_TEST_CASE_SERVLETS[path]
servlet = servlet_klass.new({})
servlet.send "do_#{req.method}", req, res
else
filename = "htdocs#{path.gsub(/[^\/\\.\w\s]/, '_')}"
unless PAGE_CACHE[filename]
::File.open("#{Mechanize::TestCase::TEST_DIR}/#{filename}", 'rb') do |io|
PAGE_CACHE[filename] = io.read
end
end
res.body = PAGE_CACHE[filename]
case filename
when /\.txt$/
res['Content-Type'] = 'text/plain'
when /\.jpg$/
res['Content-Type'] = 'image/jpeg'
end
end
res['Content-Type'] ||= 'text/html'
res.code ||= "200"
response_klass = Net::HTTPResponse::CODE_TO_OBJ[res.code.to_s]
response = response_klass.new res.http_version, res.code, res.message
res.header.each do |k,v|
v = v.first if v.length == 1
response[k] = v
end
res.cookies.each do |cookie|
response.add_field 'Set-Cookie', cookie.to_s
end
response['Content-Type'] ||= 'text/html'
response['Content-Length'] = res['Content-Length'] || res.body.length.to_s
io = StringIO.new(res.body)
response.instance_variable_set :@socket, io
def io.read clen, dest = nil, _ = nil
if dest then
dest << super(clen)
else
super clen
end
end
body_exist = req.response_body_permitted? &&
response_klass.body_permitted?
response.instance_variable_set :@body_exist, body_exist
yield response if block_given?
response
end
end
class Net::HTTPRequest # :nodoc:
attr_accessor :query, :body, :cookies, :user
def host
'example'
end
def port
80
end
end
class Response # :nodoc:
include Net::HTTPHeader
attr_reader :code
attr_accessor :body, :query, :cookies
attr_accessor :query_params, :http_version
attr_accessor :header
def code=(c)
@code = c.to_s
end
alias :status :code
alias :status= :code=
def initialize
@header = {}
@body = String.new
@code = nil
@query = nil
@cookies = []
@http_version = '1.1'
end
def read_body
yield body
end
def message
''
end
end
mechanize-2.10.1/lib/mechanize/unauthorized_error.rb 0000644 0000041 0000041 00000000632 14645745627 022603 0 ustar www-data www-data # frozen_string_literal: true
class Mechanize::UnauthorizedError < Mechanize::ResponseCodeError
attr_reader :challenges
def initialize page, challenges, message
super page, message
@challenges = challenges
end
def to_s
out = super
if @challenges then
realms = @challenges.map(&:realm_name).join ', '
out << " -- available realms: #{realms}"
end
out
end
end
mechanize-2.10.1/lib/mechanize/response_code_error.rb 0000644 0000041 0000041 00000001450 14645745627 022711 0 ustar www-data www-data # frozen_string_literal: true
# This error is raised when Mechanize encounters a response code it does not
# know how to handle. Currently, this exception will be thrown if Mechanize
# encounters response codes other than 200, 301, or 302. Any other response
# code is up to the user to handle.
class Mechanize::ResponseCodeError < Mechanize::Error
attr_reader :response_code
attr_reader :page
def initialize(page, message = nil)
super message
@page = page
@response_code = page.code.to_s
end
def to_s
response_class = Net::HTTPResponse::CODE_TO_OBJ[@response_code]
out = String.new("#{@response_code} => #{response_class} ")
out << "for #{@page.uri} " if @page.respond_to? :uri # may be HTTPResponse
out << "-- #{super}"
end
alias inspect to_s
end
mechanize-2.10.1/lib/mechanize/robots_disallowed_error.rb 0000644 0000041 0000041 00000001354 14645745627 023603 0 ustar www-data www-data # frozen_string_literal: true
# Exception that is raised when an access to a resource is disallowed by
# robots.txt or by HTML document itself.
class Mechanize::RobotsDisallowedError < Mechanize::Error
def initialize(url)
if url.is_a?(URI)
@url = url.to_s
@uri = url
else
@url = url.to_s
end
end
# Returns the URL (string) of the resource that caused this error.
attr_reader :url
# Returns the URL (URI object) of the resource that caused this
# error. URI::InvalidURIError may be raised if the URL happens to
# be invalid or not understood by the URI library.
def uri
@uri ||= URI.parse(url)
end
def to_s
"Robots access is disallowed for URL: #{url}"
end
alias :inspect :to_s
end
mechanize-2.10.1/lib/mechanize/download.rb 0000644 0000041 0000041 00000003557 14645745627 020471 0 ustar www-data www-data # frozen_string_literal: true
##
# Download is a pluggable parser for downloading files without loading them
# into memory first. You may subclass this class to handle content types you
# do not wish to load into memory first.
#
# See Mechanize::PluggableParser for instructions on using this class.
class Mechanize::Download
include Mechanize::Parser
##
# The filename for this file based on the content-disposition of the
# response or the basename of the URL
attr_accessor :filename
##
# Accessor for the IO-like that contains the body
attr_reader :body_io
alias content body_io
##
# Creates a new download retrieved from the given +uri+ and +response+
# object. The +body_io+ is an IO-like containing the HTTP response body and
# +code+ is the HTTP status.
def initialize uri = nil, response = nil, body_io = nil, code = nil
@uri = uri
@body_io = body_io
@code = code
@full_path = false unless defined? @full_path
fill_header response
extract_filename
yield self if block_given?
end
##
# The body of this response as a String.
#
# Take care, this may use lots of memory if the response body is large.
def body
@body_io.read.tap { @body_io.rewind }
end
##
# Saves a copy of the body_io to +filename+
# returns the filename
def save filename = nil
filename = find_free_name filename
save! filename
end
alias save_as save
##
# Use this method to save the content of body_io to +filename+.
# This method will overwrite any existing filename that exists with the
# same name.
# returns the filename
def save! filename = nil
filename ||= @filename
dirname = File.dirname filename
FileUtils.mkdir_p dirname
::File.open(filename, 'wb')do |io|
until @body_io.eof? do
io.write @body_io.read 16384
end
end
filename
end
end
mechanize-2.10.1/lib/mechanize/page.rb 0000644 0000041 0000041 00000035027 14645745627 017573 0 ustar www-data www-data # frozen_string_literal: true
##
# This class encapsulates an HTML page. If Mechanize finds a content
# type of 'text/html', this class will be instantiated and returned.
#
# Example:
#
# require 'mechanize'
#
# agent = Mechanize.new
# agent.get('http://google.com/').class # => Mechanize::Page
class Mechanize::Page < Mechanize::File
extend Forwardable
extend Mechanize::ElementMatcher
DEFAULT_RESPONSE = {
'content-type' => 'text/html',
}.freeze
attr_accessor :mech
##
# Possible encodings for this page based on HTTP headers and meta elements
attr_reader :encodings
def initialize(uri=nil, response=nil, body=nil, code=nil, mech=nil)
response ||= DEFAULT_RESPONSE
@meta_content_type = nil
@encoding = nil
@encodings = [nil]
raise 'no' if mech and not Mechanize === mech
@mech = mech
reset
@encodings << Mechanize::Util.detect_charset(body) if body
@encodings.concat self.class.response_header_charset(response)
if body
@encodings.concat self.class.meta_charset body
meta_content_type = self.class.meta_content_type body
@meta_content_type = meta_content_type if meta_content_type
end
@encodings << mech.default_encoding if mech and mech.default_encoding
super uri, response, body, code
end
def title
@title ||=
if doc = parser
title = doc.xpath('string(((/html/head | /html | /head | /)/title)[1])').to_s
title.empty? ? nil : title
end
end
def response_header_charset
self.class.response_header_charset(response)
end
def meta_charset
self.class.meta_charset(body)
end
def detected_encoding
Mechanize::Util.detect_charset(body)
end
def encoding=(encoding)
reset
@encoding = encoding
if @parser
parser_encoding = @parser.encoding
if parser_encoding && encoding && parser_encoding.casecmp(encoding) != 0
# lazy reinitialize the parser with the new encoding
@parser = nil
end
end
encoding
end
def encoding
parser.encoding
rescue NoMethodError
nil
end
# Return whether parser result has errors related to encoding or not.
# false indicates just parser has no encoding errors, not encoding is valid.
def encoding_error?(parser=nil)
parser = self.parser unless parser
return false if parser.errors.empty?
parser.errors.any? do |error|
error.message.scrub =~ /(indicate\ encoding)|
(Invalid\ bytes)|
(Invalid\ char)|
(input\ conversion\ failed)/x
end
end
def parser
return @parser if @parser
return unless @body
url = @uri && @uri.to_s
if @encoding
@parser = mech.html_parser.parse html_body, url, @encoding
elsif mech.force_default_encoding
@parser = mech.html_parser.parse html_body, url, @mech.default_encoding
else
@encodings.reverse_each do |encoding|
@parser = mech.html_parser.parse html_body, url, encoding
break unless encoding_error? @parser
end
end
@parser
end
alias :root :parser
def pretty_print(q) # :nodoc:
q.object_group(self) {
q.breakable
q.group(1, '{url', '}') {q.breakable; q.pp uri }
q.breakable
q.group(1, '{meta_refresh', '}') {
meta_refresh.each { |link| q.breakable; q.pp link }
}
q.breakable
q.group(1, '{title', '}') { q.breakable; q.pp title }
q.breakable
q.group(1, '{iframes', '}') {
iframes.each { |link| q.breakable; q.pp link }
}
q.breakable
q.group(1, '{frames', '}') {
frames.each { |link| q.breakable; q.pp link }
}
q.breakable
q.group(1, '{links', '}') {
links.each { |link| q.breakable; q.pp link }
}
q.breakable
q.group(1, '{forms', '}') {
forms.each { |form| q.breakable; q.pp form }
}
}
end
alias inspect pretty_inspect # :nodoc:
def reset
@bases = nil
@forms = nil
@frames = nil
@iframes = nil
@links = nil
@labels = nil
@labels_hash = nil
@meta_refresh = nil
@parser = nil
@title = nil
end
# Return the canonical URI for the page if there is a link tag
# with href="canonical".
def canonical_uri
link = at('link[@rel="canonical"][@href]')
return unless link
href = link['href']
URI href
rescue URI::InvalidURIError
URI Mechanize::Util.uri_escape href
end
# Get the content type
def content_type
@meta_content_type || response['content-type']
end
##
# :method: search
#
# Shorthand for +parser.search+.
#
# See Nokogiri::XML::Node#search for details.
##
# :method: css
#
# Shorthand for +parser.css+.
#
# See also Nokogiri::XML::Node#css for details.
##
# :method: xpath
#
# Shorthand for +parser.xpath+.
#
# See also Nokogiri::XML::Node#xpath for details.
##
# :method: at
#
# Shorthand for +parser.at+.
#
# See also Nokogiri::XML::Node#at for details.
##
# :method: at_css
#
# Shorthand for +parser.at_css+.
#
# See also Nokogiri::XML::Node#at_css for details.
##
# :method: at_xpath
#
# Shorthand for +parser.at_xpath+.
#
# See also Nokogiri::XML::Node#at_xpath for details.
def_delegators :parser, :search, :css, :xpath, :at, :at_css, :at_xpath
alias / search
alias % at
##
# :method: form_with
#
# :call-seq:
# form_with(criteria)
# form_with(criteria) { |form| ... }
#
# Find a single form matching +criteria+. See +forms_with+ for
# details of +criteria+.
#
# Examples:
# page.form_with(action: '/post/login.php') do |f|
# ...
# end
##
# :method: form_with!(criteria)
#
# :call-seq:
# form_with!(criteria)
# form_with!(criteria) { |form| ... }
#
# Same as +form_with+ but raises an ElementNotFoundError if no button matches
# +criteria+
##
# :method: forms_with
#
# :call-seq:
# forms_with(name)
# forms_with(name: name_matcher, id: id_matcher, class: class_matcher,
# search: search_expression, xpath: xpath_expression, css: css_expression,
# action: action_matcher, ...)
#
# Find all forms form matching criteria. If a string is given, it
# is taken as a name attribute value. If a hash is given, forms
# are narrowed by the key-value pairs as follows.
#
# :id, :dom_id: selects forms with a #dom_id value that matches this
# value.
#
# :class, :dom_class: selects forms with a #dom_class value that
# matches this value. Note that class attribute values are compared
# literally as string, so forms_with(class: "a") does not match a
# form with class="a b". Use forms_with(css: "form.a") instead.
#
# :search: only selects forms matching this selector expression.
#
# :xpath: only selects forms matching this XPath expression.
#
# :css: only selects forms matching this CSS selector expression.
#
# :action, :method, etc.: narrows forms by a given attribute value
# using the === operator.
#
# Example:
# page.forms_with(css: '#content table.login_box form', method: /\APOST\z/i, ).each do |f|
# ...
# end
elements_with :form
##
# :method: link_with
#
# :call-seq:
# link_with(criteria)
# link_with(criteria) { |link| ... }
#
# Find a single link matching +criteria+. See +forms_with+ for
# details of +criteria+, where for "form(s)" read "link(s)".
#
# Example:
# page.link_with(href: /foo/).click
##
# :method: link_with!
#
# :call-seq:
# link_with!(criteria)
# link_with!(criteria) { |link| ... }
#
# Same as +link_with+ but raises an ElementNotFoundError if no button matches
# +criteria+
##
# :method: links_with
#
# :call-seq:
# links_with(criteria)
#
# Find all links matching +criteria+. See +forms_with+ for details
# of +criteria+, where for "form(s)" read "link(s)".
#
# Example:
# page.links_with(href: /foo/).each do |link|
# puts link.href
# end
elements_with :link
##
# :method: base_with
#
# :call-seq:
# base_with(criteria)
# base_with(criteria) { |base| ... }
#
# Find a single base tag matching +criteria+. See +forms_with+ for
# details of +criteria+, where for "form(s)" read "base tag(s)".
#
# Example:
# page.base_with(href: /foo/).click
##
# :method: base_with!(criteria)
#
# :call-seq:
# base_with!(criteria)
# base_with!(criteria) { |base| ... }
#
# Same as +base_with+ but raises an ElementNotFoundError if no button matches
# +criteria+
##
# :method: bases_with
#
# :call-seq: bases_with(criteria)
#
# Find all base tags matching +criteria+. See +forms_with+ for
# details of +criteria+, where for "form(s)" read "base tag(s)".
#
# Example:
# page.bases_with(href: /foo/).each do |base|
# puts base.href
# end
elements_with :base
##
# :method: frame_with
#
# :call-seq:
# frame_with(criteria)
# frame_with(criteria) { |frame| ... }
#
# Find a single frame tag matching +criteria+. See +forms_with+ for
# details of +criteria+, where for "form(s)" read "frame tag(s)".
#
# Example:
# page.frame_with(src: /foo/).click
##
# :method: frame_with!
#
# :call-seq:
# frame_with!(criteria)
# frame_with!(criteria) { |frame| ... }
#
# Same as +frame_with+ but raises an ElementNotFoundError if no button matches
# +criteria+
##
# :method: frames_with
#
# :call-seq: frames_with(criteria)
#
# Find all frame tags matching +criteria+. See +forms_with+ for
# details of +criteria+, where for "form(s)" read "frame tag(s)".
#
# Example:
# page.frames_with(src: /foo/).each do |frame|
# p frame.src
# end
elements_with :frame
##
# :method: iframe_with
#
# :call-seq:
# iframe_with(criteria)
# iframe_with(criteria) { |iframe| ... }
#
# Find a single iframe tag matching +criteria+. See +forms_with+ for
# details of +criteria+, where for "form(s)" read "iframe tag(s)".
#
# Example:
# page.iframe_with(src: /foo/).click
##
# :method: iframe_with!
#
# :call-seq:
# iframe_with!(criteria)
# iframe_with!(criteria) { |iframe| ... }
#
# Same as +iframe_with+ but raises an ElementNotFoundError if no button
# matches +criteria+
##
# :method: iframes_with
#
# :call-seq: iframes_with(criteria)
#
# Find all iframe tags matching +criteria+. See +forms_with+ for
# details of +criteria+, where for "form(s)" read "iframe tag(s)".
#
# Example:
# page.iframes_with(src: /foo/).each do |iframe|
# p iframe.src
# end
elements_with :iframe
##
# :method: image_with
#
# :call-seq:
# image_with(criteria)
# image_with(criteria) { |image| ... }
#
# Find a single image matching +criteria+. See +forms_with+ for
# details of +criteria+, where for "form(s)" read "image(s)".
#
# Example:
# page.image_with(alt: /main/).fetch.save
##
# :method: image_with!
#
# :call-seq:
# image_with!(criteria)
# image_with!(criteria) { |image| ... }
#
# Same as +image_with+ but raises an ElementNotFoundError if no button matches
# +criteria+
##
# :method: images_with
#
# :call-seq: images_with(criteria)
#
# Find all images matching +criteria+. See +forms_with+ for
# details of +criteria+, where for "form(s)" read "image(s)".
#
# Example:
# page.images_with(src: /jpg\Z/).each do |img|
# img.fetch.save
# end
elements_with :image
##
# Return a list of all link and area tags
def links
@links ||= %w{ a area }.map do |tag|
search(tag).map do |node|
Link.new(node, @mech, self)
end
end.flatten
end
##
# Return a list of all form tags
def forms
@forms ||= search('form').map do |html_form|
form = Mechanize::Form.new(html_form, @mech, self)
form.action ||= @uri.to_s
form
end
end
##
# Return a list of all meta refresh elements
def meta_refresh
query = @mech.follow_meta_refresh == :anywhere ? 'meta' : 'head > meta'
@meta_refresh ||= search(query).map do |node|
MetaRefresh.from_node node, self
end.compact
end
##
# Return a list of all base tags
def bases
@bases ||=
search('base').map { |node| Base.new(node, @mech, self) }
end
##
# Return a list of all frame tags
def frames
@frames ||=
search('frame').map { |node| Frame.new(node, @mech, self) }
end
##
# Return a list of all iframe tags
def iframes
@iframes ||=
search('iframe').map { |node| Frame.new(node, @mech, self) }
end
##
# Return a list of all img tags
def images
@images ||=
search('img').map { |node| Image.new(node, self) }
end
def image_urls
@image_urls ||= images.map(&:url).uniq
end
##
# Return a list of all label tags
def labels
@labels ||=
search('label').map { |node| Label.new(node, self) }
end
def labels_hash
unless @labels_hash
hash = {}
labels.each do |label|
hash[label.node['for']] = label if label.for
end
@labels_hash = hash
end
return @labels_hash
end
class << self
def charset content_type
charset = content_type[/;(?:\s*,)?\s*charset\s*=\s*([^()<>@,;:\\\"\/\[\]?={}\s]+)/i, 1]
return nil if charset == 'none'
charset
end
alias charset_from_content_type charset
end
def self.response_header_charset response
charsets = []
response.each do |header, value|
next unless header == 'content-type'
next unless value =~ /charset/i
charsets << charset(value)
end
charsets
end
##
# Retrieves all charsets from +meta+ tags in +body+
def self.meta_charset body
# HACK use .map
body.scan(//i).map do |meta|
if meta =~ /charset\s*=\s*(["'])?\s*(.+)\s*\1/i then
$2
elsif meta =~ /http-equiv\s*=\s*(["'])?content-type\1/i then
meta =~ /content\s*=\s*(["'])?(.*?)\1/i
m_charset = charset $2 if $2
m_charset if m_charset
end
end.compact
end
##
# Retrieves the last content-type set by a +meta+ tag in +body+
def self.meta_content_type body
body.scan(//i).reverse.map do |meta|
if meta =~ /http-equiv\s*=\s*(["'])?content-type\1/i then
meta =~ /content=(["'])?(.*?)\1/i
return $2
end
end
nil
end
private
def html_body
if @body
@body.empty? ? '' : @body
else
''
end
end
end
require 'mechanize/headers'
require 'mechanize/page/image'
require 'mechanize/page/label'
require 'mechanize/page/link'
require 'mechanize/page/base'
require 'mechanize/page/frame'
require 'mechanize/page/meta_refresh'
mechanize-2.10.1/lib/mechanize/file.rb 0000644 0000041 0000041 00000004422 14645745627 017571 0 ustar www-data www-data # frozen_string_literal: true
##
# This is the base class for the Pluggable Parsers. If Mechanize cannot find
# an appropriate class to use for the content type, this class will be used.
# For example, if you download an image/jpeg, Mechanize will not know how to
# parse it, so this class will be instantiated.
#
# This is a good class to use as the base class for building your own
# pluggable parsers.
#
# == Example
#
# require 'mechanize'
#
# agent = Mechanize.new
# agent.get('http://example.com/foo.jpg').class #=> Mechanize::File
class Mechanize::File
include Mechanize::Parser
##
# The HTTP response body, the raw file contents
attr_accessor :body
##
# The filename for this file based on the content-disposition of the
# response or the basename of the URL
attr_accessor :filename
alias content body
##
# Creates a new file retrieved from the given +uri+ and +response+ object.
# The +body+ is the HTTP response body and +code+ is the HTTP status.
def initialize uri = nil, response = nil, body = nil, code = nil
@uri = uri
@body = body
@code = code
@full_path = false unless defined? @full_path
fill_header response
extract_filename
yield self if block_given?
end
##
# Use this method to save the content of this object to +filename+.
# returns the filename
#
# file.save 'index.html'
# file.save 'index.html' # saves to index.html.1
#
# uri = URI 'http://localhost/test.html'
# file = Mechanize::File.new uri, nil, ''
# filename = file.save # saves to test.html
# puts filename # test.html
def save filename = nil
filename = find_free_name filename
save! filename
end
alias save_as save
##
# Use this method to save the content of this object to +filename+.
# This method will overwrite any existing filename that exists with the
# same name.
# returns the filename
#
# file.save 'index.html'
# file.save! 'index.html' # overwrite original file
# filename = file.save! 'index.html' # overwrite original file with filename 'index.html'
def save! filename = nil
filename ||= @filename
dirname = File.dirname filename
FileUtils.mkdir_p dirname
::File.open(filename, 'wb')do |f|
f.write body
end
filename
end
end
mechanize-2.10.1/lib/mechanize/version.rb 0000644 0000041 0000041 00000000107 14645745627 020333 0 ustar www-data www-data # frozen_string_literal: true
class Mechanize
VERSION = "2.10.1"
end
mechanize-2.10.1/lib/mechanize/file_connection.rb 0000644 0000041 0000041 00000000663 14645745627 022013 0 ustar www-data www-data # frozen_string_literal: true
##
# Wrapper to make a file URI work like an http URI
class Mechanize::FileConnection
@instance = nil
def self.new *a
@instance ||= super
end
def request uri, request
file_path = uri.select(:host, :path)
.select { |part| part && (part.length > 0) }
.join(":")
yield Mechanize::FileResponse.new(Mechanize::Util.uri_unescape(file_path))
end
end
mechanize-2.10.1/lib/mechanize/redirect_not_get_or_head_error.rb 0000644 0000041 0000041 00000001100 14645745627 025052 0 ustar www-data www-data # frozen_string_literal: true
##
# Raised when a POST, PUT, or DELETE request results in a redirect
# see RFC 2616 10.3.2, 10.3.3 http://www.ietf.org/rfc/rfc2616.txt
class Mechanize::RedirectNotGetOrHeadError < Mechanize::Error
attr_reader :page, :response_code, :verb, :uri
def initialize(page, verb)
@page = page
@verb = verb
@uri = page.uri
@response_code = page.code
end
def to_s
method = @verb.to_s.upcase
"#{@response_code} redirect received after a #{method} request"
end
alias :inspect :to_s
end
mechanize-2.10.1/lib/mechanize/http/ 0000755 0000041 0000041 00000000000 14645745627 017302 5 ustar www-data www-data mechanize-2.10.1/lib/mechanize/http/auth_store.rb 0000644 0000041 0000041 00000006016 14645745627 022007 0 ustar www-data www-data # frozen_string_literal: true
##
# A credential store for HTTP authentication.
#
# uri = URI 'http://example'
#
# store = Mechanize::HTTP::AuthStore.new
# store.add_auth uri, 'user1', 'pass'
# store.add_auth uri, 'user2', 'pass', 'realm'
#
# user, pass = store.credentials_for uri, 'realm' #=> 'user2', 'pass'
# user, pass = store.credentials_for uri, 'other' #=> 'user1', 'pass'
#
# store.remove_auth uri # removes all credentials
class Mechanize::HTTP::AuthStore
attr_reader :auth_accounts # :nodoc:
attr_reader :default_auth # :nodoc:
##
# Creates a new AuthStore
def initialize
@auth_accounts = Hash.new do |h, uri|
h[uri] = {}
end
@default_auth = nil
end
##
# Adds credentials +user+, +pass+ for the server at +uri+. If +realm+ is
# set the credentials are used only for that realm. If +realm+ is not set
# the credentials become the default for any realm on that URI.
#
# +domain+ and +realm+ are exclusive as NTLM does not follow RFC
# 2617. If +domain+ is given it is only used for NTLM authentication.
def add_auth uri, user, pass, realm = nil, domain = nil
uri = URI uri unless URI === uri
raise ArgumentError,
'NTLM domain given with realm which NTLM does not use' if
realm and domain
uri += '/'
auth_accounts[uri][realm] = [user, pass, domain]
self
end
##
# USE OF add_default_auth IS NOT RECOMMENDED AS IT MAY EXPOSE PASSWORDS TO
# THIRD PARTIES
#
# Adds credentials +user+, +pass+ as the default authentication credentials.
# If no other credentials are available these will be returned from
# credentials_for.
#
# If +domain+ is given it is only used for NTLM authentication.
def add_default_auth user, pass, domain = nil
warn <<-WARN
You have supplied default authentication credentials that apply to ANY SERVER.
Your username and password can be retrieved by ANY SERVER using Basic
authentication.
THIS EXPOSES YOUR USERNAME AND PASSWORD TO DISCLOSURE WITHOUT YOUR KNOWLEDGE.
Use add_auth to set authentication credentials that will only be delivered
only to a particular server you specify.
WARN
@default_auth = [user, pass, domain]
end
##
# Returns true if credentials exist for the +challenges+ from the server at
# +uri+.
def credentials? uri, challenges
challenges.any? do |challenge|
credentials_for uri, challenge.realm_name
end
end
##
# Retrieves credentials for +realm+ on the server at +uri+.
def credentials_for uri, realm
uri = URI uri unless URI === uri
uri += '/'
uri.user = nil
uri.password = nil
realms = @auth_accounts[uri]
realms[realm] || realms[nil] || @default_auth
end
##
# Removes credentials for +realm+ on the server at +uri+. If +realm+ is not
# set all credentials for the server at +uri+ are removed.
def remove_auth uri, realm = nil
uri = URI uri unless URI === uri
uri += '/'
if realm then
auth_accounts[uri].delete realm
else
auth_accounts.delete uri
end
self
end
end
mechanize-2.10.1/lib/mechanize/http/www_authenticate_parser.rb 0000644 0000041 0000041 00000006457 14645745627 024601 0 ustar www-data www-data # frozen_string_literal: true
require 'strscan'
##
# Parses the WWW-Authenticate HTTP header into separate challenges.
class Mechanize::HTTP::WWWAuthenticateParser
attr_accessor :scanner # :nodoc:
##
# Creates a new header parser for WWW-Authenticate headers
def initialize
@scanner = nil
end
##
# Parsers the header. Returns an Array of challenges as strings
def parse www_authenticate
challenges = []
@scanner = StringScanner.new www_authenticate
while true do
break if @scanner.eos?
start = @scanner.pos
challenge = Mechanize::HTTP::AuthChallenge.new
scheme = auth_scheme
if scheme == 'Negotiate'
scan_comma_spaces
end
break unless scheme
challenge.scheme = scheme
space = spaces
if scheme == 'NTLM' then
if space then
challenge.params = @scanner.scan(/.*/)
end
challenge.raw = www_authenticate[start, @scanner.pos]
challenges << challenge
next
else
scheme.capitalize!
end
next unless space
params = {}
while true do
pos = @scanner.pos
name, value = auth_param
name.downcase! if name =~ /^realm$/i
unless name then
challenge.params = params
challenges << challenge
if @scanner.eos? then
challenge.raw = www_authenticate[start, @scanner.pos]
break
end
@scanner.pos = pos # rewind
challenge.raw = www_authenticate[start, @scanner.pos].sub(/(,+)? *$/, '')
challenge = nil # a token should be next, new challenge
break
else
params[name] = value
end
spaces
@scanner.scan(/(, *)+/)
end
end
challenges
end
##
# 1*SP
#
# Parses spaces
def spaces
@scanner.scan(/ +/)
end
##
# scans a comma followed by spaces
# needed for Negotiation, NTLM
#
def scan_comma_spaces
@scanner.scan(/, +/)
end
##
# token = 1*Hello World!
mechanize-2.10.1/test/htdocs/no_title_test.html 0000644 0000041 0000041 00000000122 14645745627 021612 0 ustar www-data www-dataPost Form 1
Post Form 2
Post Form 3
Post Form 4
mechanize-2.10.1/test/htdocs/tc_pretty_print.html 0000644 0000041 0000041 00000001215 14645745627 022173 0 ustar www-data www-datamechanize-2.10.1/test/htdocs/form_multival.html 0000644 0000041 0000041 00000001725 14645745627 021630 0 ustar www-data www-data
| Personalized Home | Sign in |
Advertising Programs - Business Solutions - About Google
©2006 Google
Referer Servlet (noreferrer) Referer Servlet forced to http (noreferrer) Referer Servlet forced to https (noreferrer)
mechanize-2.10.1/test/htdocs/tc_bad_charset.html 0000644 0000041 0000041 00000000260 14645745627 021666 0 ustar www-data www-data
Biaystok
mechanize-2.10.1/test/htdocs/tc_bad_links.html 0000644 0000041 0000041 00000000115 14645745627 021354 0 ustar www-data www-data Alt Text mechanize-2.10.1/test/htdocs/form_no_input_name.html 0000644 0000041 0000041 00000000701 14645745627 022617 0 ustar www-data www-dataWelcome, robot!
mechanize-2.10.1/test/htdocs/file_upload.html 0000644 0000041 0000041 00000002027 14645745627 021227 0 ustar www-data www-dataFile Upload Test
mechanize-2.10.1/test/htdocs/tc_base_link.html 0000644 0000041 0000041 00000000173 14645745627 021361 0 ustar www-data www-datamechanize-2.10.1/test/htdocs/test_click.html 0000644 0000041 0000041 00000000521 14645745627 021065 0 ustar www-data www-data
Do not index nor archive this page!
mechanize-2.10.1/test/htdocs/tc_follow_meta_loop_1.html 0000644 0000041 0000041 00000000264 14645745627 023214 0 ustar www-data www-data This page has a meta refresh. mechanize-2.10.1/test/htdocs/dir with spaces/ 0000755 0000041 0000041 00000000000 14645745627 021026 5 ustar www-data www-data mechanize-2.10.1/test/htdocs/dir with spaces/foo.html 0000644 0000041 0000041 00000000007 14645745627 022474 0 ustar www-data www-data HELLO mechanize-2.10.1/test/htdocs/form_test.html 0000644 0000041 0000041 00000013655 14645745627 020757 0 ustar www-data www-dataPost Form 1
Get Form 1
Post Form 2
Get Form 2
Post Form 3
Post Form 4 - Multipart
Get Form 3
form test mechanize-2.10.1/test/htdocs/tc_encoded_links.html 0000644 0000041 0000041 00000000125 14645745627 022230 0 ustar www-data www-data test link mechanize-2.10.1/test/test_mechanize_file_connection.rb 0000644 0000041 0000041 00000001720 14645745627 023336 0 ustar www-data www-data # frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeFileConnection < Mechanize::TestCase def test_request file_path = File.expand_path(__FILE__) uri = URI.parse "file://#{file_path}" conn = Mechanize::FileConnection.new body = +'' conn.request uri, nil do |response| assert_equal(file_path, response.file_path) response.read_body do |part| body << part end end assert_equal File.read(__FILE__), body.gsub(/\r\n/, "\n") end def test_request_on_uri_with_windows_drive uri_string = "file://C:/path/to/file.html" expected_file_path = "C:/path/to/file.html" uri = URI.parse(uri_string) conn = Mechanize::FileConnection.new called = false yielded_file_path = nil conn.request(uri, nil) do |response| called = true yielded_file_path = response.file_path end assert(called) assert_equal(expected_file_path, yielded_file_path) end end mechanize-2.10.1/test/test_mechanize_download.rb 0000644 0000041 0000041 00000004275 14645745627 022017 0 ustar www-data www-data # frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeDownload < Mechanize::TestCase def setup super @parser = Mechanize::Download end def test_body uri = URI.parse 'http://example/foo.html' body_io = StringIO.new '0123456789' download = @parser.new uri, nil, body_io assert_equal '0123456789', download.body assert_equal 0, download.body_io.pos end def test_save_string_io uri = URI.parse 'http://example/foo.html' body_io = StringIO.new '0123456789' download = @parser.new uri, nil, body_io in_tmpdir do filename = download.save assert File.exist? 'foo.html' assert_equal "foo.html", filename end end def test_save_bang uri = URI.parse 'http://example/foo.html' body_io = StringIO.new '0123456789' download = @parser.new uri, nil, body_io in_tmpdir do filename = download.save! assert File.exist? 'foo.html' assert_equal "foo.html", filename end end def test_save_bang_does_not_allow_command_injection skip if windows? uri = URI.parse 'http://example/foo.html' body_io = StringIO.new '0123456789' download = @parser.new uri, nil, body_io in_tmpdir do download.save!('| ruby -rfileutils -e \'FileUtils.touch("vul.txt")\'') refute_operator(File, :exist?, "vul.txt") end end def test_save_tempfile uri = URI.parse 'http://example/foo.html' Tempfile.open @NAME do |body_io| body_io.unlink body_io.write '0123456789' body_io.flush body_io.rewind download = @parser.new uri, nil, body_io in_tmpdir do filename = download.save assert File.exist? 'foo.html' assert_equal "foo.html", filename filename = download.save assert File.exist? 'foo.html.1' assert_equal "foo.html.1", filename filename = download.save assert File.exist? 'foo.html.2' assert_equal "foo.html.2", filename end end end def test_filename uri = URI.parse 'http://example/foo.html' body_io = StringIO.new '0123456789' download = @parser.new uri, nil, body_io assert_equal "foo.html", download.filename end end mechanize-2.10.1/test/test_mechanize_redirect_limit_reached_error.rb 0000644 0000041 0000041 00000000717 14645745627 026070 0 ustar www-data www-data # frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeRedirectLimitReachedError < Mechanize::TestCase def setup super @error = Mechanize::RedirectLimitReachedError.new fake_page, 10 end def test_message assert_equal 'Redirect limit of 10 reached', @error.message end def test_redirects assert_equal 10, @error.redirects end def test_response_code assert_equal 200, @error.response_code end end mechanize-2.10.1/test/test_mechanize_file_request.rb 0000644 0000041 0000041 00000001004 14645745627 022662 0 ustar www-data www-data # frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeFileRequest < Mechanize::TestCase def setup @uri = URI.parse 'file:///nonexistent' @r = Mechanize::FileRequest.new @uri end def test_initialize assert_equal @uri, @r.uri assert_equal '/nonexistent', @r.path assert_respond_to @r, :[]= assert_respond_to @r, :add_field assert_respond_to @r, :each_header end def test_response_body_permitted_eh assert @r.response_body_permitted? end end mechanize-2.10.1/test/test_mechanize_page.rb 0000644 0000041 0000041 00000015567 14645745627 021132 0 ustar www-data www-data # frozen_string_literal: true require 'mechanize/test_case' class TestMechanizePage < Mechanize::TestCase def setup super @uri = URI 'http://example/' end def test_selector_methods page = html_page <<-BODY
BODY
assert_equal "http://example/b.jpg",
page.image_with(:src => 'b.jpg').url.to_s
end
def test_images_with
page = html_page <<-BODY
BODY
images = page.images_with(:src => /jpg\Z/).map { |img| img.url.to_s }
assert_equal %w[http://example/a.jpg http://example/b.jpg], images
end
def test_links
page = html_page <<-BODY
BODY
assert_equal page.links.first.href, "foo.html"
end
def test_parser_no_attributes
page = html_page <<-BODY
BODY
{
:search => "//img[@class='pretty']",
:xpath => "//img[@class='pretty']",
:css => "img.pretty",
:class => "pretty",
:dom_class => "pretty",
}.each { |key, expr|
images = page.images_with(key => expr)
message = "selecting with #{key.inspect}"
assert_equal 2, images.size
assert_equal "pretty", images[0].dom_class, message
assert_equal "a.jpg", images[0].src, message
assert_equal "pretty", images[1].dom_class, message
assert_equal "c.png", images[1].src, message
}
end
def test_search_bad_selectors
page = html_page <<-BODY
foo
BODY
assert_empty page.images_with(:search => '//a')
assert_empty page.links_with(:search => '//img')
end
def test_multiple_titles
page = html_page <<-BODY
Hello World
HTML html_page(html) # refute_raises end end mechanize-2.10.1/test/test_mechanize_parser.rb 0000644 0000041 0000041 00000015410 14645745627 021475 0 ustar www-data www-data # frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeParser < Mechanize::TestCase class P include Mechanize::Parser attr_accessor :filename attr_accessor :response attr_accessor :uri def initialize @uri = URI 'http://example' @full_path = false end end def setup super @parser = P.new end def test_extract_filename @parser.response = {} assert_equal 'index.html', @parser.extract_filename end def test_extract_filename_content_disposition @parser.uri = URI 'http://example/foo' @parser.response = { 'content-disposition' => 'attachment; filename=genome.jpeg' } assert_equal 'genome.jpeg', @parser.extract_filename end def test_extract_filename_content_disposition_bad @parser.uri = URI 'http://example/foo' @parser.response = { 'content-disposition' => "inline; filename*=UTF-8''X%20Y.jpg" } assert_equal 'foo.html', @parser.extract_filename @parser.response = { 'content-disposition' => "inline; filename=\"\"" } assert_equal 'foo.html', @parser.extract_filename end def test_extract_filename_content_disposition_path @parser.uri = URI 'http://example' @parser.response = { 'content-disposition' => 'attachment; filename="../genome.jpeg"' } assert_equal 'example/genome.jpeg', @parser.extract_filename(true) @parser.response = { 'content-disposition' => 'attachment; filename="foo/genome.jpeg"' } assert_equal 'example/genome.jpeg', @parser.extract_filename(true) end def test_extract_filename_content_disposition_path_windows @parser.uri = URI 'http://example' @parser.response = { 'content-disposition' => 'attachment; filename="..\\\\genome.jpeg"' } assert_equal 'example/genome.jpeg', @parser.extract_filename(true) @parser.response = { 'content-disposition' => 'attachment; filename="foo\\\\genome.jpeg"' } assert_equal 'example/genome.jpeg', @parser.extract_filename(true) end def test_extract_filename_content_disposition_full_path @parser.uri = URI 'http://example/foo' @parser.response = { 'content-disposition' => 'attachment; filename=genome.jpeg' } assert_equal 'example/genome.jpeg', @parser.extract_filename(true) end def test_extract_filename_content_disposition_quoted @parser.uri = URI 'http://example' @parser.response = { 'content-disposition' => 'attachment; filename="\"some \"file\""' } assert_equal '_some__file_', @parser.extract_filename end def test_extract_filename_content_disposition_special @parser.uri = URI 'http://example/foo' @parser.response = { 'content-disposition' => 'attachment; filename="/\\\\<>:\\"|?*"' } assert_equal '_______', @parser.extract_filename chars = (0..12).map { |c| c.chr }.join chars += "\\\r" chars += (14..31).map { |c| c.chr }.join @parser.response = { 'content-disposition' => "attachment; filename=\"#{chars}\"" } assert_equal '_' * 32, @parser.extract_filename end def test_extract_filename_content_disposition_windows_special @parser.uri = URI 'http://example' windows_special = %w[ AUX COM1 COM2 COM3 COM4 COM5 COM6 COM7 COM8 COM9 CON LPT1 LPT2 LPT3 LPT4 LPT5 LPT6 LPT7 LPT8 LPT9 NUL PRN ] windows_special.each do |special| @parser.response = { 'content-disposition' => "attachment; filename=#{special}" } assert_equal "_#{special}", @parser.extract_filename end end def test_extract_filename_content_disposition_empty @parser.uri = URI 'http://example' @parser.response = { 'content-disposition' => 'inline; filename="/"' } assert_equal '', @parser.extract_filename end def test_extract_filename_host @parser.response = {} @parser.uri = URI 'http://example' assert_equal 'example/index.html', @parser.extract_filename(true) end def test_extract_filename_special_character @parser.response = {} invisible = "\t\n\v\f\r" invisible.chars.each do |char| begin @parser.uri = URI "http://example/#{char}" assert_equal 'index.html', @parser.extract_filename, char.inspect rescue URI::InvalidURIError # ignore end end escaped = "<>\"\\|" escaped.chars.each do |char| escaped_char = CGI.escape char @parser.uri = URI "http://example/#{escaped_char}" assert_equal "#{escaped_char}.html", @parser.extract_filename, char end @parser.uri = URI "http://example/?" assert_equal 'index.html_', @parser.extract_filename, 'empty query' @parser.uri = URI "http://example/:" assert_equal '_.html', @parser.extract_filename, 'colon' @parser.uri = URI "http://example/*" assert_equal '_.html', @parser.extract_filename, 'asterisk' end def test_extract_filename_uri @parser.response = {} @parser.uri = URI 'http://example/foo' assert_equal 'foo.html', @parser.extract_filename @parser.uri += '/foo.jpg' assert_equal 'foo.jpg', @parser.extract_filename end def test_extract_filename_uri_full_path @parser.response = {} @parser.uri = URI 'http://example/foo' assert_equal 'example/foo.html', @parser.extract_filename(true) @parser.uri += '/foo.jpg' assert_equal 'example/foo.jpg', @parser.extract_filename(true) end def test_extract_filename_uri_query @parser.response = {} @parser.uri = URI 'http://example/?id=5' assert_equal 'index.html_id=5', @parser.extract_filename @parser.uri += '/foo.html?id=5' assert_equal 'foo.html_id=5', @parser.extract_filename end def test_extract_filename_uri_slash @parser.response = {} @parser.uri = URI 'http://example/foo/' assert_equal 'example/foo/index.html', @parser.extract_filename(true) @parser.uri += '/foo///' assert_equal 'example/foo/index.html', @parser.extract_filename(true) end def test_extract_filename_windows_special @parser.uri = URI 'http://example' @parser.response = {} windows_special = %w[ AUX COM1 COM2 COM3 COM4 COM5 COM6 COM7 COM8 COM9 CON LPT1 LPT2 LPT3 LPT4 LPT5 LPT6 LPT7 LPT8 LPT9 NUL PRN ] windows_special.each do |special| @parser.uri += "/#{special}" assert_equal "_#{special}.html", @parser.extract_filename end end def test_fill_header @parser.fill_header 'a' => 'b' expected = { 'a' => 'b' } assert_equal expected, @parser.response end def test_fill_header_nil @parser.fill_header nil assert_empty @parser.response end end mechanize-2.10.1/test/test_mechanize_http_auth_challenge.rb 0000644 0000041 0000041 00000002604 14645745627 024204 0 ustar www-data www-data # frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeHttpAuthChallenge < Mechanize::TestCase def setup super @uri = URI 'http://example/' @AR = Mechanize::HTTP::AuthRealm @AC = Mechanize::HTTP::AuthChallenge @challenge = @AC.new 'Digest', { 'realm' => 'r' }, 'Digest realm=r' end def test_realm_basic @challenge.scheme = 'Basic' expected = @AR.new 'Basic', @uri, 'r' assert_equal expected, @challenge.realm(@uri + '/foo') end def test_realm_digest expected = @AR.new 'Digest', @uri, 'r' assert_equal expected, @challenge.realm(@uri + '/foo') end def test_realm_digest_case challenge = @AC.new 'Digest', { 'realm' => 'R' }, 'Digest realm=R' expected = @AR.new 'Digest', @uri, 'R' assert_equal expected, challenge.realm(@uri + '/foo') end def test_realm_unknown @challenge.scheme = 'Unknown' e = assert_raises Mechanize::Error do @challenge.realm(@uri + '/foo') end assert_equal 'unknown HTTP authentication scheme Unknown', e.message end def test_realm_name assert_equal 'r', @challenge.realm_name end def test_realm_name_case challenge = @AC.new 'Digest', { 'realm' => 'R' }, 'Digest realm=R' assert_equal 'R', challenge.realm_name end def test_realm_name_ntlm challenge = @AC.new 'Negotiate, NTLM' assert_nil challenge.realm_name end end mechanize-2.10.1/test/test_mechanize_pluggable_parser.rb 0000644 0000041 0000041 00000002653 14645745627 023524 0 ustar www-data www-data # frozen_string_literal: true require 'mechanize/test_case' class TestMechanizePluggableParser < Mechanize::TestCase def setup super @pp = @mech.pluggable_parser end def test_aref @pp['text/html'] = Mechanize::Download assert_equal Mechanize::Download, @pp['text/html'] end def test_csv @pp.csv = Mechanize::Download assert_equal Mechanize::Download, @pp['text/csv'] end def test_html assert_equal Mechanize::Page, @pp['text/html'] @pp.html = Mechanize::Download assert_equal Mechanize::Download, @pp['text/html'] end def test_parser assert_equal Mechanize::XmlFile, @pp.parser('text/xml') assert_equal Mechanize::File, @pp.parser(nil) end def test_parser_mime @pp['image/png'] = :png assert_equal :png, @pp.parser('x-image/x-png') assert_equal :png, @pp.parser('image/png') assert_equal Mechanize::Image, @pp.parser('image') end def test_parser_bogus assert_nil @pp['bogus'] assert_equal Mechanize::File, @pp.parser('bogus') end def test_pdf @pp.pdf = Mechanize::Download assert_equal Mechanize::Download, @pp['application/pdf'] end def test_xml assert_equal Mechanize::XmlFile, @pp['text/xml'] assert_equal Mechanize::XmlFile, @pp['application/xml'] @pp.xml = Mechanize::Download assert_equal Mechanize::Download, @pp['text/xml'] assert_equal Mechanize::Download, @pp['application/xml'] end end mechanize-2.10.1/test/test_mechanize_headers.rb 0000644 0000041 0000041 00000001500 14645745627 021607 0 ustar www-data www-data # frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeHeaders < Mechanize::TestCase def setup super @headers = Mechanize::Headers.new @headers['content-type'] = 'text/html' @headers['Content-encoding'] = 'gzip' @headers['SERVER'] = 'Apache/2.2' end def test_aref assert_equal('Apache/2.2', @headers['server']) assert_equal('text/html', @headers['Content-Type']) end def test_key? assert_equal(true, @headers.key?('content-Encoding')) end def test_canonical_each all_keys = ['Content-Type', 'Content-Encoding', 'Server'] keys = all_keys.dup @headers.canonical_each { |key, value| case keys.delete(key) when *all_keys # ok else flunk "unexpected key: #{key}" end } assert_equal([], keys) end end mechanize-2.10.1/test/test_mechanize_form_textarea.rb 0000644 0000041 0000041 00000003105 14645745627 023037 0 ustar www-data www-data # frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeFormTextarea < Mechanize::TestCase def setup super @page = @mech.get("http://localhost/tc_textarea.html") end def test_empty_text_area form = @page.forms_with(:name => 'form1').first assert_equal('', form.field_with(:name => 'text1').value) form.text1 = 'Hello World' assert_equal('Hello World', form.field_with(:name => 'text1').value) page = @mech.submit(form) assert_equal(1, page.links.length) assert_equal('text1:Hello World', page.links[0].text) end def test_non_empty_textfield form = @page.forms_with(:name => 'form2').first assert_equal('sample text', form.field_with(:name => 'text1').value) page = @mech.submit(form) assert_equal(1, page.links.length) assert_equal('text1:sample text', page.links[0].text) end def test_multi_textfield form = @page.form_with(:name => 'form3') assert_equal(2, form.fields_with(:name => 'text1').length) assert_equal('', form.fields_with(:name => 'text1')[0].value) assert_equal('sample text', form.fields_with(:name => 'text1')[1].value) form.text1 = 'Hello World' assert_equal('Hello World', form.fields_with(:name => 'text1')[0].value) assert_equal('sample text', form.fields_with(:name => 'text1')[1].value) page = @mech.submit(form) assert_equal(2, page.links.length) link = page.links_with(:text => 'text1:sample text') assert_equal(1, link.length) link = page.links_with(:text => 'text1:Hello World') assert_equal(1, link.length) end end mechanize-2.10.1/test/test_mechanize_page_link.rb 0000644 0000041 0000041 00000024227 14645745627 022140 0 ustar www-data www-data # coding: utf-8 # frozen_string_literal: true require 'mechanize/test_case' puts "Nokogiri::VERSION_INFO: #{Nokogiri::VERSION_INFO}" class TestMechanizePageLink < Mechanize::TestCase WINDOWS_1255 = <<-HTML
BODY
assert_equal "http://other.example/a.jpg", page.images.first.url
end
def test_extname
assert_equal ".jpg", img("src" => "a.jpg").extname
assert_equal ".PNG", img("src" => "a.PNG").extname
assert_equal ".aaa", img("src" => "unknown.aaa").extname
assert_equal "", img("src" => "nosuffiximage").extname
assert_nil img("width" => "1", "height" => "1").extname
assert_equal ".jpg", img("src" => "a.jpg?cache_buster").extname
end
def test_mime_type
assert_equal "image/jpeg", img("src" => "a.jpg").mime_type
assert_equal "image/png", img("src" => "a.PNG").mime_type
assert_nil img("src" => "unknown.aaa").mime_type
assert_nil img("src" => "nosuffiximage").mime_type
end
def test_fetch
image = img "src" => "http://localhost/button.jpg"
fetched = image.fetch
assert_equal fetched, @mech.page
assert_equal "http://localhost/button.jpg", fetched.uri.to_s
assert_equal "http://example/", requests.first['Referer']
assert @mech.visited? "http://localhost/button.jpg"
end
def test_fetch_referer_http_page_rel_src
# | rel-src http-src https-src
# http page | *page* page page
# https page | page empty empty
page = html_page '
'
page.images.first.fetch
assert_equal 'http', page.uri.scheme
assert_equal true, page.images.first.relative?
assert_equal "http://example/", requests.first['Referer']
end
def test_fetch_referer_http_page_abs_src
# | rel-src http-src https-src
# http page | page *page* *page*
# https page | page empty empty
page = html_page '
'
page.images.first.fetch
assert_equal 'http', page.uri.scheme
assert_equal false, page.images.first.relative?
assert_equal "http://example/", requests.first['Referer']
end
def test_fetch_referer_https_page_rel_src
# | rel-src http-src https-src
# http page | page page page
# https page | *page* empty empty
page = html_page ''
page.uri = URI 'https://example/'
page.images.first.fetch
assert_equal 'https', page.uri.scheme
assert_equal true, page.images.first.relative?
assert_equal "https://example/", requests.first['Referer']
end
def test_fetch_referer_https_page_abs_src
# | rel-src http-src https-src
# http page | page page page
# https page | page *empty* *empty*
page = html_page ''
page.uri = URI 'https://example/'
page.images.first.fetch
assert_equal 'https', page.uri.scheme
assert_equal false, page.images.first.relative?
assert_nil requests.first['Referer']
end
def test_image_referer_http_page_abs_src
page = html_page ''
assert_equal 'http', page.uri.scheme
assert_equal @uri, page.images.first.image_referer.uri
end
def test_image_referer_http_page_rel_src
page = html_page ''
assert_equal 'http', page.uri.scheme
assert_equal @uri, page.images.first.image_referer.uri
end
def test_image_referer_https_page_abs_src
page = html_page ''
page.uri = URI 'https://example/'
assert_equal 'https', page.uri.scheme
assert_nil page.images.first.image_referer.uri
end
def test_image_referer_https_page_rel_src
page = html_page ''
page.uri = URI 'https://example/'
assert_equal 'https', page.uri.scheme
assert_equal URI('https://example/'), page.images.first.image_referer.uri
end
def test_no_src_attribute
page = html_page 'Do not follow this or this! BODY page.links[0].click page.links[1].click @mech.robots = true assert_raises Mechanize::RobotsDisallowedError do page.links[0].click end assert_raises Mechanize::RobotsDisallowedError do page.links[1].click end end def test_click_link_noreferrer link = node 'a', 'href' => '/index.html', 'rel' => 'noreferrer' link = Mechanize::Page::Link.new link, @mech, fake_page @mech.click link assert_nil requests.first['referer'] end def test_click_link_rel_nofollow page = html_page <<-BODY
You can follow this link but not this! BODY page.links[0].click page.links[1].click @mech.robots = true page.links[0].click assert_raises Mechanize::RobotsDisallowedError do page.links[1].click end end def test_click_link_parent page = page URI 'http://example/a/index.html' link = node 'a', 'href' => '../index.html' link = Mechanize::Page::Link.new link, @mech, page @mech.click link assert_equal '/index.html', requests.first.path end def test_click_link_parent_extra page = page URI 'http://example/a/index.html' link = node 'a', 'href' => '../../index.html' link = Mechanize::Page::Link.new link, @mech, page @mech.click link assert_equal '/index.html', requests.first.path end def test_click_link_hpricot_style # HACK move to test_search in Page page = @mech.get("http://localhost/tc_encoded_links.html") page = @mech.click(page.search('a').first) assert_equal("http://localhost/form_post?a=b&b=c", page.uri.to_s) end def test_click_link_query page = @mech.get("http://localhost/tc_encoded_links.html") link = page.links.first assert_equal('/form_post?a=b&b=c', link.href) page = @mech.click(link) assert_equal("http://localhost/form_post?a=b&b=c", page.uri.to_s) end def test_click_link_space page = @mech.get("http://localhost/tc_bad_links.html") @mech.click page.links.first assert_match(/alt_text.html$/, @mech.history.last.uri.to_s) assert_equal(2, @mech.history.length) end def test_click_more @mech.get 'http://localhost/test_click.html' @mech.click 'A Button' assert_equal 'http://localhost/frame_test.html?words=nil', @mech.page.uri.to_s @mech.back @mech.click 'A Link' assert_equal 'http://localhost/index.html', @mech.page.uri.to_s @mech.back @mech.click @mech.page.link_with(:text => 'A Link') assert_equal 'http://localhost/index.html', @mech.page.uri.to_s end def test_cookies uri = URI 'http://example' jar = HTTP::CookieJar.new jar.parse 'a=b', uri @mech.cookie_jar = jar refute_empty @mech.cookies end def test_cookie_jar assert_kind_of Mechanize::CookieJar, @mech.cookie_jar jar = HTTP::CookieJar.new @mech.cookie_jar = jar assert_equal jar, @mech.cookie_jar end def test_delete page = @mech.delete('http://localhost/verb', { 'q' => 'foo' }) assert_equal 1, @mech.history.length assert_equal 'DELETE', page.header['X-Request-Method'] end def test_delete_redirect page = @mech.delete('http://localhost/redirect') assert_equal(page.uri.to_s, 'http://localhost/verb') assert_equal 'GET', page.header['X-Request-Method'] end def test_download page = nil in_tmpdir do open 'download', 'w' do |io| page = @mech.download 'http://example', io refute io.closed? end assert_operator 1, :<=, File.stat('download').size end assert_empty @mech.history assert_kind_of Mechanize::Page, page end def test_download_filename page = nil in_tmpdir do page = @mech.download 'http://example', 'download' assert_operator 1, :<=, File.stat('download').size end assert_empty @mech.history assert_kind_of Mechanize::Page, page end def test_download_filename_error in_tmpdir do assert_raises Mechanize::UnauthorizedError do @mech.download 'http://example/digest_auth', 'download' end refute File.exist? 'download' end end def test_download_does_not_allow_command_injection skip if windows? in_tmpdir do @mech.download('http://example', '| ruby -rfileutils -e \'FileUtils.touch("vul.txt")\'') refute_operator(File, :exist?, "vul.txt") end end def test_get uri = URI 'http://localhost' page = @mech.get uri, { :q => 'h' }, 'http://example', { 'X-H' => 'v' } assert_equal URI('http://localhost/?q=h'), page.uri assert_equal URI('http://localhost'), uri end def test_get_HTTP page = @mech.get('HTTP://localhost/', { :q => 'hello' }) assert_kind_of URI::HTTP, page.uri assert_equal 'localhost', page.uri.host assert_equal 80, page.uri.port assert_equal '/?q=hello', page.uri.request_uri end def test_get_anchor page = @mech.get('http://localhost/?foo=bar"') assert_equal('http://localhost/?foo=bar%22', page.uri.to_s) end def test_get_bad_url assert_raises ArgumentError do @mech.get '/foo.html' end end def test_get_auth @mech.add_auth @uri, 'user', 'pass' page = @mech.get @uri + '/basic_auth' assert_equal 'You are authenticated', page.body end def test_get_auth_bad @mech.add_auth(@uri, 'aaron', 'aaron') e = assert_raises Mechanize::UnauthorizedError do @mech.get(@uri + '/basic_auth') end assert_equal("401", e.response_code) end def test_get_auth_none e = assert_raises Mechanize::UnauthorizedError do @mech.get(@uri + '/basic_auth') end assert_equal("401", e.response_code) end def test_get_auth_realm @mech.add_auth @uri, 'user', 'pass', 'Blah' page = @mech.get @uri + '/basic_auth' assert_equal 'You are authenticated', page.body end def test_get_conditional assert_empty @mech.history page = @mech.get 'http://localhost/if_modified_since' assert_match(/You did not send/, page.body) assert_equal 1, @mech.history.length page2 = @mech.get 'http://localhost/if_modified_since' assert_equal 2, @mech.history.length assert_equal page.object_id, page2.object_id end def test_get_digest_auth @mech.add_auth @uri, 'user', 'pass' page = @mech.get @uri + '/digest_auth' assert_equal 'You are authenticated', page.body end def test_get_follow_meta_refresh @mech.follow_meta_refresh = true page = @mech.get('http://localhost/tc_follow_meta.html') assert_equal(2, @mech.history.length) assert_equal('http://localhost/tc_follow_meta.html', @mech.history.first.uri.to_s) assert_equal('http://localhost/index.html', page.uri.to_s) assert_equal('http://localhost/index.html', @mech.history.last.uri.to_s) [5, 6].each { |limit| @mech.redirection_limit = limit begin @mech.get('http://localhost/tc_follow_meta_loop_1.html') rescue => e assert_instance_of Mechanize::RedirectLimitReachedError, e assert_equal limit, e.redirects if limit % 2 == 0 assert_equal '/tc_follow_meta_loop_1.html', e.page.uri.path else assert_equal '/tc_follow_meta_loop_2.html', e.page.uri.path end end } end def test_get_follow_meta_refresh_anywhere @mech.follow_meta_refresh = :anywhere @mech.get('http://localhost/tc_meta_in_body.html') assert_equal 2, requests.length end def test_get_follow_meta_refresh_disabled page = @mech.get('http://localhost/tc_follow_meta.html') assert_equal('http://localhost/tc_follow_meta.html', page.uri.to_s) assert_equal(1, page.meta_refresh.length) end def test_get_follow_meta_refresh_empty_url @mech.follow_meta_refresh = true @mech.follow_meta_refresh_self = true page = @mech.get('http://example/refresh_with_empty_url') assert_equal(3, @mech.history.length) assert_equal('http://example/refresh_with_empty_url', @mech.history[0].uri.to_s) assert_equal('http://example/refresh_with_empty_url', @mech.history[1].uri.to_s) assert_equal('http://example/', page.uri.to_s) assert_equal('http://example/', @mech.history.last.uri.to_s) end def test_get_follow_meta_refresh_in_body @mech.follow_meta_refresh = true @mech.get('http://localhost/tc_meta_in_body.html') assert_equal 1, requests.length end def test_get_follow_meta_refresh_no_url @mech.follow_meta_refresh = true @mech.follow_meta_refresh_self = true page = @mech.get('http://example/refresh_without_url') assert_equal(3, @mech.history.length) assert_equal('http://example/refresh_without_url', @mech.history[0].uri.to_s) assert_equal('http://example/refresh_without_url', @mech.history[1].uri.to_s) assert_equal('http://example/', page.uri.to_s) assert_equal('http://example/', @mech.history.last.uri.to_s) end def test_get_follow_meta_refresh_referer_not_sent @mech.follow_meta_refresh = true @mech.get('http://localhost/tc_follow_meta.html') assert_equal 2, @mech.history.length assert_nil requests.last['referer'] end def test_get_referer_download download = Mechanize::Download.new URI 'http://example/prev' uri = URI 'http://example' page = @mech.get uri, { :q => 'h' }, download, { 'X-H' => 'v' } assert_equal URI('http://example/?q=h'), page.uri assert_equal URI('http://example'), uri assert_equal 'http://example/prev', requests.first['referer'] end def test_get_robots @mech.robots = true assert_equal "Page Title", @mech.get("http://localhost/index.html").title assert_raises Mechanize::RobotsDisallowedError do @mech.get "http://localhost/norobots.html" end end def test_follow_meta_refresh_self refute @mech.agent.follow_meta_refresh_self @mech.follow_meta_refresh_self = true assert @mech.agent.follow_meta_refresh_self end def test_get_gzip page = @mech.get("http://localhost/gzip?file=index.html") assert_kind_of(Mechanize::Page, page) assert_match('Hello World', page.body) end def test_content_encoding_hooks_header h = {'X-ResponseContentEncoding' => 'agzip'} # test of X-ResponseContentEncoding feature assert_raises(Mechanize::Error, 'Unsupported Content-Encoding: agzip') do @mech.get("http://localhost/gzip?file=index.html", nil, nil, h) end @mech.content_encoding_hooks << lambda{|agent, uri, response, response_body_io| response['content-encoding'] = 'gzip' if response['content-encoding'] == 'agzip'} page = @mech.get("http://localhost/gzip?file=index.html", nil, nil, h) assert_match('Hello World', page.body) end def external_cmd(io); Zlib::GzipReader.new(io).read; end def test_content_encoding_hooks_body_io h = {'X-ResponseContentEncoding' => 'unsupported_content_encoding'} @mech.content_encoding_hooks << lambda{|agent, uri, response, response_body_io| if response['content-encoding'] == 'unsupported_content_encoding' response['content-encoding'] = 'none' response_body_io.string = external_cmd(response_body_io) end} page = @mech.get("http://localhost/gzip?file=index.html", nil, nil, h) assert_match('Hello World', page.body) end def test_get_http_refresh @mech.follow_meta_refresh = true page = @mech.get('http://example/http_refresh?refresh_time=0') assert_equal('http://example/', page.uri.to_s) assert_equal(2, @mech.history.length) assert_nil requests.last['referer'] end def test_get_http_refresh_delay @mech.follow_meta_refresh = true class << @mech.agent attr_accessor :slept def sleep *args @slept = args end end @mech.get('http://localhost/http_refresh?refresh_time=1') assert_equal [1], @mech.agent.slept end def test_get_http_refresh_disabled page = @mech.get('http://localhost/http_refresh?refresh_time=0') assert_equal('http://localhost/http_refresh?refresh_time=0', page.uri.to_s) end def test_get_query page = @mech.get('http://localhost/', { :q => 'hello' }) assert_equal('http://localhost/?q=hello', page.uri.to_s) page = @mech.get('http://localhost/', { :q => %w[hello world]}) assert_equal('http://localhost/?q=hello&q=world', page.uri.to_s) page = @mech.get('http://localhost/', { :paging => { start: 1, limit: 25 } }) assert_equal('http://localhost/?paging%5Bstart%5D=1&paging%5Blimit%5D=25', page.uri.to_s) end def test_get_redirect page = @mech.get('http://localhost/redirect') assert_equal(page.uri.to_s, 'http://localhost/verb') assert_equal 'GET', page.header['X-Request-Method'] end def test_get_redirect_found page = @mech.get('http://localhost/response_code?code=302&ct=test/xml') assert_equal('http://localhost/index.html', page.uri.to_s) assert_equal(2, @mech.history.length) end def test_get_redirect_infinite assert_raises(Mechanize::RedirectLimitReachedError) { @mech.get('http://localhost/infinite_refresh') } end def test_get_referer request = nil @mech.pre_connect_hooks << lambda { |_, req| request = req } @mech.get('http://localhost/', [], 'http://tenderlovemaking.com/') assert_equal 'http://tenderlovemaking.com/', request['Referer'] end def test_get_referer_file uri = URI 'http://tenderlovemaking.com/crossdomain.xml' file = Mechanize::File.new uri @mech.get('http://localhost', [], file) # HACK no assertion of behavior end def test_get_referer_none @mech.get('http://localhost/') @mech.get('http://localhost/') assert_equal(2, requests.length) requests.each do |request| assert_nil request['referer'] end end def test_get_scheme_unsupported assert_raises Mechanize::UnsupportedSchemeError do begin @mech.get('ftp://server.com/foo.html') rescue Mechanize::UnsupportedSchemeError => error assert_equal 'ftp', error.scheme assert_equal 'ftp://server.com/foo.html', error.uri.to_s raise end end end def test_get_space @mech.get("http://localhost/tc_bad_links.html ") assert_match(/tc_bad_links.html$/, @mech.history.last.uri.to_s) assert_equal(1, @mech.history.length) end def test_get_tilde page = @mech.get('http://localhost/?foo=~2') assert_equal('http://localhost/?foo=~2', page.uri.to_s) end def test_get_weird @mech.get('http://localhost/?action=bing&bang=boom=1|a=|b=|c=') @mech.get('http://localhost/?a=b&b=c&c=d') @mech.get("http://localhost/?a=#{[0xd6].pack('U')}") # HACK no assertion of behavior end def test_get_yield pages = nil @mech.get("http://localhost/file_upload.html") { |page| pages = page } assert pages assert_equal('File Upload Form', pages.title) end def test_get_file body = @mech.get_file 'http://localhost/frame_test.html' assert_kind_of String, body refute_empty body end def test_get_file_download # non-Mechanize::File body = @mech.get_file 'http://localhost/button.jpg' assert_kind_of String, body refute_empty body end def test_head page = @mech.head('http://localhost/verb', { 'q' => 'foo' }) assert_equal 0, @mech.history.length assert_equal 'HEAD', page.header['X-Request-Method'] end def test_head_redirect page = @mech.head('http://localhost/redirect') assert_equal(page.uri.to_s, 'http://localhost/verb') assert_equal 'HEAD', page.header['X-Request-Method'] end def test_history 2.times do |i| assert_equal(i, @mech.history.size) @mech.get("http://localhost/") end page = @mech.get("http://localhost/form_test.html") assert_equal("http://localhost/form_test.html", @mech.history.last.uri.to_s) assert_equal("http://localhost/", @mech.history[-2].uri.to_s) assert @mech.visited?("http://localhost/") assert @mech.visited?("/form_test.html"), 'relative' assert !@mech.visited?("http://google.com/") assert @mech.visited?(page.links.first) end def test_history_added_gets_called added_page = nil @mech.history_added = lambda { |page| added_page = page } assert_equal @mech.get('http://localhost/tc_blank_form.html'), added_page end def test_history_order @mech.max_history = 2 assert_equal(0, @mech.history.length) @mech.get('http://localhost/form_test.html') assert_equal(1, @mech.history.length) @mech.get('http://localhost/empty_form.html') assert_equal(2, @mech.history.length) @mech.get('http://localhost/tc_checkboxes.html') assert_equal(2, @mech.history.length) assert_equal('http://localhost/empty_form.html', @mech.history[0].uri.to_s) assert_equal('http://localhost/tc_checkboxes.html', @mech.history[1].uri.to_s) end def test_initialize mech = Mechanize.new assert_equal 50, mech.max_history end def test_html_parser_equals @mech.html_parser = {} assert_raises(NoMethodError) { @mech.get('http://localhost/?foo=~2').links } end def test_idle_timeout_default assert_equal 5, Mechanize.new.idle_timeout end def test_idle_timeout_equals @mech.idle_timeout = 15 assert_equal 15, @mech.idle_timeout end def test_keep_alive_equals assert @mech.keep_alive @mech.keep_alive = false refute @mech.keep_alive end def test_keep_alive_time assert_equal 0, @mech.keep_alive_time @mech.keep_alive_time = 1 assert_equal 1, @mech.keep_alive_time end def test_log assert_nil @mech.log end def test_log_equals @mech.log = Logger.new $stderr refute_nil @mech.log assert_nil Mechanize.log end def test_max_file_buffer_equals @mech.max_file_buffer = 1024 assert_equal 1024, @mech.agent.max_file_buffer end def test_max_history_equals @mech.max_history = 10 0.upto(10) do |i| assert_equal(i, @mech.history.size) @mech.get("http://localhost/") end 0.upto(10) do |i| assert_equal(10, @mech.history.size) @mech.get("http://localhost/") end end def test_open_timeout_equals @mech.open_timeout = 5 assert_equal 5, @mech.open_timeout end def test_parse_download @mech.pluggable_parser['application/octet-stream'] = Mechanize::Download response = Net::HTTPOK.allocate response.instance_variable_set(:@header, 'content-type' => ['application/octet-stream']) download = @mech.parse @uri, response, StringIO.new('raw') assert_kind_of Mechanize::Download, download assert_kind_of StringIO, download.content end def test_parse_html response = Net::HTTPOK.allocate response.instance_variable_set :@header, 'content-type' => ['text/html'] page = @mech.parse URI('http://example/'), response, '' assert_kind_of Mechanize::Page, page end def test_post @mech.post "http://example", 'gender' => 'female' assert_equal "gender=female", requests.first.body end def test_post_auth requests = [] @mech.pre_connect_hooks << proc { |agent, request| requests << request.class } @mech.add_auth(@uri, 'user', 'pass') page = @mech.post(@uri + '/basic_auth') assert_equal('You are authenticated', page.body) assert_equal(2, requests.length) r1 = requests[0] r2 = requests[1] assert_equal(r1, r2) end def test_post_entity @mech.post "http://localhost/form_post", 'json' => '["""]' assert_equal "json=%5B%22%22%22%5D", requests.first.body end def test_post_multiple_values @mech.post "http://localhost/form_post", [%w[gender female], %w[gender male]] assert_equal "gender=female&gender=male", requests.first.body end def test_post_multipart page = @mech.post('http://localhost/file_upload', { :name => 'Some file', :userfile1 => File.open(__FILE__) }) name = File.basename __FILE__ assert_match( "Content-Disposition: form-data; name=\"userfile1\"; filename=\"#{name}\"", page.body ) assert_operator page.body.bytesize, :>, file_contents_without_cr(__FILE__).length end def test_post_file_upload_nonascii name = 'ユーザファイル1' file_upload = Mechanize::Form::FileUpload.new({'name' => 'userfile1'}, name) file_upload.file_data = File.read(__FILE__) file_upload.mime_type = 'application/zip' page = @mech.post('http://localhost/file_upload', { :name => 'Some file', :userfile1 => file_upload }) assert_match( "Content-Disposition: form-data; name=\"userfile1\"; filename=\"#{name}\"".dup.force_encoding(Encoding::ASCII_8BIT), page.body ) assert_match("Content-Type: application/zip", page.body) assert_operator page.body.bytesize, :>, file_contents_without_cr(__FILE__).length end def test_post_file_upload name = File.basename(__FILE__) file_upload = Mechanize::Form::FileUpload.new({'name' => 'userfile1'}, name) file_upload.file_data = File.read(__FILE__) file_upload.mime_type = 'application/zip' page = @mech.post('http://localhost/file_upload', { :name => 'Some file', :userfile1 => file_upload }) assert_match( "Content-Disposition: form-data; name=\"userfile1\"; filename=\"#{name}\"", page.body ) assert_match("Content-Type: application/zip", page.body) assert_operator page.body.bytesize, :>, file_contents_without_cr(__FILE__).length end def test_post_redirect page = @mech.post('http://localhost/redirect') assert_equal(page.uri.to_s, 'http://localhost/verb') assert_equal 'GET', page.header['X-Request-Method'] end def test_put page = @mech.put('http://localhost/verb', 'foo') assert_equal 1, @mech.history.length assert_equal 'PUT', page.header['X-Request-Method'] end def test_put_redirect page = @mech.put('http://localhost/redirect', 'foo') assert_equal(page.uri.to_s, 'http://localhost/verb') assert_equal 'GET', page.header['X-Request-Method'] end def test_read_timeout_equals @mech.read_timeout = 5 assert_equal 5, @mech.read_timeout end def test_timeouts_for_file_connection uri = URI.parse "file://#{File.expand_path __FILE__}" @mech.read_timeout = 5 @mech.open_timeout = 5 assert @mech.get(uri) end def test_referer host_path = "localhost/tc_referer.html?t=1" ['http', 'https'].each { |proto| referer = "#{proto}://#{host_path}" [ "", "@", "user1@", ":@", "user1:@", ":password1@", "user1:password1@", ].each { |userinfo| url = "#{proto}://#{userinfo}#{host_path}" [url, url + "#foo"].each { |furl| [ ['relative', true], ['insecure', proto == 'http'], ['secure', true], ['relative noreferrer', false], ['insecure noreferrer', false], ['secure noreferrer', false], ].each_with_index { |(type, bool), i| rpage = @mech.get(furl) page = rpage.links[i].click assert_equal bool ? referer : '', page.body, "%s link from %s" % [type, furl] } rpage = @mech.get(furl) page = rpage.forms.first.submit assert_equal referer, page.body, "post from %s" % furl } } } end def test_retry_change_requests_equals unless Gem::Requirement.new("< 4.0.0").satisfied_by?(Gem::Version.new(Net::HTTP::Persistent::VERSION)) # see https://github.com/drbrain/net-http-persistent/pull/100 skip("net-http-persistent 4.0.0 and later does not support retry_change_requests") end refute @mech.retry_change_requests @mech.retry_change_requests = true assert @mech.retry_change_requests end def test_set_proxy http = @mech.agent.http @mech.set_proxy 'localhost', 8080, 'user', 'pass' assert_equal 'localhost', @mech.proxy_addr assert_equal 8080, @mech.proxy_port assert_equal 'user', @mech.proxy_user assert_equal 'pass', @mech.proxy_pass assert_equal URI('http://user:pass@localhost:8080'), http.proxy_uri end def test_shutdown uri = URI 'http://localhost' jar = HTTP::CookieJar.new jar.parse 'a=b', uri @mech.cookie_jar = jar @mech.get("http://localhost/") assert_match(/Hello World/, @mech.current_page.body) refute_empty @mech.cookies @mech.shutdown assert_empty @mech.history assert_empty @mech.cookies end def test_start body = nil Mechanize.start do |m| body = m.get("http://localhost/").body end assert_match(/Hello World/, body) end def test_submit_bad_form_method page = @mech.get("http://localhost/bad_form_test.html") assert_raises ArgumentError do @mech.submit(page.forms.first) end end def test_submit_check_one page = @mech.get('http://localhost/tc_checkboxes.html') form = page.forms.first form.checkboxes_with(:name => 'green')[1].check page = @mech.submit(form) assert_equal(1, page.links.length) assert_equal('green:on', page.links.first.text) end def test_submit_check_two page = @mech.get('http://localhost/tc_checkboxes.html') form = page.forms.first form.checkboxes_with(:name => 'green')[0].check form.checkboxes_with(:name => 'green')[1].check page = @mech.submit(form) assert_equal(2, page.links.length) assert_equal('green:on', page.links[0].text) assert_equal('green:on', page.links[1].text) end def test_submit_enctype page = @mech.get("http://localhost/file_upload.html") assert_equal('multipart/form-data', page.forms[0].enctype) form = page.forms.first form.file_uploads.first.file_name = __FILE__ form.file_uploads.first.mime_type = "text/plain" form.file_uploads.first.file_data = "Hello World\n\n" page = @mech.submit(form) basename = File.basename __FILE__ assert_match( "Content-Disposition: form-data; name=\"userfile1\"; filename=\"#{basename}\"", page.body ) assert_match( "Content-Disposition: form-data; name=\"name\"", page.body ) assert_match('Content-Type: text/plain', page.body) assert_match('Hello World', page.body) assert_match('foo[aaron]', page.body) end def test_submit_file_data page = @mech.get("http://localhost/file_upload.html") assert_equal('multipart/form-data', page.forms[1].enctype) form = page.forms[1] form.file_uploads.first.file_name = __FILE__ form.file_uploads.first.file_data = File.read __FILE__ page = @mech.submit(form) contents = File.binread(__FILE__).gsub(/\r\n/, "\n") basename = File.basename __FILE__ assert_match( "Content-Disposition: form-data; name=\"green[eggs]\"; filename=\"#{basename}\"", page.body ) assert_match(contents, page.body) end def test_submit_file_name page = @mech.get("http://localhost/file_upload.html") assert_equal('multipart/form-data', page.forms[1].enctype) form = page.forms[1] form.file_uploads.first.file_name = __FILE__ page = @mech.submit(form) contents = File.binread __FILE__ basename = File.basename __FILE__ assert_match( "Content-Disposition: form-data; name=\"green[eggs]\"; filename=\"#{basename}\"", page.body ) assert_match(contents, page.body) end def test_submit_get form = node 'form', 'method' => 'GET', 'action' => '/?a=b' form = Mechanize::Form.new form, @mech, fake_page @mech.submit form assert_equal '/', requests.first.path end def test_submit_headers page = @mech.get 'http://localhost:2000/form_no_action.html' assert form = page.forms.first form.action = '/http_headers' page = @mech.submit form, nil, 'foo' => 'bar' headers = page.body.split("\n").map { |x| x.split('|', 2) }.flatten headers = Hash[*headers] assert_equal 'bar', headers['foo'] end def test_submit_multipart page = @mech.get("http://localhost/file_upload.html") assert_equal('multipart/form-data', page.forms[1].enctype) form = page.forms[1] form.file_uploads.first.file_name = __FILE__ form.file_uploads.first.mime_type = "text/plain" form.file_uploads.first.file_data = "Hello World\n\n" page = @mech.submit(form) basename = File.basename __FILE__ assert_match( "Content-Disposition: form-data; name=\"green[eggs]\"; filename=\"#{basename}\"", page.body ) end def test_submit_no_file page = @mech.get("http://localhost/file_upload.html") form = page.forms.first form.field_with(:name => 'name').value = 'Aaron' @page = @mech.submit(form) assert_match('Aaron', @page.body) assert_match( "Content-Disposition: form-data; name=\"userfile1\"; filename=\"\"", @page.body ) end def test_submit_post form = node 'form', 'method' => 'POST', 'action' => '/?a=b' form = Mechanize::Form.new form, @mech, fake_page @mech.submit form assert_equal '/?a=b', requests.first.path end def test_submit_post_pound form = node 'form', 'method' => 'POST', 'action' => '/#1' form = Mechanize::Form.new form, @mech, fake_page @mech.submit form assert_equal '/', requests.first.path end def test_submit_too_many_radiobuttons page = @mech.get("http://localhost/form_test.html") form = page.form_with(:name => 'post_form1') form.radiobuttons.each { |r| r.checked = true } assert_raises Mechanize::Error do @mech.submit(form) end end def test_transact @mech.get("http://localhost/frame_test.html") assert_equal(1, @mech.history.length) @mech.transact { |a| 5.times { @mech.get("http://localhost/frame_test.html") } assert_equal(6, @mech.history.length) } assert_equal(1, @mech.history.length) end def test_user_agent_alias_equals_unknown assert_raises ArgumentError do @mech.user_agent_alias = "Aaron's Browser" end end def test_verify_mode assert_equal OpenSSL::SSL::VERIFY_PEER, @mech.verify_mode @mech.verify_mode = OpenSSL::SSL::VERIFY_NONE assert_equal OpenSSL::SSL::VERIFY_NONE, @mech.verify_mode end def test_visited_eh @mech.get("http://localhost/content_type_test?ct=application/pdf") assert \ @mech.visited?("http://localhost/content_type_test?ct=application/pdf") assert \ !@mech.visited?("http://localhost/content_type_test") assert \ !@mech.visited?("http://localhost/content_type_test?ct=text/html") end def test_visited_eh_link @mech.get("http://example/index.html") page = page URI 'http://example' link = node 'a', 'href' => '/index.html' link = Mechanize::Page::Link.new link, page, @mech assert @mech.visited? link end def test_visited_eh_redirect @mech.get("http://localhost/response_code?code=302") assert_equal("http://localhost/index.html", @mech.current_page.uri.to_s) assert @mech.visited?('http://localhost/response_code?code=302') end def test_no_frames_exists page = @mech.get("http://localhost/empty_form.html"); assert_nil page.frame_with(:name => 'noframe') assert_raises Mechanize::ElementNotFoundError do page.frame_with!(:name => 'noframe') end end def assert_header(page, header) headers = {} page.body.split(/[\r\n]+/).each do |page_header| headers.[]=(*page_header.chomp.split(/\|/)) end header.each do |key, value| assert(headers.has_key?(key)) assert_equal(value, headers[key]) end end end mechanize-2.10.1/test/test_mechanize_form_multi_select_list.rb 0000644 0000041 0000041 00000004061 14645745627 024750 0 ustar www-data www-data # frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeFormMultiSelectList < Mechanize::TestCase def setup super page = html_page <<-BODY
BODY form = page.forms.first @select = form.fields.first end def test_inspect assert_match "value: #{%w[2]}", @select.inspect end def test_inspect_select_all @select.select_all assert_match "value: #{%w[1 2 3 4 5 6]}", @select.inspect end def test_option_with option = @select.option_with value: '1' assert_equal '1', option.value option = @select.option_with search: 'option[@selected]' assert_equal '2', option.value end def test_options_with options = @select.options_with :value => /[12]/ assert_equal 2, options.length end def test_query_value assert_equal [%w[select 2]], @select.query_value @select.options.last.click assert_equal [%w[select 2], %w[select 6]], @select.query_value end def test_query_value_empty @select.options.last.click @select.options.last.instance_variable_set :@value, '' assert_equal [%w[select 2], ['select', '']], @select.query_value end def test_select_all @select.select_all assert_equal %w[1 2 3 4 5 6], @select.value end def test_select_none @select.select_none assert_empty @select.value end def test_selected_options assert_equal [@select.options[1]], @select.selected_options @select.options.last.click assert_equal [@select.options[1], @select.options.last], @select.selected_options end def test_value assert_equal %w[2], @select.value end def test_value_equals @select.value = %w[a 1 2] assert_equal %w[a 1 2], @select.value end end mechanize-2.10.1/.yardopts 0000644 0000041 0000041 00000000133 14645745627 015455 0 ustar www-data www-data --main=README.md lib/**/*.rb - README.md LICENSE.txt CHANGELOG.md GUIDE.rdoc EXAMPLES.rdoc mechanize-2.10.1/Rakefile 0000644 0000041 0000041 00000002737 14645745627 015270 0 ustar www-data www-data require 'rubygems' begin require "bundler/gem_tasks" rescue LoadError end require 'rdoc/task' require 'rake/testtask' task :prerelease => [:clobber_rdoc, :test] desc "Update SSL Certificate" task('ssl_cert') do |p| sh "openssl genrsa -des3 -out server.key 1024" sh "openssl req -new -key server.key -out server.csr" sh "cp server.key server.key.org" sh "openssl rsa -in server.key.org -out server.key" sh "openssl x509 -req -days 365 -in server.csr -signkey server.key -out server.crt" sh "cp server.key server.pem" sh "mv server.key server.csr server.crt server.pem test/data/" sh "rm server.key.org" end RDoc::Task.new do |rdoc| rdoc.main = "README.md" rdoc.rdoc_dir = 'doc' rdoc.rdoc_files.include( "CHANGELOG.md", "EXAMPLES.rdoc", "GUIDE.rdoc", "LICENSE.txt", "README.md", "lib/**/*.rb") end desc "Run tests" Rake::TestTask.new { |t| t.test_files = Dir['test/**/test*.rb'] t.verbose = true } task publish_docs: %w[rdoc] do sh 'rsync', '-avzO', '--delete', 'doc/', 'docs-push.seattlerb.org:/data/www/docs.seattlerb.org/mechanize/' end desc "Run rubocop checks" task :rubocop => ["rubocop:security", "rubocop:frozen_string_literals"] namespace "rubocop" do desc "Run rubocop security check" task :security do sh "rubocop lib --only Security" end desc "Run rubocop string literals check" task :frozen_string_literals do sh "rubocop lib test --auto-correct-all --only Style/FrozenStringLiteralComment" end end task default: [:rubocop, :test] mechanize-2.10.1/Gemfile 0000644 0000041 0000041 00000000047 14645745627 015106 0 ustar www-data www-data source "https://rubygems.org" gemspec mechanize-2.10.1/mechanize.gemspec 0000644 0000041 0000041 00000005762 14645745627 017134 0 ustar www-data www-data # coding: utf-8 # frozen_string_literal: true require_relative 'lib/mechanize/version' Gem::Specification.new do |spec| spec.name = "mechanize" spec.version = Mechanize::VERSION spec.homepage = "https://github.com/sparklemotion/mechanize" spec.summary = 'The Mechanize library is used for automating interaction with websites' spec.description = [ "The Mechanize library is used for automating interaction with websites.", "Mechanize automatically stores and sends cookies, follows redirects,", "and can follow links and submit forms. Form fields can be populated and", "submitted. Mechanize also keeps track of the sites that you have visited as", "a history.", ].join("\n") spec.authors = [ 'Eric Hodel', 'Aaron Patterson', 'Mike Dalessio', 'Akinori MUSHA', 'Lee Jarvis', ] spec.email = [ 'drbrain@segment7.net', 'aaron.patterson@gmail.com', 'mike.dalessio@gmail.com', 'knu@idaemons.org', 'ljjarvis@gmail.com', ] spec.metadata = { 'yard.run' => 'yard', 'bug_tracker_uri' => 'https://github.com/sparklemotion/mechanize/issues', 'changelog_uri' => 'https://github.com/sparklemotion/mechanize/blob/main/CHANGELOG.md', 'documentation_uri' => 'https://www.rubydoc.info/gems/mechanize', 'homepage_uri' => 'https://github.com/sparklemotion/mechanize', 'source_code_uri' => 'https://github.com/sparklemotion/mechanize' } spec.license = "MIT" spec.require_paths = ["lib"] spec.files = %x(git ls-files).split($/) spec.test_files = spec.files.grep(%r{^test/}) spec.extra_rdoc_files += Dir['*.rdoc', '*.md'] spec.rdoc_options = ["--main", "README.md"] spec.required_ruby_version = ">= 2.6.0" spec.add_runtime_dependency("addressable", "~> 2.8") spec.add_runtime_dependency("domain_name", ">= 0.5.20190701", "~> 0.5") spec.add_runtime_dependency("http-cookie", ">= 1.0.3", "~> 1.0") spec.add_runtime_dependency("mime-types", "~> 3.0") spec.add_runtime_dependency("net-http-digest_auth", ">= 1.4.1", "~> 1.4") # careful! some folks are relying on older versions of net-http-persistent # - see the socks proxy patch in use at #507 and #464 # - see use of retry_change_requests that was removed at #558 spec.add_runtime_dependency("net-http-persistent", ">= 2.5.2", "< 5.0.dev") spec.add_runtime_dependency("nokogiri", ">= 1.11.2", "~> 1.11") spec.add_runtime_dependency("webrick", "~> 1.7") spec.add_runtime_dependency("webrobots", "~> 0.1.2") spec.add_runtime_dependency("rubyntlm", ">= 0.6.3", "~> 0.6") spec.add_runtime_dependency("base64") # removed from bundled gems in 3.4, and needed by rubyntlm (which doesn't declare this dependency) spec.add_runtime_dependency("nkf") # removed from bundled gems in 3.4 spec.add_development_dependency("minitest", "~> 5.14") spec.add_development_dependency("rake", "~> 13.0") spec.add_development_dependency("rdoc", "~> 6.3") spec.add_development_dependency("rubocop", "~> 1.12") end mechanize-2.10.1/EXAMPLES.rdoc 0000644 0000041 0000041 00000011055 14645745627 015703 0 ustar www-data www-data = Mechanize examples Note: Several examples show methods chained to the end of do/end blocks.do...end is the same as curly braces ({...}). For
example, do ... end.submit is the same as { ...
}.submit.
== Google
require 'rubygems'
require 'mechanize'
a = Mechanize.new { |agent|
agent.user_agent_alias = 'Mac Safari'
}
a.get('http://google.com/') do |page|
search_result = page.form_with(:id => 'gbqf') do |search|
search.q = 'Hello world'
end.submit
search_result.links.each do |link|
puts link.text
end
end
== File Upload
Upload a file to flickr.
require 'rubygems'
require 'mechanize'
abort "#{$0} login passwd filename" if (ARGV.size != 3)
a = Mechanize.new { |agent|
# Flickr refreshes after login
agent.follow_meta_refresh = true
}
a.get('http://flickr.com/') do |home_page|
signin_page = a.click(home_page.link_with(:text => /Sign In/))
my_page = signin_page.form_with(:name => 'login_form') do |form|
form.login = ARGV[0]
form.passwd = ARGV[1]
end.submit
# Click the upload link
upload_page = a.click(my_page.link_with(:text => /Upload/))
# We want the basic upload page.
upload_page = a.click(upload_page.link_with(:text => /basic Uploader/))
# Upload the file
upload_page.form_with(:method => 'POST') do |upload_form|
upload_form.file_uploads.first.file_name = ARGV[2]
end.submit
end
== Pluggable Parsers
Let's say you want HTML pages to automatically be parsed with Rubyful Soup.
This example shows you how:
require 'rubygems'
require 'mechanize'
require 'rubyful_soup'
class SoupParser < Mechanize::Page
attr_reader :soup
def initialize(uri = nil, response = nil, body = nil, code = nil)
@soup = BeautifulSoup.new(body)
super(uri, response, body, code)
end
end
agent = Mechanize.new
agent.pluggable_parser.html = SoupParser
Now all HTML pages will be parsed with the SoupParser class, and automatically
give you access to a method called 'soup' where you can get access to the
Beautiful Soup for that page.
== Using a proxy
require 'rubygems'
require 'mechanize'
agent = Mechanize.new
agent.set_proxy 'localhost', 8000
page = agent.get(ARGV[0])
puts page.body
== The transact method
Mechanize#transact runs the given block and then resets the page history. I.e.
after the block has been executed, you're back at the original page; no need
to count how many times to call the back method at the end of a loop (while
accounting for possible exceptions).
This example also demonstrates subclassing Mechanize.
require 'rubygems'
require 'mechanize'
class TestMech < Mechanize
def process
get 'http://rubygems.org/'
search_form = page.forms.first
search_form.words = 'WWW'
submit search_form
page.links_with(:href => %r{/projects/} ).each do |link|
next if link.href =~ %r{/projects/support/}
puts 'Loading %-30s %s' % [link.href, link.text]
begin
transact do
click link
# Do stuff, maybe click more links.
end
# Now we're back at the original page.
rescue => e
$stderr.puts "#{e.class}: #{e.message}"
end
end
end
end
TestMech.new.process
== Client Certificate Authentication (Mutual Auth)
In most cases a client certificate is created as an additional layer of
security for certain websites. The specific case that this was initially
tested on was for automating the download of archived images from a banks
(Wachovia) lockbox system. Once the certificate is installed into your
browser you will have to export it and split the certificate and private key
into separate files.
require 'rubygems'
require 'mechanize'
# create Mechanize instance
agent = Mechanize.new
# set the path of the certificate file
agent.cert = 'example.cer'
# set the path of the private key file
agent.key = 'example.key'
# get the login form & fill it out with the username/password
login_form = agent.get("http://example.com/login_page").form('Login')
login_form.Userid = 'TestUser'
login_form.Password = 'TestPassword'
# submit login form
agent.submit(login_form, login_form.buttons.first)
Exported files are usually in .p12 format (IE 7 & Firefox 2.0) which stands
for PKCS #12. You can convert them from p12 to pem format by using the
following commands:
openssl pkcs12 -in input_file.p12 -clcerts -out example.key -nocerts -nodes
openssl pkcs12 -in input_file.p12 -clcerts -out example.cer -nokeys
mechanize-2.10.1/README.md 0000644 0000041 0000041 00000004036 14645745627 015074 0 ustar www-data www-data # Mechanize
* https://www.rubydoc.info/gems/mechanize/
* https://github.com/sparklemotion/mechanize
[](https://github.com/sparklemotion/mechanize/actions/workflows/ci.yml)
## Description
The Mechanize library is used for automating interaction with websites. Mechanize automatically stores and sends cookies, follows redirects, and can follow links and submit forms. Form fields can be populated and submitted. Mechanize also keeps track of the sites that you have visited as a history.
## Dependencies
* Ruby >= 2.6
* Gems:
* `addressable`
* `domain_name`
* `http-cookie`
* `mime-types`
* `net-http-digest_auth`
* `net-http-persistent`
* `nokogiri`
* `rubyntlm`
* `webrick`
* `webrobots`
## Support:
The bug tracker is available here:
* https://github.com/sparklemotion/mechanize/issues
## Examples
If you are just starting, check out [GUIDE.rdoc](https://github.com/sparklemotion/mechanize/blob/main/GUIDE.rdoc) or [EXAMPLES.rdoc](https://github.com/sparklemotion/mechanize/blob/main/EXAMPLES.rdoc).
## Developers
Use bundler to install dependencies:
```
bundle install
```
Run all tests with:
```
bundle exec rake test
```
See also Mechanize::TestCase to read about the built-in testing infrastructure.
## Authors
* Eric Hodel
* Akinori MUSHA
* Aaron Patterson
* Lee Jarvis
* Mike Dalessio
## Acknowledgments
This library was heavily influenced by its namesake in the Perl world. A big
thanks goes to [Andy Lester](http://petdance.com), the author of the original Perl module WWW::Mechanize which is available [here](http://search.cpan.org/dist/WWW-Mechanize/). Ruby Mechanize would not be around without you!
Thank you to Michael Neumann for starting the Ruby version. Thanks to everyone who's helped out in various ways. Finally, thank you to the people using this library!
## License
This library is distributed under the MIT license. Please see [LICENSE.txt](https://github.com/sparklemotion/mechanize/blob/main/LICENSE.txt).
mechanize-2.10.1/examples/ 0000755 0000041 0000041 00000000000 14645745627 015430 5 ustar www-data www-data mechanize-2.10.1/examples/mech-dump.rb 0000644 0000041 0000041 00000000136 14645745627 017634 0 ustar www-data www-data require 'rubygems'
require 'mechanize'
agent = Mechanize.new
puts agent.get(ARGV[0]).inspect
mechanize-2.10.1/examples/rubygems.rb 0000644 0000041 0000041 00000001231 14645745627 017607 0 ustar www-data www-data # This example logs a user in to rubygems and prints out the body of the
# page after logging the user in.
require 'rubygems'
require 'mechanize'
require 'logger'
# Create a new mechanize object
mech = Mechanize.new
mech.log = Logger.new $stderr
mech.agent.http.debug_output = $stderr
# Load the rubygems website
page = mech.get('https://rubygems.org/')
page = mech.click page.link_with(:text => /Sign in/) # Click the login link
form = page.forms[1] # Select the first form
form["session[who]"] = ARGV[0]
form["session[password]"] = ARGV[1]
form["commit"] = "Sign in"
# Submit the form
page = form.submit form.buttons.first
puts page.body # Print out the body
mechanize-2.10.1/examples/proxy_req.rb 0000644 0000041 0000041 00000000214 14645745627 020002 0 ustar www-data www-data require 'rubygems'
require 'mechanize'
agent = Mechanize.new
agent.set_proxy('localhost', '8000')
page = agent.get(ARGV[0])
puts page.body
mechanize-2.10.1/examples/wikipedia_links_to_philosophy.rb 0000644 0000041 0000041 00000006055 14645745627 024111 0 ustar www-data www-data require 'mechanize'
require 'tsort'
##
# This example implements the alt-text of http://xkcd.com/903/ which states:
#
# Wikipedia trivia: if you take any article, click on the first link in the
# article text not in parentheses or italics, and then repeat, you will
# eventually end up at "Philosophy".
class WikipediaLinksToPhilosophy
def initialize
@agent = Mechanize.new
@agent.user_agent_alias = 'Mac Safari' # Wikipedia blocks "mechanize"
@history = @agent.history
@wiki_url = URI 'http://en.wikipedia.org'
@search_url = @wiki_url + '/w/index.php'
@random_url = @wiki_url + '/wiki/Special:Random'
@title = nil
@seen = nil
end
##
# Retrieves the title of the current page
def extract_title
@page.title =~ /(.*) - Wikipedia/
@title = $1
end
##
# Retrieves the initial page. If +query+ is not given a random page is
# chosen
def fetch_first_page query
if query then
search query
else
random
end
end
##
# The search is finished if we've seen the page before or we've reached
# Philosophy
def finished?
@seen or @title == 'Philosophy'
end
##
# Follows the first non-parenthetical, non-italic link in the main body of
# the article.
def follow_first_link
puts "#{@title} (#{@page.uri})"
# > p > a rejects italics
links = @page.root.css('.mw-content-ltr p > a[href^="/wiki/"]')
# reject disambiguation and special pages, images and files
links = links.reject do |link_node|
link_node['href'] =~ %r%/wiki/\w+:|\(disambiguation\)%
end
links = links.reject do |link_node|
in_parenthetical? link_node
end
link = links.first
if link.nil?
puts "Could not parse #{@page.uri}"
exit 1
end
# convert a Nokogiri HTML element back to a mechanize link
link = Mechanize::Page::Link.new link, @agent, @page
return if @seen = @agent.visited?(link)
@page = link.click
extract_title
end
##
# Is +link_node+ in an open parenthetical section?
def in_parenthetical? link_node
siblings = link_node.parent.children
seen = false
before = siblings.reject do |node|
seen or (seen = node == link_node)
end
preceding_text = before.map { |node| node.text }.join
open = preceding_text.count '('
close = preceding_text.count ')'
open > close
end
##
# Prints the result of the search
def print_result
if @seen then
puts "[Loop detected]"
else
puts @title
end
puts
# subtract initial search or Special:Random
puts "After #{@agent.history.length - 1} pages"
end
##
# Retrieves a random page from wikipedia
def random
@page = @agent.get @random_url
extract_title
end
##
# Entry point
def run query = nil
fetch_first_page query
follow_first_link until finished?
print_result
end
##
# Searches for +query+ on wikipedia
def search query
@page = @agent.get @search_url, search: query
extract_title
end
end
WikipediaLinksToPhilosophy.new.run ARGV.shift if $0 == __FILE__
mechanize-2.10.1/examples/spider.rb 0000644 0000041 0000041 00000000725 14645745627 017247 0 ustar www-data www-data require 'rubygems'
require 'mechanize'
agent = Mechanize.new
agent.max_history = nil # unlimited history
stack = agent.get(ARGV[0]).links
while l = stack.pop
next unless l.uri
host = l.uri.host
next unless host.nil? or host == agent.history.first.uri.host
next if agent.visited? l.href
puts "crawling #{l.uri}"
begin
page = l.click
next unless Mechanize::Page === page
stack.push(*page.links)
rescue Mechanize::ResponseCodeError
end
end
mechanize-2.10.1/examples/latest_user_agents.rb 0000644 0000041 0000041 00000006271 14645745627 021656 0 ustar www-data www-data require 'mechanize'
require 'ostruct'
# LatestUAFetcher fetches latest user agents from `WhatIsMyBrowser.com`.
# It can use to update `Mechanize::AGENT_ALIASES`.
class LatestUAFetcher
attr_reader :user_agents
USER_AGENT_TYPES = OpenStruct.new(
linux_firefox: "Linux Firefox",
mac_firefox: "Mac Firefox",
mac_safari: "Mac Safari",
windows_chrome: "Windows Chrome",
windows_edge: "Windows Edge",
windows_firefox: "Windows Firefox",
android: "Android",
iphone: "iPhone",
ipad: "iPad",
)
BASE_URL = 'https://www.whatismybrowser.com/guides/the-latest-user-agent'
def initialize
@agent = Mechanize.new.tap { |a| a.user_agent_alias = 'Mac Firefox' }
@user_agents = {}
end
def run
return unless user_agents.empty?
sleep_time = 1
fetch_user_agents('chrome')
fetch_user_agents('firefox')
fetch_user_agents('safari')
fetch_user_agents('edge')
end
def ordered_user_agents
USER_AGENT_TYPES.to_h.values.each_with_object({}) do |type, ordered_user_agents|
ordered_user_agents[type] = user_agents[type]
end
end
private
def fetch_user_agents(browser_name, sleep_time = 1)
puts "fetch #{browser_name} UA..."
send(browser_name)
puts "sleeping... (#{sleep_time}s)"
sleep sleep_time
end
def edge
page = @agent.get("#{BASE_URL}/edge")
windows_dom = page.css("h2:contains('Latest Edge on Windows User Agents')")
@user_agents[USER_AGENT_TYPES.windows_edge] = windows_dom.css('+ .listing-of-useragents .code').first.text
end
def firefox
page = @agent.get("#{BASE_URL}/firefox")
desktop_dom = page.css("h2:contains('Latest Firefox on Desktop User Agents')")
table_dom = desktop_dom.css('+ .listing-of-useragents')
@user_agents[USER_AGENT_TYPES.linux_firefox] = table_dom.css('td:contains("Linux")').css("+ td .code:contains('Ubuntu; Linux x86_64')").text
@user_agents[USER_AGENT_TYPES.windows_firefox] = table_dom.css('td:contains("Windows")').css('+ td .code').text
@user_agents[USER_AGENT_TYPES.mac_firefox] = table_dom.css('td:contains("Macos")').css('+ td .code').text
end
def safari
page = @agent.get("#{BASE_URL}/safari")
macos_dom = page.css("h2:contains('Latest Safari on macOS User Agents')")
ios_dom = page.css("h2:contains('Latest Safari on iOS User Agents')")
@user_agents[USER_AGENT_TYPES.mac_safari] = macos_dom.css('+ .listing-of-useragents .code').first.text
@user_agents[USER_AGENT_TYPES.iphone] = ios_dom.css('+ .listing-of-useragents').css("tr:contains('Iphone') .code").text
@user_agents[USER_AGENT_TYPES.ipad] = ios_dom.css('+ .listing-of-useragents').css("tr:contains('Ipad') .code").text
end
def chrome
page = @agent.get("#{BASE_URL}/chrome")
windows_dom = page.css("h2:contains('Latest Chrome on Windows 10 User Agents')")
android_dom = page.css("h2:contains('Latest Chrome on Android User Agents')")
@user_agents[USER_AGENT_TYPES.windows_chrome] = windows_dom.css('+ .listing-of-useragents .code').first.text
@user_agents[USER_AGENT_TYPES.android] = android_dom.css('+ .listing-of-useragents .code').first.text
end
end
if $0 == __FILE__
agent = LatestUAFetcher.new
agent.run
pp agent.ordered_user_agents
end
mechanize-2.10.1/examples/flickr_upload.rb 0000644 0000041 0000041 00000001006 14645745627 020570 0 ustar www-data www-data require 'rubygems'
require 'mechanize'
agent = Mechanize.new
# Get the flickr sign in page
page = agent.get 'http://flickr.com/signin/flickr/'
# Fill out the login form
form = page.form_with :name => 'flickrloginform'
form.email = ARGV[0]
form.password = ARGV[1]
form.submit
# Go to the upload page
page = page.link_with(:text => 'Upload').click
# Fill out the form
form = page.forms.action('/photos_upload_process.gne').first
form.file_uploads.name('file1').first.file_name = ARGV[2]
form.submit
mechanize-2.10.1/CHANGELOG.md 0000644 0000041 0000041 00000131550 14645745627 015430 0 ustar www-data www-data # Mechanize CHANGELOG
## 2.10.1 / 2024-06-12
* Improve page encoding error recovery on pages with broken encoding when used with libxml2 >= 2.12.0. (#644) @flavorjones
## 2.10.0 / 2024-01-22
* Add `nkf` and `base64` as explicit dependencies, since they are being unbundled in Ruby 3.4. (#634) @flavorjones
## 2.9.2 / 2024-01-15
* Correct spelling errors in documentation. (#631) @p-linnane
* Updated User-Agent strings to represent modern browser versions. (#632) @takatea
## 2.9.1 / 2023-04-17
### Update
* Updated User-Agent strings to represent modern browser versions. (#612) Thank you, @takatea!
## 2.9.0 / 2023-04-07
### Requirements
* Mechanize now requires Ruby 2.6 or newer.
### Improvement
* Mechanize can now parse frozen strings. (#610)
## 2.8.5 / 2022-06-09
### Security
Fixes low-severity CVE-2022-31033, "Authorization header leak on port redirect." See [GHSA-64qm-hrgp-pgr9](https://github.com/sparklemotion/mechanize/security/advisories/GHSA-64qm-hrgp-pgr9) for more details.
## 2.8.4 / 2022-01-17
### Fix
* `Mechanize::CookieJar#load` calls `Psych.safe_load` when using Psych >= 3.1
## 2.8.3 / 2021-11-11
### Update
* Update the "Linux Firefox" user agent string to rev94 (#587) Thank you, @ncs1!
## 2.8.2 / 2021-08-06
### Dependencies
* Update dependency on Addressable from `~>2.7` to `~>2.8`. (#584) @yidingww
## 2.8.1 / 2021-05-09
### Fix
* Gracefully handle parsing errors that contain an invalid byte sequence. Previously, if libxml2 registered a parsing error that itself contained invalid encoding, an exception might be raised. (#553)
## 2.8.0 / 2021-04-01
### Requirements
* Mechanize now requires Ruby 2.5 or newer.
* Move from `ntlm-http` to `rubyntlm` gem. (#495, #574)
### New Features
* Page::Link#uri now handles non-ASCII `href`s. (#569) @terryyin
* FileConnection supports Windows drive letters (#483)
* Credential headers 'Authorization' and 'Cookie' are deleted on cross-origin redirects. (#538) @kyoshidajp
* ContentDispositionParser handles ISO8601 date headers, to be robust with websites that ignore RFC2183. (#554) @reitermarkus
### Bug fix
* POST headers 'Content-Length', 'Content-MD5', and 'Content-Type' are deleted in a case-insensitive manner on redirects. Previously these headers were treated as case-sensitive.
## 2.7.7 / 2021-02-01
* Security fixes for CVE-2021-21289
Mechanize `>= v2.0`, `< v2.7.7` allows for OS commands to be injected into several classes'
methods via implicit use of Ruby's `Kernel.open` method. Exploitation is possible only if
untrusted input is used as a local filename and passed to any of these calls:
- `Mechanize::CookieJar#load`: since v2.0 (see 208e3ed)
- `Mechanize::CookieJar#save_as`: since v2.0 (see 5b776a4)
- `Mechanize#download`: since v2.2 (see dc91667)
- `Mechanize::Download#save` and `#save!` since v2.1 (see 98b2f51, bd62ff0)
- `Mechanize::File#save` and `#save_as`: since v2.1 (see 2bf7519)
- `Mechanize::FileResponse#read_body`: since v2.0 (see 01039f5)
See https://github.com/sparklemotion/mechanize/security/advisories/GHSA-qrqm-fpv6-6r8g for more
information.
Also see #547, #548. Thank you, @kyoshidajp!
* New Features
* Support for Ruby 3.0 by adding `webrick` as a runtime dependency. (#557) @pvalena
* Bug fix
* Ignore input fields with blank names (#542, #536)
## 2.7.6
* New Features
* Mechanize#set_proxy accepts an HTTP URL/URI. (#513)
* Bug fix
* Fix element(s)_with(search: selector) methods not working for forms, form fields and frames. (#444)
* Improve the filename parser for the `Content-Disposition` header. (#496, #517)
* Accept `Content-Encoding: identity`. (#515)
* Mechanize::Page#title no longer picks a title in an embedded SVG/RDF element. (#503)
* Make Mechanize::Form#has_field? boolean. (#501)
## 2.7.5
* New Features
* All 4xx responses and RedirectLimitReachedError when fetching robots.txt are treated as full allow just like Googlebot does.
* Enable support for mime-types > 3.
* Bug fix
* Don't cause infinite loop when `GET /robots.txt` redirects. (#457)
* Fix basic authentication for a realm that contains uppercase characters. (#458, #459)
* Fix encoding error when uploading a file which name is non-ASCII. (#333)
## 2.7.4
* New Features
* Accept array-like and hash-like values as query/parameter value.
A new utility method Mechanize::Util.each_parameter is added, and Mechanize::Util.build_query_string is enhanced
for this feature.
* Allow passing a `Form::FileUpload` instance to `#post`. #350 by Sam
Rawlins.
* Capture link when scheme is unsupported. #362 by Jon Rowe.
* Pre-defined User-Agent stings are updated to those of more recent versions, and new aliases for IE 10/11 and Edge are added.
* Support for mime-types 1.x is restored while keeping compatible with mime-types 2.x.
* Mechanize::Page now responds to #xpath, #css, #at_xpath, #at_css, and #%.
* element(s)_with methods now accept :xpath and :css options for doing xpath/css
selector searching.
* Pass URI information to Nokogiri where applicable. #405 @lulalala
* Bug fix
* Don't raise an exception if a connection has set a {read,open}_timeout and
a `file://` request is made. (#397)
* Fix whitespace bug in WWW-Authenticate. #451, #450, by Rasmus Bergholdt
* Don't allow redirect from a non-file URL to a file URL for security reasons. (#455)
## 2.7.3
* New Features
* Allow net-http-persistent instance to be named. #324, John Weir.
* #save and #save! return filename #340
* Updated mime-types requirement to 2.x versions. #348 by Jeff Nyman.
* Bug fix
* Ensure Download#save! defaults back to original filename if
none is provided (#300)
## 2.7.2
* Bug fix
* API compatibility issues with Mechanize::CookieJar cookies has been
addressed. https://github.com/sparklemotion/http-cookie/issues/2 #326
## 2.7.1
* Bug fix
* Ensure images with no "src" attribute still return correct URLs. #317
* Fixes Mechanize::Parser#extract_filename where an empty string filename
in the response caused unhandled exception. #318
## 2.7.0
* New Features
* Mechanize::Agent#response_read will now raise a
Mechanize::ResponseReadError instead of an EOFError and avoid losing
requested content. #296.
* Depend on http-cookie, add backwards compatible deprecations. #257 Akinori MUSHA.
* Added `Download#save!` for overwriting existing files. #300 Sean Kim.
* Bug fix
* Ensure page URLs with whitespace in them are escaped #313 @pacop.
* Added a workaround for a bug of URI#+ that led to failure in resolving a relative path containing double slash like "/../http://.../". #304
## 2.6.0
* New Features
* Mechanize#start and Mechanize#shutdown (Thanks, Damian Janowski!)
* Added Mechanize::Agent#allowed_error_codes for setting an Array
of status codes which should not raise an error. #248 Laurence Rowe.
* Added `File.save!` for overwriting existing files #219.
* DirectorySaver::save_to now accepts an option to decode filename. #262
* element(s)_with methods now accept a :search option for doing xpath/css
selector searching. #287 Philippe Bourgau
* Added httponly option for Mechanize::Cookie #242 by Paolo Perego.
* Added Mechanize::XmlFile as a default pluggable parser for handling
XML responses. #289
* Minor enhancements
* Added Mechanize::Download#save_as as an alias to #save. #246
* Fix documentation for `Mechanize::Page` element matchers. #269
* Added Mechanize::Form::Field#raw_value for fetching a fields value
before it's sent through Mechanize::Util.html_unescape. #283
* Added iPad and Android user agents. #277 by sambit, #278 by seansay.
* Bug fix
* Mechanize#cert and Mechanize#key now return the values set by #cert= and #key=. #244, #245 (Thanks, Robert Gogolok!)
* Mechanize no longer submits disabled form fields. #276 by Bogdan Gusiev, #279 by Ricardo Valeriano.
* Mechanize::File#save now behaves like Mechanize::Download#save in
that it will create the parent directory before saving. #272, #280 by Ryan Kowalick
* Ensure `application/xml` is registered as an XML parser in
`PluggableParser`, not just `text/xml`. #266 James Gregory
* Mechanize now writes cookiestxt with a prefixed dot for wildcard domain
handling. #295 by Mike Morearty.
## 2.5.2
* New Features
* Mechanize::CookieJar#save_as takes a keyword option "session" to say
that session cookies should be saved. Based on #230 by Jim Jones.
* Minor enhancements
* Added Mechanize#follow_redirect= as an alias to redirect_ok=.
* Bug fix
* Fixed casing of the Mac Firefox user-agent alias to match Linux Firefox.
In mechanize 3 the old "Mac FireFox" user-agent alias will be removed.
Pull request #231 by Gavin Miller.
* Mechanize now authenticates using the raw challenge, not a reconstructed
one, to avoid dealing with quoting rules of RFC 2617. Fixes failures in #231 due to net-http-digest_auth 1.2.1
* Fixed Content-Disposition parameter parser to be case insensitive. #233
* Fixed redirection counting in following meta refresh. #240
## 2.5.1
* Bug fix
* Mechanize no longer copies POST requests during a redirect which was
introduced by #215. Pull request #229 by Godfrey Chan.
## 2.5
* Minor enhancements
* Added Mechanize#ignore_bad_chunking for working around servers that don't
terminate chunked transfer-encoding properly. Enabling this may cause
data loss. Issue #116
* Removed content-type check from Mechanize::Page allowing forced parsing
of incorrect or missing content-types. Issue #221 by GarthSnyder
* Bug fixes
* Fixed typos in EXAMPLES and GUIDES. Pull Request #213 by Erkan Yilmaz.
* Fixed handling of a quoted content-disposition size. Pull Request #220 by
Jason Rust
* Mechanize now ignores a missing gzip footer like browsers do. Issue #224
by afhbl
* Mechanize handles saving of files with the same name better now. Pull
Request #223 by Godfrey Chan, Issue #219 by Jon Hart
* Mechanize now sends headers across redirects. Issue #215 by Chris Gahan
* Mechanize now raises Mechanize::ResponseReadError when the server does not
terminate chunked transfer-encoding properly. Issue #116
* Mechanize no longer raises an exception when multiple identical
radiobuttons are checked. Issue #214 by Matthias Guenther
* Fixed documentation for pre_connect_hooks and post_connect_hooks. Issue #226 by Robert Poor
* Worked around ruby 1.8 run with -Ku and ISO-8859-1 encoded characters in
URIs. Issue #228 by Stanislav O.Pogrebnyak
## 2.4
* Security fix:
Mechanize#auth and Mechanize#basic_auth allowed disclosure of passwords to
malicious servers and have been deprecated.
In prior versions of mechanize only one set of HTTP authentication
credentials were allowed for all connections. If a mechanize instance
connected to more than one server then a malicious server detecting
mechanize could ask for HTTP Basic authentication. This would expose the
username and password intended only for one server.
Mechanize#auth and Mechanize#basic_auth now warn when used.
To fix the warning switch to Mechanize#add_auth which requires the URI
the credentials are intended for, the username and the password.
Optionally an HTTP authentication realm or NTLM domain may be provided.
* Minor enhancement
* Improved exception messages for 401 Unauthorized responses. Mechanize now
tells you if you were missing credentials, had an incorrect password, etc.
## 2.3 / 2012-02-20
* Minor enhancements
* Add support for the Max-Age attribute in the Set-Cookie header.
* Added Mechanize::Download#body for compatibility with Mechanize::File when
using Mechanize#get_file with Mechanize::Image or other Download-based
pluggable parser. Issue #202 by angas
* Mechanize#max_file_buffer may be set to nil to disable creation of
Tempfiles.
* Bug fixes
* Applied Mechanize#max_file_buffer to the Content-Encoding handlers as well
to prevent extra Tempfiles for small gzip or deflate response
* Increased the default Mechanize#max_file_buffer to 100,000 bytes. This
gives ~5MB of response bodies in memory with the default history setting
of 50 pages (depending on GC behavior).
* Ignore empty path/domain attributes.
* Cookies with an empty Expires attribute value were stored as session
cookies but cookies without the Expires attribute were not. Issue #78
## 2.2.1 / 2012-02-13
* Bug fixes
* Add missing file to the gem, ensure that missing files won't cause
failures again. Issue #201 by Alex
* Fix minor grammar issue in README. Issue #200 by Shane Becker.
## 2.2 / 2012-02-12
* API changes
* MetaRefresh#href is not normalized to an absolute URL, but set to the
original value and resolved later. It is even set to nil when the
Refresh URL is unspecified or empty.
* Minor enhancements
* Expose ssl_version from net-http-persistent. Patch by astera.
* SSL parameters and proxy may now be set at any time. Issue #194 by
dsisnero.
* Improved Mechanize::Page with #image_with and #images_with and
Mechanize::Page::Image various img element attribute accessors, #caption, #extname, #mime_type and #fetch. Pull request #173 by kitamomonga
* Added MIME type parsing for content-types in Mechanize::PluggableParser
for fine-grained parser choices. Parsers will be chosen based on exact
match, simplified type or media type in that order. See
Mechanize::PluggableParser#[]=.
* Added Mechanize#download which downloads a response body to an IO-like or
filename.
* Added Mechanize::DirectorySaver which saves responses in a single
directory. Issue #187 by yoshie902a.
* Added Mechanize::Page::Link#noreferrer?
* The documentation for Mechanize::Page#search and #at now show that both
XPath and CSS expressions are allowed. Issue #199 by Shane Becker.
* Bug fixes
* Fixed handling of a HEAD request with Accept-Encoding: gzip. Issue #198
by Oleg Dashevskii
* Use #resolve for resolving a Location header value. fixes #197
* A Refresh value can have whitespaces around the semicolon and equal sign.
* MetaRefresh#click no longer sends out Referer.
* A link with an empty href is now resolved correctly where previously the
query part was dropped.
## 2.1.1 / 2012-02-03
* Bug fixes
* Set missing idle_timeout default. Issue #196
* Meta refresh URIs are now escaped (excluding %). Issue #177
* Fix charset name extraction. Issue #180
* A Referer URI sent on request no longer includes user information
or fragment part.
* Tempfiles for storing response bodies are unlinked upon creation to avoid
possible lack of finalization. Issue #183
* The default maximum history size is now 50 pages to avoid filling up a
disk with tempfiles accidentally. Related to Issue #183
* Errors in bodies with deflate and gzip responses now result in a
Mechanize::Error instead of silently being ignored and causing future
errors. Issue #185
* Mechanize now raises an UnauthorizedError instead of crashing when a 403
response does not contain a www-authenticate header. Issue #181
* Mechanize gives a useful exception when attempting to click buttons across
pages. Issue #186
* Added note to Mechanize#cert_store describing how to add certificates in
case your system does not come with a default set. Issue #179
* Invalid content-disposition headers are now ignored. Issue #191
* Fix NTLM by recognizing the "Negotiation" challenge instead of endlessly
looping. Issue #192
* Allow specification of the NTLM domain through Mechanize#auth. Issue #193
* Documented how to convert a Mechanize::ResponseReadError into a File or
Page, along with a new method #force_parse. Issue #176
## 2.1 / 2011-12-20
* Deprecations
* Mechanize#get no longer accepts an options hash.
* Mechanize::Util::to_native_charset has been removed.
* Minor enhancements
* Mechanize now depends on net-http-persistent 2.3+. This new version
brings idle timeouts to help with the dreaded "too many connection resets"
issue when POSTing to a closed connection. Issue #123
* SSL connections will be verified against the system certificate store by
default.
* Added Mechanize#retry_change_requests to allow mechanize to retry POST and
other non-idempotent requests when you know it is safe to do so. Issue #123
* Mechanize can now stream files directly to disk without loading them into
memory first through Mechanize::Download, a pluggable parser for
downloading files.
All responses larger than Mechanize#max_file_buffer are downloaded to a
Tempfile. For backwards compatibility Mechanize::File subclasses still
load the response body into memory.
To force all unknown content types to download to disk instead of memory
set:
agent.pluggable_parser.default = Mechanize::Download
* Added Mechanize#content_encoding_hooks which allow handling of
non-standard content encodings like "agzip". Patch #125 by kitamomonga
* Added dom_class to elements and the element matcher like dom_id. Patch #156 by Dan Hansen.
* Added support for the HTML5 keygen form element. See
http://dev.w3.org/html5/spec/Overview.html#the-keygen-element Patch #157
by Victor Costan.
* Mechanize no longer follows meta refreshes that have no "url=" in the
content attribute to avoid infinite loops. To follow a meta refresh to
the same page set Mechanize#follow_meta_refresh_self to true. Issue #134
by Jo Hund.
* Updated 'Mac Safari' User-Agent alias to Safari 5.1.1. 'Mac Safari 4' can
be used for the old 'Mac Safari' alias.
* When given multiple HTTP authentication options mechanize now picks the
strongest method.
* Improvements to HTTP authorization:
* mechanize raises Mechanize::UnauthorizedError for 401 responses which is
a subclass of Mechanize::ResponseCodeError.
* Added support for NTLM authentication, but this has not been tested.
* Mechanize::Cookie.new accepts attributes in a hash.
* Mechanize::CookieJar#<<(cookie) (alias: add!) is added. Issue #139
* Different mechanize instances may now have different loggers. Issue #122
* Mechanize now accepts a proxy port as a service name or number string.
Issue #167
* Bug fixes
* Mechanize now handles cookies just as most modern browsers do,
roughly based on RFC 6265.
* domain=.example.com (which is invalid) is considered identical to
domain=example.com.
* A cookie with domain=example.com is sent to host.sub.example.com
as well as host.example.com and example.com.
* A cookie with domain=TLD (no dots) is accepted and sent if the
host name is TLD, and rejected otherwise. To retain compatibility
and convention, host/domain names starting with "local" are exempt
from this rule.
* A cookie with no domain attribute is only sent to the original
host.
* A cookie with an Effective TLD is rejected based on the public
suffix list. (cf. http://publicsuffix.org/)
* "Secure" cookies are not sent via non-https connection.
* Subdomain match is not performed against an IP address.
* It is recommended that you clear out existing cookie jars for
regeneration because previously saved cookies may not have been
parsed correctly.
* Mechanize takes more care to avoid saving files with certain unsafe names.
You should still take care not to use mechanize to save files directly
into your home directory ($HOME). Issue #163.
* Mechanize#cookie_jar= works again. Issue #126
* The original Referer value persists on redirection. Issue #150
* Do not send a referer on a Refresh header based redirection.
* Fixed encoding error in tests when LANG=C. Patch #142 by jinschoi.
* The order of items in a form submission now match the DOM order. Patch #129 by kitamomonga
* Fixed proxy example in EXAMPLE. Issue #146 by NielsKSchjoedt
## 2.0.1 / 2011-06-28
Mechanize now uses minitest to avoid 1.9 vs 1.8 assertion availability in
test/unit
* Bug Fixes
* Restored Mechanize#set_proxy. Issue #117, #118, #119
* Mechanize::CookieJar#load now lazy-loads YAML. Issue #118
* Mechanize#keep_alive_time no longer crashes but does nothing as
net-http-persistent does not support HTTP/1.0 keep-alive extensions.
## 2.0 / 2011-06-27
Mechanize is now under the MIT license
* API changes
* WWW::Mechanize has been removed. Use Mechanize.
* Pre connect hooks are now called with the agent and the request. See
Mechanize#pre_connect_hooks.
* Post connect hooks are now called with the agent and the response. See
Mechanize#post_connect_hooks.
* Mechanize::Chain is gone, as an internal API this should cause no problems.
* Mechanize#fetch_page no longer accepts an options Hash.
* Mechanize#put now accepts headers instead of an options Hash as the last
argument
* Mechanize#delete now accepts headers instead of an options Hash as the
last argument
* Mechanize#request_with_entity now accepts headers instead of an options
Hash as the last argument
* Mechanize no longer raises RuntimeError directly, Mechanize::Error or
ArgumentError are raised instead.
* The User-Agent header has changed. It no longer includes the WWW- prefix
and now includes the ruby version. The URL has been updated as well.
* Mechanize now requires ruby 1.8.7 or newer.
* Hpricot support has been removed as webrobots requires nokogiri.
* Mechanize#get no longer accepts the referer as the second argument.
* Mechanize#get no longer allows the HTTP method to be changed (:verb
option).
* Mechanize::Page::Meta is now Mechanize::Page::MetaRefresh to accurately
depict its responsibilities.
* Mechanize::Page#meta is now Mechanize::Page#meta_refresh as it only
contains meta elements with http-equiv of "refresh"
* Mechanize::Page#charset is now Mechanize::Page::charset. GH #112, patch
by Godfrey Chan.
* Deprecations
* Mechanize#get with an options hash is deprecated and will be removed after
October, 2011.
* Mechanize::Util::to_native_charset is deprecated as it is no longer used
by Mechanize.
* New Features
* Add header reference methods to Mechanize::File so that a response
object gets compatible with Net::HTTPResponse.
* Mechanize#click accepts a regexp or string to click a button/link in the
current page. It works as expected when not passed a string or regexp.
* Provide a way to only follow permanent redirects (301)
automatically: agent.redirect_ok = :permanent GH #73
* Mechanize now supports HTML5 meta charset. GH #113
* Documented various Mechanize accessors. GH #66
* Mechanize now uses net-http-digest_auth. GH #31
* Mechanize now implements session cookies. GH #78
* Mechanize now implements deflate decoding. GH #40
* Mechanize now allows a certificate and key to be passed directly. GH #71
* Mechanize::Form::MultiSelectList now implements #option_with and #options_with. GH #42
* Add Mechanize::Page::Link#rel and #rel?(kind) to read and test the rel
attribute.
* Add Mechanize::Page#canonical_uri to read a tag.
* Add support for Robots Exclusion Protocol (i.e. robots.txt) and
nofollow/noindex in meta tags and the rel attribute. Automatic
exclusion can be turned on by setting:
agent.robots = true
* Manual robots.txt test can be performed with
Mechanize#robots_allowed? and #robots_disallowed?.
* Mechanize::Form now supports the accept-charset attribute. GH #96
* Mechanize::ResponseReadError is raised if there is an exception while
reading the response body. This allows recovery from broken HTTP servers
(or connections). GH #90
* Mechanize#follow_meta_refresh set to :anywhere will follow meta refresh
found outside of a document's head. GH #99
* Add support for HTML5's rel="noreferrer" attribute which indicates
no "Referer" information should be sent when following the link.
* A frame will now load its content when #content is called. GH #111
* Added Mechanize#default_encoding to provide a default for pages with no
encoding specified. GH #104
* Added Mechanize#force_default_encoding which only uses
Mechanize#default_encoding for parsing HTML. GH #104
* Bug Fixes:
* Fixed a bug where Referer is not sent when accessing a relative
URI starting with "http".
* Fix handling of Meta Refresh with relative paths. GH #39
* Mechanize::CookieJar now supports RFC 2109 correctly. GH #85
* Fixed typo in EXAMPLES.rdoc. GH #74
* The base element is now handled correctly for images. GH #72
* Image buttons with no name attribute are now included in the form's button
list. GH#56
* Improved handling of non ASCII-7bit compatible characters in links (only
an issue on ruby 1.8). GH #36, GH #75
* Loading cookies.txt is faster. GH #38
* Mechanize no longer sends cookies for a.b.example to axb.example. GH #41
* Mechanize no longer sends the button name as a form field for image
buttons. GH #45
* Blank cookie values are now skipped. GH #80
* Mechanize now adds a '.' to cookie domains if no '.' was sent. This is
not allowed by RFC 2109 but does appear in RFC 2965. GH #86
* file URIs are now read in binary mode. GH #83
* Content-Encoding: x-gzip is now treated like gzip per RFC 2616.
* Mechanize now unescapes URIs for meta refresh. GH #68
* Mechanize now has more robust HTML charset detection. GH #43
* Mechanize::Form::Textarea is now created from a textarea element. GH #94
* A meta content-type now overrides the HTTP content type. GH #114
* Mechanize::Page::Link#uri now handles both escaped and unescaped hrefs.
GH #107
## 1.0.0
* New Features:
* An optional verb may be passed to Mechanize#get GH #26
* The WWW constant is deprecated. Switch to the top level constant Mechanize
* SelectList#option_with and options_with for finding options
* Bug Fixes:
* Rescue errors from bogus encodings
* 7bit content-encoding support. Thanks sporkmonger! GH #2
* Fixed a bug with iconv conversion. Thanks awesomeman! GH #9
* meta redirects outside the head are not followed. GH #13
* Form submissions work with nil page encodings. GH #25
* Fixing default values with serialized cookies. GH #3
* Checkboxes and fields are sorted by page appearance before submitting. #11
## 0.9.3
* Bug Fixes:
* Do not apply encoding if encoding equals 'none' Thanks Akinori MUSHA!
* Fixed Page#encoding= when changing the value from or to nil. Made
it return the assigned value while at it. (Akinori MUSHA)
* Custom request headers may be supplied WWW::Mechanize#request_headers
RF #24516
* HTML Parser may be set on a per instance level WWW::Mechanize#html_parser
RF #24693
* Fixed string encoding in ruby 1.9. RF #2433
* Rescuing Zlib::DataErrors (Thanks Kelley Reynolds)
* Fixing a problem with frozen SSL objects. RF #24950
* Do not send a referer on meta refresh. RF #24945
* Fixed a bug with double semi-colons in Content-Disposition headers
* Properly handling cookies that specify a path. RF #25259
## 0.9.2 / 2009/03/05
* New Features:
* Mechanize#submit and Form#submit take arbitrary headers(thanks penguincoder)
* Bug Fixes:
* Fixed a bug with bad cookie parsing
* Form::RadioButton#click unchecks other buttons (RF #24159)
* Fixed problems with Iconv (RF #24190, RF #24192, RF #24043)
* POST parameters should be CGI escaped
* Made Content-Type match case insensitive (Thanks Kelly Reynolds)
* Non-string form parameters work
## 0.9.1 2009/02/23
* New Features:
* Encoding may be specified for a page: Page#encoding=
* Bug Fixes:
* m17n fixes. ありがとう konn!
* Fixed a problem with base tags. ありがとう Keisuke
* HEAD requests do not record in the history
* Default encoding to ISO-8859-1 instead of ASCII
* Requests with URI instances should not be polluted RF #23472
* Nonce count fixed for digest auth requests. Thanks Adrian Slapa!
* Fixed a referer issue with requests using a uri. RF #23472
* WAP content types will now be parsed
* Rescued poorly formatted cookies. Thanks Kelley Reynolds!
## 0.9.0
* Deprecations
* WWW::Mechanize::List is gone!
* Mechanize uses Nokogiri as it's HTML parser but you may switch to
Hpricot by using WWW::Mechanize.html_parser = Hpricot
* Bug Fixes:
* Nil check on page when base tag is used #23021
## 0.8.5
* Deprecations
* WWW::Mechanize::List will be deprecated in 0.9.0, and warnings have
been added to help you upgrade.
* Bug Fixes:
* Stopped raising EOF exceptions on HEAD requests. ありがとう：HIRAKU Kuroda
* Fixed exceptions when a logger is set and file:// requests are made.
* Made Mechanize 1.9 compatible
* Not setting the port in the host header for SSL sites.
* Following refresh headers. Thanks Tim Connor!
* Cookie Jar handles cookie domains containing ports, like
'mydomain.com:443' (Thanks Michal Ochman!)
* Fixing strange uri escaping problems [#22604]
* Making content-type determintation more robust. (thanks Han Holl!)
* Dealing with links that are query string only. [#22402]
* Nokogiri may be dropped in as a replacement.
WWW::Mechanize.html_parser = Nokogiri::HTML
* Making sure the correct page is added to the history on meta refresh.
[#22708]
* Mechanize#get requests no longer send a referer unless they are relative
requests.
## 0.8.4
* Bug Fixes:
* Setting the port number on the host header.
* Fixing Authorization headers for picky servers
## 0.8.3
* Bug Fixes:
* Making sure logger is set during SSL connections.
## 0.8.2
* Bug Fixes:
* Doh! I was accidentally setting headers twice.
## 0.8.1
* Bug Fixes:
* Fixed problem with nil pointer when logger is set
## 0.8.0
* New Features:
* Lifecycle hooks. Mechanize#pre_connect_hooks, Mechanize#post_connect_hooks
* file:/// urls are now supported
* Added Mechanize::Page#link_with, frame_with for searching for links using
+criteria+.
* Implementing PUT, DELETE, and HEAD requests
* Bug Fixes:
* Fixed an infinite loop when content-length and body length don't match.
* Only setting headers once
* Adding IIS authentication support
## 0.7.8
* Bug Fixes:
* Fixed bug when receiving a 304 response (HTTPNotModified) on a page not
cached in history.
* #21428 Default to HTML parser for 'application/xhtml+xml' content-type.
* Fixed an issue where redirects were resending posted data
## 0.7.7
* New Features:
* Page#form_with takes a +criteria+ hash.
* Page#form is changed to Page#form_with
* Mechanize#get takes custom http headers. Thanks Mike Dalessio!
* Form#click_button submits a form defaulting to the current button.
* Form#set_fields now takes a hash. Thanks Tobi!
* Mechanize#redirection_limit= for setting the max number of redirects.
* Bug Fixes:
* Added more examples. Thanks Robert Jackson.
* #20480 Making sure the Host header is set.
* #20672 Making sure cookies with weird semicolons work.
* Fixed bug with percent signs in urls.
http://d.hatena.ne.jp/kitamomonga/20080410/ruby_mechanize_percent_url_bug
* #21132 Not checking for EOF errors on redirect
* Fixed a weird gzipping error.
* #21233 Smarter multipart boundary. Thanks Todd Willey!
* #20097 Supporting meta tag cookies.
## 0.7.6
* New Features:
* Added support for reading Mozilla cookie jars. Thanks Chris Riddoch!
* Moving text, password, hidden, int to default. Thanks Tim Harper!
* Mechanize#history_added callback for page loads. Thanks Tobi Reif!
* Mechanize#scheme_handlers callbacks for handling unsupported schemes on
links.
* Bug Fixes:
* Ignoring scheme case
http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=470642
* Not encoding tildes in uris. Thanks Bruno. [#19380]
* Resetting request bodys when retrying form posts. Thanks Bruno. [#19379]
* Throwing away keep alive connections on EPIPE and ECONNRESET.
* Duplicating request headers when retrying a 401. Thanks Hiroshi Ichikawa.
* Simulating an EOF error when a response length is bad. Thanks Tobias Gruetzmacher.
http://rubyforge.org/tracker/index.php?func=detail&aid=19178&group_id=1453&atid=5711
* Defaulting option tags to the inner text.
http://rubyforge.org/tracker/index.php?func=detail&aid=19976&group_id=1453&atid=5709
* Supporting blank strings for option values.
http://rubyforge.org/tracker/index.php?func=detail&aid=19975&group_id=1453&atid=5709
## 0.7.5
* Fixed a bug when fetching files and not pages. Thanks Mat Schaffer!
## 0.7.4
* doh!
## 0.7.3
* Pages are now yielded to a blocks given to WWW::Mechanize#get
* WWW::Mechanize#get now takes hash arguments for uri parameters.
* WWW::Mechanize#post takes an IO object as a parameter and posts correctly.
* Fixing a strange zlib inflate problem on windows
## 0.7.2
* Handling gzipped responses with no Content-Length header
## 0.7.1
* Added iPhone to the user agent aliases. [#17572]
* Fixed a bug with EOF errors in net/http. [#17570]
* Handling 0 length gzipped responses. [#17471]
## 0.7.0
* Removed Ruby 1.8.2 support
* Changed parser to lazily parse links
* Lazily parsing document
* Adding verify_callback for SSL requests. Thanks Mike Dalessio!
* Fixed a bug with Accept-Language header. Thanks Bill Siggelkow.
## 0.6.11
* Detecting single quotes in meta redirects.
* Adding pretty inspect for ruby versions > 1.8.4 (Thanks Joel Kociolek)
http://rubyforge.org/tracker/index.php?func=detail&aid=13150&group_id=1453&atid=5709
* Fixed bug with file name in multipart posts
http://rubyforge.org/tracker/?func=detail&aid=15594&group_id=1453&atid=5709
* Posting forms relative to the originating page. Thanks Mortee.
* Added a FAQ
http://rubyforge.org/tracker/?func=detail&aid=15772&group_id=1453&atid=5709
## 0.6.10
* Made digest authentication work with POSTs.
* Made sure page was HTML before following meta refreshes.
http://rubyforge.org/tracker/index.php?func=detail&aid=12260&group_id=1453&atid=5709
* Made sure that URLS with a host and no path would default to '/' for history
purposes.
http://rubyforge.org/tracker/index.php?func=detail&aid=12368&group_id=1453&atid=5709
* Avoiding memory leaks with transact. Thanks Tobias Gruetzmacher!
http://rubyforge.org/tracker/index.php?func=detail&aid=12057&group_id=1453&atid=5711
* Fixing a problem with # signs in the file name. Thanks Tobias Gruetzmacher!
http://rubyforge.org/tracker/index.php?func=detail&aid=12510&group_id=1453&atid=5711
* Made sure that blank form values are submitted.
http://rubyforge.org/tracker/index.php?func=detail&aid=12505&group_id=1453&atid=5709
* Mechanize now respects the base tag. Thanks Stephan Dale.
http://rubyforge.org/tracker/index.php?func=detail&aid=12468&group_id=1453&atid=5709
* Aliasing inspect to pretty_inspect. Thanks Eric Promislow.
http://rubyforge.org/pipermail/mechanize-users/2007-July/000157.html
## 0.6.9
* Updating UTF-8 support for urls
* Adding AREA tags to the links list.
http://rubyforge.org/pipermail/mechanize-users/2007-May/000140.html
* WWW::Mechanize#follow_meta_refresh will allow you to automatically follow
meta refresh tags. [#10032]
* Adding x-gzip to accepted content-encoding. Thanks Simon Strandgaard
http://rubyforge.org/tracker/index.php?func=detail&aid=11167&group_id=1453&atid=5711
* Added Digest Authentication support. Thanks to Ryan Davis and Eric Hodel,
you get a gold star!
## 0.6.8
* Keep alive can be shut off now with WWW::Mechanize#keep_alive
* Conditional requests can be shut off with WWW::Mechanize#conditional_requests
* Monkey patched Net::HTTP#keep_alive?
* [#9877] Moved last request time. Thanks Max Stepanov
* Added WWW::Mechanize::File#save
* Defaulting file name to URI or Content-Disposition
* Updating compatibility with hpricot
* Added more unit tests
## 0.6.7
* Fixed a bug with keep-alive requests
* [#9549] fixed problem with cookie paths
## 0.6.6
* Removing hpricot overrides
* Fixed a bug where alt text can be nil. Thanks Yannick!
* Unparsable expiration dates in cookies are now treated as session cookies
* Caching connections
* Requests now default to keep alive
* [#9434] Fixed bug where html entities weren't decoded
* [#9150] Updated mechanize history to deal with redirects
## 0.6.5
* Copying headers to a hash to prevent memory leaks
* Speeding up page parsing
* Aliased fields to elements
* Adding If-Modified-Since header
* Added delete_field! to form. Thanks to Sava Chankov
* Updated uri escaping to support high order characters. Thanks to Henrik Nyh.
* Better handling relative URIs. Thanks to Henrik Nyh
* Now handles pipes in URLs
http://rubyforge.org/tracker/?func=detail&aid=7140&group_id=1453&atid=5709
* Now escaping html entities in form fields.
http://rubyforge.org/tracker/?func=detail&aid=7563&group_id=1453&atid=5709
* Added MSIE 7.0 user agent string
## 0.6.4
* Adding the "redirect_ok" method to Mechanize to stop mechanize from
following redirects.
http://rubyforge.org/tracker/index.php?func=detail&aid=6571&group_id=1453&atid=5712
* Added protected method Mechanize#set_headers so that subclasses can set
custom headers.
http://rubyforge.org/tracker/?func=detail&aid=7208&group_id=1453&atid=5712
* Aliased Page#referer to Page#page
* Fixed a bug when clicking relative urls
http://rubyforge.org/pipermail/mechanize-users/2006-November/000035.html
* Fixing a bug when bad version or max age is passed to Cookie::parse
http://rubyforge.org/pipermail/mechanize-users/2006-November/000033.html
* Fixing a bug with response codes. [#6526]
* Fixed bug [#6548]. Input type of 'button' was not being added as a button.
* Fixed bug [#7139]. REXML parser calls hpricot parser by accident
## 0.6.3
* Added keys and values methods to Form
* Added has_value? to Form
* Added a has_field? method to Form
* The add_field! method on Form now creates a field for you on the form.
Thanks to Mat Schaffer for the patch.
http://rubyforge.org/pipermail/mechanize-users/2006-November/000025.html
* Fixed a bug when form actions have html ecoded entities in them.
http://rubyforge.org/pipermail/mechanize-users/2006-October/000019.html
* Fixed a bug when links or frame sources have html encoded entities in the
href or src.
* Fixed a bug where '#' symbols are encoded
http://rubyforge.org/forum/message.php?msg_id=14747
## 0.6.2
* Added a yield to Page#form so that dealing with forms can be more DSL like.
* Added the parsed page to the ResponseCodeError so that the parsed results
can be accessed even in the event of an error.
http://rubyforge.org/pipermail/mechanize-users/2006-September/000007.html
* Updated documentation (Thanks to Paul Smith)
## 0.6.1
* Added a method to Form called "submit". Now forms can be submitted by
calling a method on the form.
* Added a click method to links
* Added an REXML pluggable parser for backwards compatibility. To use it,
just do this:
agent.pluggable_parser.html = WWW::Mechanize::REXMLPage
* Fixed a bug with referrers by adding a page attribute to forms and links.
* Fixed a bug where domain names were case sensitive.
http://tenderlovemaking.com/2006/09/04/road-to-ruby-mechanize-060/#comment-53
* Fixed a bug with URI escaped links.
http://rubyforge.org/pipermail/mechanize-users/2006-September/000002.html
* Fixed a bug when options in select lists don't have a value. Thanks Dan Higham
[#5837] Code in lib/mechanize/form_elements.rb is incorrect.
* Fixed a bug with loading text in to links.
http://rubyforge.org/pipermail/mechanize-users/2006-September/000000.html
## 0.6.0
* Changed main parser to use hpricot
* Made WWW::Mechanize::Page class searchable like hpricot
* Updated WWW::Mechanize#click to support hpricot links like this:
@agent.click (page/"a").first
* Clicking a Frame is now possible:
@agent.click (page/"frame").first
* Removed deprecated attr_finder
* Removed REXML helper methods since the main parser is now hpricot
* Overhauled cookie parser to use WEBrick::Cookie
## 0.5.4
* Added WWW::Mechanize#trasact for saving history state between in a
transaction. See the EXAMPLES file. Thanks Johan Kiviniemi.
* Added support for gzip compressed pages
* Forms can now be accessed like a hash. For example, to set the value
of an input field named 'name' to "Aaron", you can do this:
form['name'] = "Aaron"
Or to get the value of a field named 'name', do this:
puts form['name']
* File uploads will now read the file specified in FileUpload#file_name
* FileUpload can use an IO object in FileUpload#file_data
* Fixed a bug with saving files on windows
* Fixed a bug with the filename being set in forms
## 0.5.3
* Mechanize#click will now act on the first element of an array. So if an
array of links is passed to WWW::Mechanize#click, the first link is clicked.
That means the syntax for clicking links is shortened and still supports
selecting a link. The following are equivalent:
agent.click page.links.first
agent.click page.links
* Fixed a bug with spaces in href's and get's
* Added a tick, untick, and click method to radio buttons so that
radiobuttons can be "clicked"
* Added a tick, untick, and click method to check boxes so that
checkboxes can be "clicked"
* Options on Select lists can now be "tick"ed, and "untick"ed.
* Fixed a potential bug conflicting with rails. Thanks Eric Kolve
* Updated log4r support for a speed increase. Thanks Yinon Bentor
* Added inspect methods and pretty printing
## 0.5.2
* Fixed a bug with input names that are nil
* Added a warning when using attr_finder because attr_finder will be deprecated
in 0.6.0 in favor of method calls. So this syntax:
@agent.links(:text => 'foo')
should be changed to this:
@agent.links.text('foo')
* Added support for selecting multiple options in select tags that support
multiple options. See WWW::Mechanize::MultiSelectList.
* New select list methods have been added, select_all, select_none.
* Options for select lists can now be "clicked" which toggles their selection,
they can be "selected" and "unselected". See WWW::Mechanize::Option
* Added a method to set multiple fields at the same time,
WWW::Mechanize::Form#set_fields. Which can be used like so:
form.set_fields( :foo => 'bar', :name => 'Aaron' )
## 0.5.1
* Fixed bug with file uploads
* Added performance tweaks to the cookie class
## 0.5.0
* Added pluggable parsers. (Thanks to Eric Kolve for the idea)
* Changed namespace so all classes are under WWW::Mechanize.
* Updating Forms so that fields can be used as accessors (Thanks Gregory Brown)
* Added WWW::Mechanize::File as default object used for unknown content types.
* Added 'save_as' method to Mechanize::File, so any page can be saved.
* Adding 'save_as' and 'load' to CookieJar so that cookies can be saved
between sessions.
* Added WWW::Mechanize::FileSaver pluggable parser to automatically save files.
* Added WWW::Mechanize::Page#title for page titles
* Added OpenSSL certificate support (Thanks Mike Dalessio)
* Removed support for body filters in favor of pluggable parsers.
* Fixed cookie bug adding a '/' when the url is missing one (Thanks Nick Dainty)
## 0.4.7
* Fixed bug with no action in forms. Thanks to Adam Wiggins
* Setting a default user-agent string
* Added house cleaning to the cookie jar so expired cookies don't stick around.
* Added new method WWW::Form#field to find the first field with a given name.
(thanks to Gregory Brown)
* Added WWW::Mechanize#get_file for fetching non text/html files
## 0.4.6
* Added support for proxies
* Added a uri field to WWW::Link
* Added a error class WWW::Mechanize::ContentTypeError
* Added image alt text to link text
* Added an visited? method to WWW::Mechanize
* Added Array#value= which will set the first value to the argument. That
allows syntax as such: form.fields.name('q').value = 'xyz'
Before it was like this: form.fields.name('q').first.value = 'xyz'
## 0.4.5
* Added support for multiple values of the same name
* Updated build_query_string to take an array of arrays (Thanks Michal Janeczek)
* Added WWW::Mechanize#body_filter= so that response bodies can be preprocessed
* Added WWW::Page#body_filter= so that response bodies can be preprocessed
* Added support for more date formats in the cookie parser
* Fixed a bug with empty select lists
* Fixing a problem with cookies not handling no spaces after semicolons
## 0.4.4
* Fixed error in method signature, basic_authentication is now basic_auth
* Fixed bug with encoding names in file uploads (Big thanks to Alex Young)
* Added options to the select list
## 0.4.3
* Added syntactic sugar for finding things
* Fixed bug with HttpOnly option in cookies
* Fixed a bug with cookie date parsing
* Defaulted dropdown lists to the first element
* Added unit tests
## 0.4.2
* Added support for iframes
* Made mechanize dependant on ruby-web rather than narf
* Added unit tests
* Fixed a bunch of warnings
## 0.4.1
* Added support for file uploading
* Added support for frames (Thanks Gabriel[mailto:leerbag@googlemail.com])
* Added more unit tests
* Fixed some bugs
## 0.4.0
* Added more unit tests
* Added a cookie jar with better cookie support, included expiration of cookies
and general cookie security.
* Updated mechanize to use built in net/http if ruby version is new enough.
* Added support for meta refresh tags
* Defaulted form actions to 'GET'
* Fixed various bugs
* Added more unit tests
* Added a response code exception
* Thanks to Brian Ellin (brianellin@gmail.com) for:
Added support for CA files, and support for 301 response codes