405: def fetch_page(params)
406: options = {
407: :request => nil,
408: :response => nil,
409: :connection => nil,
410: :referer => current_page(),
411: :uri => nil,
412: :verb => :get,
413: :agent => self,
414: :redirects => 0,
415: :params => [],
416: :headers => {},
417: }.merge(params)
418:
419: before_connect = Chain.new([
420: Chain::URIResolver.new(@scheme_handlers),
421: Chain::ParameterResolver.new,
422: Chain::RequestResolver.new,
423: Chain::ConnectionResolver.new(
424: @connection_cache,
425: @keep_alive,
426: @proxy_addr,
427: @proxy_port,
428: @proxy_user,
429: @proxy_pass
430: ),
431: Chain::SSLResolver.new(@ca_file, @verify_callback, @cert, @key, @pass),
432: Chain::AuthHeaders.new(@auth_hash, @user, @password, @digest),
433: Chain::HeaderResolver.new( @keep_alive,
434: @keep_alive_time,
435: @cookie_jar,
436: @user_agent),
437: Chain::CustomHeaders.new,
438: @pre_connect_hook,
439: ])
440: before_connect.handle(options)
441:
442: uri = options[:uri]
443: request = options[:request]
444: cur_page = options[:referer]
445: request_data = options[:params]
446: redirects = options[:redirects]
447: http_obj = options[:connection]
448:
449:
450: if( (page = visited_page(uri)) && page.response['Last-Modified'] )
451: request['If-Modified-Since'] = page.response['Last-Modified']
452: end if(@conditional_requests)
453:
454:
455: http_obj.open_timeout = @open_timeout if @open_timeout
456: http_obj.read_timeout = @read_timeout if @read_timeout
457: http_obj.start unless http_obj.started?
458:
459:
460: log.info("#{ request.class }: #{ request.path }") if log
461: request.each_header do |k, v|
462: log.debug("request-header: #{ k } => #{ v }")
463: end if log
464:
465:
466: attempts = 0
467: begin
468: response = http_obj.request(request, *request_data) { |r|
469: connection_chain = Chain.new([
470: Chain::ResponseReader.new(r),
471: Chain::BodyDecodingHandler.new,
472: ])
473: connection_chain.handle(options)
474: }
475: rescue EOFError, Errno::ECONNRESET, Errno::EPIPE => x
476: log.error("Rescuing EOF error") if log
477: http_obj.finish
478: raise x if attempts >= 2
479: request.body = nil
480: http_obj.start
481: attempts += 1
482: retry
483: end
484:
485: after_connect = Chain.new([
486: @post_connect_hook,
487: Chain::ResponseBodyParser.new(@pluggable_parser, @watch_for_set),
488: Chain::ResponseHeaderHandler.new(@cookie_jar, @connection_cache),
489: ])
490: after_connect.handle(options)
491:
492: res_klass = options[:res_klass]
493: response_body = options[:response_body]
494: page = options[:page]
495:
496: log.info("status: #{ page.code }") if log
497:
498: if follow_meta_refresh
499: redirect_uri = nil
500: if (page.respond_to?(:meta) && (redirect = page.meta.first))
501: redirect_uri = redirect.uri.to_s
502: elsif refresh = response['refresh']
503: parsed_refresh = refresh.match(/^\s*(\d+\.?\d*);\s*(url|URL)=(\S*)\s*$/)
504: raise StandardError, "Invalid refresh http header" unless parsed_refresh
505: delay = parsed_refresh[1]
506: location = parsed_refresh[3]
507: location = "http://#{uri.host}#{location}" unless location.include?("http")
508: if redirects + 1 > redirection_limit
509: raise RedirectLimitReachedError.new(page, redirects)
510: end
511: sleep delay.to_i
512: redirect_uri = location
513: end
514: if redirect_uri
515: @history.push(page, page.uri)
516: return fetch_page(
517: :uri => redirect_uri,
518: :referer => page,
519: :params => [],
520: :verb => :get,
521: :redirects => redirects + 1
522: )
523: end
524: end
525:
526: return page if res_klass <= Net::HTTPSuccess
527:
528: if res_klass == Net::HTTPNotModified
529: log.debug("Got cached page") if log
530: return visited_page(uri) || page
531: elsif res_klass <= Net::HTTPRedirection
532: return page unless follow_redirect?
533: log.info("follow redirect to: #{ response['Location'] }") if log
534: from_uri = page.uri
535: raise RedirectLimitReachedError.new(page, redirects) if redirects + 1 > redirection_limit
536: redirect_verb = options[:verb] == :head ? :head : :get
537: page = fetch_page( :uri => response['Location'].to_s,
538: :referer => page,
539: :params => [],
540: :verb => redirect_verb,
541: :redirects => redirects + 1
542: )
543: @history.push(page, from_uri)
544: return page
545: elsif res_klass <= Net::HTTPUnauthorized
546: raise ResponseCodeError.new(page) unless @user || @password
547: raise ResponseCodeError.new(page) if @auth_hash.has_key?(uri.host)
548: if response['www-authenticate'] =~ /Digest/i
549: @auth_hash[uri.host] = :digest
550: if response['server'] =~ /Microsoft-IIS/
551: @auth_hash[uri.host] = :iis_digest
552: end
553: @digest = response['www-authenticate']
554: else
555: @auth_hash[uri.host] = :basic
556: end
557: return fetch_page( :uri => uri,
558: :referer => cur_page,
559: :verb => request.method.downcase.to_sym,
560: :params => request_data,
561: :headers => options[:headers]
562: )
563: end
564:
565: raise ResponseCodeError.new(page), "Unhandled response", caller
566: end
