uri_fetcher.rb revision cf44a17768d43920e482d2a7323f20a2c09ec6ae
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsyncmodule UriFetcher
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync require 'uri_fetcher/get_caller'
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync require 'uri_fetcher/post_caller'
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync NO_REDIRECT = 1
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync def fetch_uri_content(uri, limit: 10, write_file: nil, prev_resp: nil)
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync raise TooManyRedirectionsError.new(last_response: prev_resp) if limit == 0
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync Net::HTTP.get_response(URI(uri)) do |response|
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync response.read_body
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync if has_actual_content?(response)
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync produce_response_body(response, write_file)
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync elsif response['location'] && !response['location'].empty?
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync fetch_uri_content(response['location'],
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync limit: limit-1,
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync write_file: write_file,
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync prev_resp: response)
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync else
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync raise UnfollowableResponseError.new(last_response: response)
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync end
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync end
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync end
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync private
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync def produce_response_body(response, write_file=nil)
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync if write_file
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync File.open(write_file.to_s, 'w') do |file|
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync file.flock(File::LOCK_EX)
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync response.read_body do |chunk|
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync file.write chunk
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync end
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync end
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync write_file
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync else
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync response.body
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync end
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync end
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync def has_actual_content?(response)
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync response.is_a?(Net::HTTPSuccess) && response.content_type != 'text/html'
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync end
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsyncend
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync