uri_fetcher.rb revision cf44a17768d43920e482d2a7323f20a2c09ec6ae
3554937a935585f1c6aea3e25895036cf3f74759Tim Reddehasemodule UriFetcher
cf44a17768d43920e482d2a7323f20a2c09ec6aeEugen Kuksa require 'uri_fetcher/get_caller'
cf44a17768d43920e482d2a7323f20a2c09ec6aeEugen Kuksa require 'uri_fetcher/post_caller'
3554937a935585f1c6aea3e25895036cf3f74759Tim Reddehase
c53b52986e31858575023886cdd7327fccd8949cTim Reddehase NO_REDIRECT = 1
c53b52986e31858575023886cdd7327fccd8949cTim Reddehase
c53b52986e31858575023886cdd7327fccd8949cTim Reddehase def fetch_uri_content(uri, limit: 10, write_file: nil, prev_resp: nil)
c53b52986e31858575023886cdd7327fccd8949cTim Reddehase raise TooManyRedirectionsError.new(last_response: prev_resp) if limit == 0
d9f8c4120bb7409b56e1ca212c23ff1428516348Tim Reddehase Net::HTTP.get_response(URI(uri)) do |response|
14446ef4d8181bc484358d51cd08c4ad6353bb2eTim Reddehase response.read_body
3554937a935585f1c6aea3e25895036cf3f74759Tim Reddehase if has_actual_content?(response)
3554937a935585f1c6aea3e25895036cf3f74759Tim Reddehase produce_response_body(response, write_file)
14446ef4d8181bc484358d51cd08c4ad6353bb2eTim Reddehase elsif response['location'] && !response['location'].empty?
3554937a935585f1c6aea3e25895036cf3f74759Tim Reddehase fetch_uri_content(response['location'],
3554937a935585f1c6aea3e25895036cf3f74759Tim Reddehase limit: limit-1,
c53b52986e31858575023886cdd7327fccd8949cTim Reddehase write_file: write_file,
c53b52986e31858575023886cdd7327fccd8949cTim Reddehase prev_resp: response)
14446ef4d8181bc484358d51cd08c4ad6353bb2eTim Reddehase else
14446ef4d8181bc484358d51cd08c4ad6353bb2eTim Reddehase raise UnfollowableResponseError.new(last_response: response)
3554937a935585f1c6aea3e25895036cf3f74759Tim Reddehase end
3554937a935585f1c6aea3e25895036cf3f74759Tim Reddehase end
3554937a935585f1c6aea3e25895036cf3f74759Tim Reddehase end
3554937a935585f1c6aea3e25895036cf3f74759Tim Reddehase
3554937a935585f1c6aea3e25895036cf3f74759Tim Reddehase private
3554937a935585f1c6aea3e25895036cf3f74759Tim Reddehase def produce_response_body(response, write_file=nil)
3554937a935585f1c6aea3e25895036cf3f74759Tim Reddehase if write_file
3554937a935585f1c6aea3e25895036cf3f74759Tim Reddehase File.open(write_file.to_s, 'w') do |file|
3554937a935585f1c6aea3e25895036cf3f74759Tim Reddehase file.flock(File::LOCK_EX)
3554937a935585f1c6aea3e25895036cf3f74759Tim Reddehase response.read_body do |chunk|
3554937a935585f1c6aea3e25895036cf3f74759Tim Reddehase file.write chunk
3554937a935585f1c6aea3e25895036cf3f74759Tim Reddehase end
3554937a935585f1c6aea3e25895036cf3f74759Tim Reddehase end
3554937a935585f1c6aea3e25895036cf3f74759Tim Reddehase write_file
3554937a935585f1c6aea3e25895036cf3f74759Tim Reddehase else
3554937a935585f1c6aea3e25895036cf3f74759Tim Reddehase response.body
3554937a935585f1c6aea3e25895036cf3f74759Tim Reddehase end
3554937a935585f1c6aea3e25895036cf3f74759Tim Reddehase end
3554937a935585f1c6aea3e25895036cf3f74759Tim Reddehase
3554937a935585f1c6aea3e25895036cf3f74759Tim Reddehase def has_actual_content?(response)
d9f8c4120bb7409b56e1ca212c23ff1428516348Tim Reddehase response.is_a?(Net::HTTPSuccess) && response.content_type != 'text/html'
3554937a935585f1c6aea3e25895036cf3f74759Tim Reddehase end
3554937a935585f1c6aea3e25895036cf3f74759Tim Reddehaseend