Class-CurlFetchWeb.php 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309
  1. <?php
  2. /**
  3. * Simple Machines Forum (SMF)
  4. *
  5. * @package SMF
  6. * @author Simple Machines http://www.simplemachines.org
  7. * @copyright 2013 Simple Machines and individual contributors
  8. * @license http://www.simplemachines.org/about/smf/license.php BSD
  9. *
  10. * @version 2.1 Alpha 1
  11. */
  12. if (!defined('SMF'))
  13. die('No direct access...');
  14. /**
  15. * Simple cURL class to fetch a web page
  16. * Properly redirects even with safe mode and basedir restrictions
  17. * Can provide simple post options to a page
  18. *
  19. * Load class
  20. * Initiate as
  21. * - $fetch_data = new cURL_fetch_web_data();
  22. * - optionaly pass an array of cURL options and redirect count
  23. * - cURL_fetch_web_data(cURL options array, Max redirects);
  24. * - $fetch_data = new cURL_fetch_web_data(array(CURLOPT_SSL_VERIFYPEER => 1), 5);
  25. *
  26. * Make the call
  27. * - $fetch_data('http://www.simplemachines.org'); // fetch a page
  28. * - $fetch_data('http://www.simplemachines.org', array('user' => 'name', 'password' => 'password')); // post to a page
  29. * - $fetch_data('http://www.simplemachines.org', parameter1&parameter2&parameter3); // post to a page
  30. *
  31. * Get the data
  32. * - $fetch_data->result('body'); // just the page content
  33. * - $fetch_data->result(); // an array of results, body, header, http result codes
  34. * - $fetch_data->result_raw(); // show all results of all calls (in the event of a redirect)
  35. * - $fetch_data->result_raw(0); // show all results of call x
  36. */
  37. class curl_fetch_web_data
  38. {
  39. /**
  40. * Set the default items for this class
  41. *
  42. * @var type
  43. */
  44. private $default_options = array(
  45. CURLOPT_RETURNTRANSFER => 1, // Get returned value as a string (don't output it)
  46. CURLOPT_HEADER => 1, // We need the headers to do our own redirect
  47. CURLOPT_FOLLOWLOCATION => 0, // Don't follow, we will do it ourselves so safe mode and open_basedir will dig it
  48. CURLOPT_USERAGENT => 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:11.0) Gecko Firefox/11.0', // set a normal looking useragent
  49. CURLOPT_CONNECTTIMEOUT => 15, // Don't wait forever on a connection
  50. CURLOPT_TIMEOUT => 90, // A page should load in this amount of time
  51. CURLOPT_MAXREDIRS => 5, // stop after this many redirects
  52. CURLOPT_ENCODING => 'gzip,deflate', // accept gzip and decode it
  53. CURLOPT_SSL_VERIFYPEER => 0, // stop cURL from verifying the peer's certificate
  54. CURLOPT_SSL_VERIFYHOST => 0, // stop cURL from verifying the peer's host
  55. CURLOPT_POST => 0, // no post data unless its passed
  56. );
  57. /**
  58. * Start the curl object
  59. * - allow for user override values
  60. *
  61. * @param type $options cURL options as an array
  62. * @param type $max_redirect use to overide the default of 3
  63. */
  64. public function __construct($options = array(), $max_redirect = 3)
  65. {
  66. // Initialize class variables
  67. $this->max_redirect = intval($max_redirect);
  68. $this->user_options = $options;
  69. }
  70. /**
  71. * Main calling function,
  72. * - will request the page data from a given $url
  73. * - optionally will post data to the page form if post data is supplied
  74. * - passed arrays will be converted to a post string joined with &'s
  75. * - calls set_options to set the curl opts array values based on the defaults and user input
  76. *
  77. * @param type $url the site we are going to fetch
  78. * @param type $post_data any post data as form name => value
  79. */
  80. public function get_url_data($url, $post_data = array())
  81. {
  82. // POSTing some data perhaps?
  83. if (!empty($post_data) && is_array($post_data))
  84. $this->post_data = $this->build_post_data($post_data);
  85. elseif (!empty($post_data))
  86. $this->post_data = trim($post_data);
  87. // set the options and get it
  88. $this->set_options();
  89. $this->curl_request(str_replace(' ', '%20', $url));
  90. return $this;
  91. }
  92. /**
  93. * Makes the actual cURL call
  94. * - stores responses (url, code, error, headers, body) in the response array
  95. * - detects 301, 302, 307 codes and will redirect to the given response header location
  96. *
  97. * @param type $url site to fetch
  98. * @param type $redirect flag to indicate if this was a redirect request or not
  99. * @return boolean
  100. */
  101. private function curl_request($url, $redirect = false)
  102. {
  103. // we do have a url I hope
  104. if ($url == '')
  105. return false;
  106. else
  107. $this->options[CURLOPT_URL] = $url;
  108. // if we have not already been redirected, set it up so we can if needed
  109. if (!$redirect)
  110. {
  111. $this->current_redirect = 1;
  112. $this->response = array();
  113. }
  114. // Initialize the curl object and make the call
  115. $cr = curl_init();
  116. curl_setopt_array($cr, $this->options);
  117. curl_exec($cr);
  118. // Get what was returned
  119. $curl_info = curl_getinfo($cr);
  120. $curl_content = curl_multi_getcontent($cr);
  121. $url = $curl_info['url']; // Last effective URL
  122. $http_code = $curl_info['http_code']; // Last HTTP code
  123. $body = (!curl_error($cr)) ? substr($curl_content, $curl_info['header_size']) : false;
  124. $error = (curl_error($cr)) ? curl_error($cr) : false;
  125. // close this request
  126. curl_close($cr);
  127. // store this 'loops' data, someone may want all of these :O
  128. $this->response[] = array(
  129. 'url' => $url,
  130. 'code' => $http_code,
  131. 'error' => $error,
  132. 'headers' => isset($this->headers) ? $this->headers : false,
  133. 'body' => $body,
  134. );
  135. // If this a redirect with a location header and we have not given up, then do it again
  136. if (preg_match('~30[127]~i', $http_code) === 1 && $this->headers['location'] != '' && $this->current_redirect <= $this->max_redirect)
  137. {
  138. $this->current_redirect++;
  139. $header_location = $this->get_redirect_url($url, $this->headers['location']);
  140. $this->redirect($header_location, $url);
  141. }
  142. }
  143. /**
  144. * Used if being redirected to ensure we have a fully qualified address
  145. *
  146. * @param type $last_url where we went to
  147. * @param type $new_url where we were redirected to
  148. * @return new url location
  149. */
  150. private function get_redirect_url($last_url = '', $new_url = '')
  151. {
  152. // Get the elements for these urls
  153. $last_url_parse = parse_url($last_url);
  154. $new_url_parse = parse_url($new_url);
  155. // redirect headers are often incomplete or relative so we need to make sure they are fully qualified
  156. $new_url_parse['scheme'] = isset($new_url_parse['scheme']) ? $new_url_parse['scheme'] : $last_url_parse['scheme'];
  157. $new_url_parse['host'] = isset($new_url_parse['host']) ? $new_url_parse['host'] : $last_url_parse['host'];
  158. $new_url_parse['path'] = isset($new_url_parse['path']) ? $new_url_parse['path'] : $last_url_parse['path'];
  159. $new_url_parse['query'] = isset($new_url_parse['query']) ? $new_url_parse['query'] : '';
  160. // Build the new URL that was in the http header
  161. return $new_url_parse['scheme'] . '://' . $new_url_parse['host'] . $new_url_parse['path'] . (!empty($new_url_parse['query']) ? '?' . $new_url_parse['query'] : '');
  162. }
  163. /**
  164. * Used to return the results to the calling program
  165. * - called as ->result() will return the full final array
  166. * - called as ->result('body') to just return the page source of the result
  167. *
  168. * @param type $area used to return an area such as body, header, error
  169. * @return type
  170. */
  171. public function result($area = '')
  172. {
  173. $max_result = count($this->response) - 1;
  174. // just return a specifed area or the entire result?
  175. if ($area == '')
  176. return $this->response[$max_result];
  177. else
  178. return isset($this->response[$max_result][$area]) ? $this->response[$max_result][$area] : $this->response[$max_result];
  179. }
  180. /**
  181. * Will return all results from all loops (redirects)
  182. * - Can be called as ->result_raw(x) where x is a specific loop results.
  183. * - Call as ->result_raw() for everything.
  184. *
  185. * @param type $response_number
  186. * @return type
  187. */
  188. public function result_raw($response_number = '')
  189. {
  190. if (!is_numeric($response_number))
  191. return $this->response;
  192. else
  193. {
  194. $response_number = min($response_number, count($this->response) - 1);
  195. return $this->response[$response_number];
  196. }
  197. }
  198. /**
  199. * Takes supplied POST data and url encodes it
  200. * - forms the date (for post) in to a string var=xyz&var2=abc&var3=123
  201. * - drops vars with @ since we don't support sending files (uploading)
  202. *
  203. * @param type $post_data
  204. * @return type
  205. */
  206. private function build_post_data($post_data)
  207. {
  208. if (is_array($post_data))
  209. {
  210. $postvars = array();
  211. // build the post data, drop ones with leading @'s since those can be used to send files, we don't support that.
  212. foreach ($post_data as $name => $value)
  213. $postvars[] = $name . '=' . urlencode($value[0] == '@' ? '' : $value);
  214. return implode('&', $postvars);
  215. }
  216. else
  217. return $post_data;
  218. }
  219. /**
  220. * Sets the final cURL options for the current call
  221. * - overwrites our default values with user supplied ones or appends new user ones to what we have
  222. * - sets the callback function now that $this is existing
  223. *
  224. */
  225. private function set_options()
  226. {
  227. // Callback to parse the returned headers, if any
  228. $this->default_options[CURLOPT_HEADERFUNCTION] = array($this, 'header_callback');
  229. // Any user options to account for
  230. if (is_array($this->user_options))
  231. {
  232. $keys = array_merge(array_keys($this->default_options), array_keys($this->user_options));
  233. $vals = array_merge($this->default_options, $this->user_options);
  234. $this->options = array_combine($keys, $vals);
  235. }
  236. else
  237. $this->options = $this->default_options;
  238. // POST data options, here we don't allow any overide
  239. if (isset($this->post_data))
  240. {
  241. $this->options[CURLOPT_POST] = 1;
  242. $this->options[CURLOPT_POSTFIELDS] = $this->post_data;
  243. }
  244. }
  245. /**
  246. * Called to initiate a redirect from a 301, 302 or 307 header
  247. * - resets the cURL options for the loop, sets the referrer flag
  248. *
  249. * @param type $target_url
  250. * @param type $referer_url
  251. */
  252. private function redirect($target_url, $referer_url)
  253. {
  254. // no no I last saw that over there ... really, 301, 302, 307
  255. $this->set_options();
  256. $this->options[CURLOPT_REFERER] = $referer_url;
  257. $this->curl_request($target_url, true);
  258. }
  259. /**
  260. * Callback function to parse returned headers
  261. * - lowercases everything to make it consistent
  262. *
  263. * @param type $cr
  264. * @param type $header
  265. * @return type
  266. */
  267. private function header_callback($cr, $header)
  268. {
  269. $_header = trim($header);
  270. $temp = explode(': ', $_header, 2);
  271. // set proper headers only
  272. if (isset($temp[0]) && isset($temp[1]))
  273. $this->headers[strtolower($temp[0])] = strtolower(trim($temp[1]));
  274. // return the length of what was passed unless you want a Failed writing header error ;)
  275. return strlen($header);
  276. }
  277. }
  278. ?>