ManageSearchEngines.php 29 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127
  1. <?php
  2. /**
  3. * This file contains all the screens that relate to search engines.
  4. *
  5. * Simple Machines Forum (SMF)
  6. *
  7. * @package SMF
  8. * @author Simple Machines http://www.simplemachines.org
  9. * @copyright 2013 Simple Machines and individual contributors
  10. * @license http://www.simplemachines.org/about/smf/license.php BSD
  11. *
  12. * @version 2.1 Alpha 1
  13. */
  14. if (!defined('SMF'))
  15. die('No direct access...');
  16. /**
  17. * Entry point for this section.
  18. */
  19. function SearchEngines()
  20. {
  21. global $context, $txt, $scripturl;
  22. isAllowedTo('admin_forum');
  23. loadLanguage('Search');
  24. loadTemplate('ManageSearch');
  25. $subActions = array(
  26. 'editspiders' => 'EditSpider',
  27. 'logs' => 'SpiderLogs',
  28. 'settings' => 'ManageSearchEngineSettings',
  29. 'spiders' => 'ViewSpiders',
  30. 'stats' => 'SpiderStats',
  31. );
  32. call_integration_hook('integrate_manage_search_engines', array(&$subActions));
  33. // Ensure we have a valid subaction.
  34. $context['sub_action'] = isset($_REQUEST['sa']) && isset($subActions[$_REQUEST['sa']]) ? $_REQUEST['sa'] : 'stats';
  35. $context['page_title'] = $txt['search_engines'];
  36. // Some more tab data.
  37. $context[$context['admin_menu_name']]['tab_data'] = array(
  38. 'title' => $txt['search_engines'],
  39. 'description' => $txt['search_engines_description'],
  40. );
  41. // Call the function!
  42. $subActions[$context['sub_action']]();
  43. }
  44. /**
  45. * This is really just the settings page.
  46. *
  47. * @param bool $return_config = false
  48. */
  49. function ManageSearchEngineSettings($return_config = false)
  50. {
  51. global $context, $txt, $modSettings, $scripturl, $sourcedir, $smcFunc;
  52. $config_vars = array(
  53. // How much detail?
  54. array('select', 'spider_mode', 'subtext' => $txt['spider_mode_note'], array($txt['spider_mode_off'], $txt['spider_mode_standard'], $txt['spider_mode_high'], $txt['spider_mode_vhigh']), 'onchange' => 'disableFields();'),
  55. 'spider_group' => array('select', 'spider_group', 'subtext' => $txt['spider_group_note'], array($txt['spider_group_none'], $txt['membergroups_members'])),
  56. array('select', 'show_spider_online', array($txt['show_spider_online_no'], $txt['show_spider_online_summary'], $txt['show_spider_online_detail'], $txt['show_spider_online_detail_admin'])),
  57. );
  58. // Set up a message.
  59. $context['settings_message'] = '<span class="smalltext">' . sprintf($txt['spider_settings_desc'], $scripturl . '?action=admin;area=logs;sa=pruning;' . $context['session_var'] . '=' . $context['session_id']) . '</span>';
  60. // Do some javascript.
  61. $javascript_function = '
  62. function disableFields()
  63. {
  64. disabledState = document.getElementById(\'spider_mode\').value == 0;';
  65. foreach ($config_vars as $variable)
  66. if ($variable[1] != 'spider_mode')
  67. $javascript_function .= '
  68. if (document.getElementById(\'' . $variable[1] . '\'))
  69. document.getElementById(\'' . $variable[1] . '\').disabled = disabledState;';
  70. $javascript_function .= '
  71. }
  72. disableFields();';
  73. call_integration_hook('integrate_modify_search_engine_settings', array(&$config_vars));
  74. if ($return_config)
  75. return $config_vars;
  76. // We need to load the groups for the spider group thingy.
  77. $request = $smcFunc['db_query']('', '
  78. SELECT id_group, group_name
  79. FROM {db_prefix}membergroups
  80. WHERE id_group != {int:admin_group}
  81. AND id_group != {int:moderator_group}',
  82. array(
  83. 'admin_group' => 1,
  84. 'moderator_group' => 3,
  85. )
  86. );
  87. while ($row = $smcFunc['db_fetch_assoc']($request))
  88. $config_vars['spider_group'][2][$row['id_group']] = $row['group_name'];
  89. $smcFunc['db_free_result']($request);
  90. // Make sure it's valid - note that regular members are given id_group = 1 which is reversed in Load.php - no admins here!
  91. if (isset($_POST['spider_group']) && !isset($config_vars['spider_group'][2][$_POST['spider_group']]))
  92. $_POST['spider_group'] = 0;
  93. // We'll want this for our easy save.
  94. require_once($sourcedir . '/ManageServer.php');
  95. // Setup the template.
  96. $context['page_title'] = $txt['settings'];
  97. $context['sub_template'] = 'show_settings';
  98. // Are we saving them - are we??
  99. if (isset($_GET['save']))
  100. {
  101. checkSession();
  102. call_integration_hook('integrate_save_search_engine_settings');
  103. saveDBSettings($config_vars);
  104. recacheSpiderNames();
  105. redirectexit('action=admin;area=sengines;sa=settings');
  106. }
  107. // Final settings...
  108. $context['post_url'] = $scripturl . '?action=admin;area=sengines;save;sa=settings';
  109. $context['settings_title'] = $txt['settings'];
  110. $context['settings_insert_below'] = '
  111. <script type="text/javascript"><!-- // --><![CDATA[
  112. ' . $javascript_function . '
  113. // ]]></script>';
  114. // Prepare the settings...
  115. prepareDBSettingContext($config_vars);
  116. }
  117. /**
  118. * View a list of all the spiders we know about.
  119. */
  120. function ViewSpiders()
  121. {
  122. global $context, $txt, $sourcedir, $scripturl, $smcFunc;
  123. if (!isset($_SESSION['spider_stat']) || $_SESSION['spider_stat'] < time() - 60)
  124. {
  125. consolidateSpiderStats();
  126. $_SESSION['spider_stat'] = time();
  127. }
  128. // Are we adding a new one?
  129. if (!empty($_POST['addSpider']))
  130. return EditSpider();
  131. // User pressed the 'remove selection button'.
  132. elseif (!empty($_POST['removeSpiders']) && !empty($_POST['remove']) && is_array($_POST['remove']))
  133. {
  134. checkSession();
  135. validateToken('admin-ser');
  136. // Make sure every entry is a proper integer.
  137. foreach ($_POST['remove'] as $index => $spider_id)
  138. $_POST['remove'][(int) $index] = (int) $spider_id;
  139. // Delete them all!
  140. $smcFunc['db_query']('', '
  141. DELETE FROM {db_prefix}spiders
  142. WHERE id_spider IN ({array_int:remove_list})',
  143. array(
  144. 'remove_list' => $_POST['remove'],
  145. )
  146. );
  147. $smcFunc['db_query']('', '
  148. DELETE FROM {db_prefix}log_spider_hits
  149. WHERE id_spider IN ({array_int:remove_list})',
  150. array(
  151. 'remove_list' => $_POST['remove'],
  152. )
  153. );
  154. $smcFunc['db_query']('', '
  155. DELETE FROM {db_prefix}log_spider_stats
  156. WHERE id_spider IN ({array_int:remove_list})',
  157. array(
  158. 'remove_list' => $_POST['remove'],
  159. )
  160. );
  161. cache_put_data('spider_search', null, 300);
  162. recacheSpiderNames();
  163. }
  164. // Get the last seens.
  165. $request = $smcFunc['db_query']('', '
  166. SELECT id_spider, MAX(last_seen) AS last_seen_time
  167. FROM {db_prefix}log_spider_stats
  168. GROUP BY id_spider',
  169. array(
  170. )
  171. );
  172. $context['spider_last_seen'] = array();
  173. while ($row = $smcFunc['db_fetch_assoc']($request))
  174. $context['spider_last_seen'][$row['id_spider']] = $row['last_seen_time'];
  175. $smcFunc['db_free_result']($request);
  176. createToken('admin-ser');
  177. $listOptions = array(
  178. 'id' => 'spider_list',
  179. 'title' => $txt['spiders'],
  180. 'items_per_page' => 20,
  181. 'base_href' => $scripturl . '?action=admin;area=sengines;sa=spiders',
  182. 'default_sort_col' => 'name',
  183. 'get_items' => array(
  184. 'function' => 'list_getSpiders',
  185. ),
  186. 'get_count' => array(
  187. 'function' => 'list_getNumSpiders',
  188. ),
  189. 'no_items_label' => $txt['spiders_no_entries'],
  190. 'columns' => array(
  191. 'name' => array(
  192. 'header' => array(
  193. 'value' => $txt['spider_name'],
  194. ),
  195. 'data' => array(
  196. 'function' => create_function('$rowData', '
  197. global $scripturl, $smcFunc;
  198. return sprintf(\'<a href="%1$s?action=admin;area=sengines;sa=editspiders;sid=%2$d">%3$s</a>\', $scripturl, $rowData[\'id_spider\'], $smcFunc[\'htmlspecialchars\']($rowData[\'spider_name\']));
  199. '),
  200. ),
  201. 'sort' => array(
  202. 'default' => 'spider_name',
  203. 'reverse' => 'spider_name DESC',
  204. ),
  205. ),
  206. 'last_seen' => array(
  207. 'header' => array(
  208. 'value' => $txt['spider_last_seen'],
  209. ),
  210. 'data' => array(
  211. 'function' => create_function('$rowData', '
  212. global $context, $txt;
  213. return isset($context[\'spider_last_seen\'][$rowData[\'id_spider\']]) ? timeformat($context[\'spider_last_seen\'][$rowData[\'id_spider\']]) : $txt[\'spider_last_never\'];
  214. '),
  215. ),
  216. ),
  217. 'user_agent' => array(
  218. 'header' => array(
  219. 'value' => $txt['spider_agent'],
  220. ),
  221. 'data' => array(
  222. 'db_htmlsafe' => 'user_agent',
  223. ),
  224. 'sort' => array(
  225. 'default' => 'user_agent',
  226. 'reverse' => 'user_agent DESC',
  227. ),
  228. ),
  229. 'ip_info' => array(
  230. 'header' => array(
  231. 'value' => $txt['spider_ip_info'],
  232. ),
  233. 'data' => array(
  234. 'db_htmlsafe' => 'ip_info',
  235. 'class' => 'smalltext',
  236. ),
  237. 'sort' => array(
  238. 'default' => 'ip_info',
  239. 'reverse' => 'ip_info DESC',
  240. ),
  241. ),
  242. 'check' => array(
  243. 'header' => array(
  244. 'value' => '<input type="checkbox" onclick="invertAll(this, this.form);" class="input_check" />',
  245. 'class' => 'centercol',
  246. ),
  247. 'data' => array(
  248. 'sprintf' => array(
  249. 'format' => '<input type="checkbox" name="remove[]" value="%1$d" class="input_check" />',
  250. 'params' => array(
  251. 'id_spider' => false,
  252. ),
  253. ),
  254. 'class' => 'centercol',
  255. ),
  256. ),
  257. ),
  258. 'form' => array(
  259. 'href' => $scripturl . '?action=admin;area=sengines;sa=spiders',
  260. 'token' => 'admin-ser',
  261. ),
  262. 'additional_rows' => array(
  263. array(
  264. 'position' => 'bottom_of_list',
  265. 'value' => '
  266. <input type="submit" name="removeSpiders" value="' . $txt['spiders_remove_selected'] . '" onclick="return confirm(\'' . $txt['spider_remove_selected_confirm'] . '\');" class="button_submit" />
  267. <input type="submit" name="addSpider" value="' . $txt['spiders_add'] . '" class="button_submit" />
  268. ',
  269. ),
  270. ),
  271. );
  272. require_once($sourcedir . '/Subs-List.php');
  273. createList($listOptions);
  274. $context['sub_template'] = 'show_list';
  275. $context['default_list'] = 'spider_list';
  276. }
  277. /**
  278. * Callback function for createList()
  279. * @param int $start
  280. * @param int $items_per_page
  281. * @param string sort
  282. */
  283. function list_getSpiders($start, $items_per_page, $sort)
  284. {
  285. global $smcFunc;
  286. $request = $smcFunc['db_query']('', '
  287. SELECT id_spider, spider_name, user_agent, ip_info
  288. FROM {db_prefix}spiders
  289. ORDER BY ' . $sort . '
  290. LIMIT ' . $start . ', ' . $items_per_page,
  291. array(
  292. )
  293. );
  294. $spiders = array();
  295. while ($row = $smcFunc['db_fetch_assoc']($request))
  296. $spiders[$row['id_spider']] = $row;
  297. $smcFunc['db_free_result']($request);
  298. return $spiders;
  299. }
  300. /**
  301. * Callback function for createList()
  302. * @return int
  303. */
  304. function list_getNumSpiders()
  305. {
  306. global $smcFunc;
  307. $request = $smcFunc['db_query']('', '
  308. SELECT COUNT(*) AS num_spiders
  309. FROM {db_prefix}spiders',
  310. array(
  311. )
  312. );
  313. list ($numSpiders) = $smcFunc['db_fetch_row']($request);
  314. $smcFunc['db_free_result']($request);
  315. return $numSpiders;
  316. }
  317. /**
  318. * Here we can add, and edit, spider info!
  319. */
  320. function EditSpider()
  321. {
  322. global $context, $smcFunc, $txt;
  323. // Some standard stuff.
  324. $context['id_spider'] = !empty($_GET['sid']) ? (int) $_GET['sid'] : 0;
  325. $context['page_title'] = $context['id_spider'] ? $txt['spiders_edit'] : $txt['spiders_add'];
  326. $context['sub_template'] = 'spider_edit';
  327. // Are we saving?
  328. if (!empty($_POST['save']))
  329. {
  330. checkSession();
  331. validateToken('admin-ses');
  332. $ips = array();
  333. // Check the IP range is valid.
  334. $ip_sets = explode(',', $_POST['spider_ip']);
  335. foreach ($ip_sets as $set)
  336. {
  337. $test = ip2range(trim($set));
  338. if (!empty($test))
  339. $ips[] = $set;
  340. }
  341. $ips = implode(',', $ips);
  342. // Goes in as it is...
  343. if ($context['id_spider'])
  344. $smcFunc['db_query']('', '
  345. UPDATE {db_prefix}spiders
  346. SET spider_name = {string:spider_name}, user_agent = {string:spider_agent},
  347. ip_info = {string:ip_info}
  348. WHERE id_spider = {int:current_spider}',
  349. array(
  350. 'current_spider' => $context['id_spider'],
  351. 'spider_name' => $_POST['spider_name'],
  352. 'spider_agent' => $_POST['spider_agent'],
  353. 'ip_info' => $ips,
  354. )
  355. );
  356. else
  357. $smcFunc['db_insert']('insert',
  358. '{db_prefix}spiders',
  359. array(
  360. 'spider_name' => 'string', 'user_agent' => 'string', 'ip_info' => 'string',
  361. ),
  362. array(
  363. $_POST['spider_name'], $_POST['spider_agent'], $ips,
  364. ),
  365. array('id_spider')
  366. );
  367. // Order by user agent length.
  368. sortSpiderTable();
  369. cache_put_data('spider_search', null, 300);
  370. recacheSpiderNames();
  371. redirectexit('action=admin;area=sengines;sa=spiders');
  372. }
  373. // The default is new.
  374. $context['spider'] = array(
  375. 'id' => 0,
  376. 'name' => '',
  377. 'agent' => '',
  378. 'ip_info' => '',
  379. );
  380. // An edit?
  381. if ($context['id_spider'])
  382. {
  383. $request = $smcFunc['db_query']('', '
  384. SELECT id_spider, spider_name, user_agent, ip_info
  385. FROM {db_prefix}spiders
  386. WHERE id_spider = {int:current_spider}',
  387. array(
  388. 'current_spider' => $context['id_spider'],
  389. )
  390. );
  391. if ($row = $smcFunc['db_fetch_assoc']($request))
  392. $context['spider'] = array(
  393. 'id' => $row['id_spider'],
  394. 'name' => $row['spider_name'],
  395. 'agent' => $row['user_agent'],
  396. 'ip_info' => $row['ip_info'],
  397. );
  398. $smcFunc['db_free_result']($request);
  399. }
  400. createToken('admin-ses');
  401. }
  402. /**
  403. * Do we think the current user is a spider?
  404. *
  405. * @todo Should this not be... you know... in a different file?
  406. * @return int
  407. */
  408. function SpiderCheck()
  409. {
  410. global $modSettings, $smcFunc;
  411. if (isset($_SESSION['id_robot']))
  412. unset($_SESSION['id_robot']);
  413. $_SESSION['robot_check'] = time();
  414. // We cache the spider data for five minutes if we can.
  415. if (($spider_data = cache_get_data('spider_search', 300)) === null)
  416. {
  417. $request = $smcFunc['db_query']('spider_check', '
  418. SELECT id_spider, user_agent, ip_info
  419. FROM {db_prefix}spiders',
  420. array(
  421. )
  422. );
  423. $spider_data = array();
  424. while ($row = $smcFunc['db_fetch_assoc']($request))
  425. $spider_data[] = $row;
  426. $smcFunc['db_free_result']($request);
  427. cache_put_data('spider_search', $spider_data, 300);
  428. }
  429. if (empty($spider_data))
  430. return false;
  431. // Only do these bits once.
  432. $ci_user_agent = strtolower($_SERVER['HTTP_USER_AGENT']);
  433. // Always attempt IPv6 first.
  434. if (strpos($_SERVER['REMOTE_ADDR'], ':') !== false)
  435. $ip_parts = convertIPv6toInts($_SERVER['REMOTE_ADDR']);
  436. else
  437. preg_match('/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/', $_SERVER['REMOTE_ADDR'], $ip_parts);
  438. foreach ($spider_data as $spider)
  439. {
  440. // User agent is easy.
  441. if (!empty($spider['user_agent']) && strpos($ci_user_agent, strtolower($spider['user_agent'])) !== false)
  442. $_SESSION['id_robot'] = $spider['id_spider'];
  443. // IP stuff is harder.
  444. elseif (!empty($ip_parts))
  445. {
  446. $ips = explode(',', $spider['ip_info']);
  447. foreach ($ips as $ip)
  448. {
  449. $ip = ip2range($ip);
  450. if (!empty($ip))
  451. {
  452. foreach ($ip as $key => $value)
  453. {
  454. if ($value['low'] > $ip_parts[$key + 1] || $value['high'] < $ip_parts[$key + 1])
  455. break;
  456. elseif (($key == 7 && strpos($_SERVER['REMOTE_ADDR'], ':') !== false) || ($key == 3 && strpos($_SERVER['REMOTE_ADDR'], ':') === false))
  457. $_SESSION['id_robot'] = $spider['id_spider'];
  458. }
  459. }
  460. }
  461. }
  462. if (isset($_SESSION['id_robot']))
  463. break;
  464. }
  465. // If this is low server tracking then log the spider here as oppossed to the main logging function.
  466. if (!empty($modSettings['spider_mode']) && $modSettings['spider_mode'] == 1 && !empty($_SESSION['id_robot']))
  467. logSpider();
  468. return !empty($_SESSION['id_robot']) ? $_SESSION['id_robot'] : 0;
  469. }
  470. /**
  471. * Log the spider presence online.
  472. *
  473. * @todo Different file?
  474. */
  475. function logSpider()
  476. {
  477. global $smcFunc, $modSettings, $context;
  478. if (empty($modSettings['spider_mode']) || empty($_SESSION['id_robot']))
  479. return;
  480. // Attempt to update today's entry.
  481. if ($modSettings['spider_mode'] == 1)
  482. {
  483. $date = strftime('%Y-%m-%d', forum_time(false));
  484. $smcFunc['db_query']('', '
  485. UPDATE {db_prefix}log_spider_stats
  486. SET last_seen = {int:current_time}, page_hits = page_hits + 1
  487. WHERE id_spider = {int:current_spider}
  488. AND stat_date = {date:current_date}',
  489. array(
  490. 'current_date' => $date,
  491. 'current_time' => time(),
  492. 'current_spider' => $_SESSION['id_robot'],
  493. )
  494. );
  495. // Nothing updated?
  496. if ($smcFunc['db_affected_rows']() == 0)
  497. {
  498. $smcFunc['db_insert']('ignore',
  499. '{db_prefix}log_spider_stats',
  500. array(
  501. 'id_spider' => 'int', 'last_seen' => 'int', 'stat_date' => 'date', 'page_hits' => 'int',
  502. ),
  503. array(
  504. $_SESSION['id_robot'], time(), $date, 1,
  505. ),
  506. array('id_spider', 'stat_date')
  507. );
  508. }
  509. }
  510. // If we're tracking better stats than track, better stats - we sort out the today thing later.
  511. else
  512. {
  513. if ($modSettings['spider_mode'] > 2)
  514. {
  515. $url = $_GET + array('USER_AGENT' => $_SERVER['HTTP_USER_AGENT']);
  516. unset($url['sesc'], $url[$context['session_var']]);
  517. $url = serialize($url);
  518. }
  519. else
  520. $url = '';
  521. $smcFunc['db_insert']('insert',
  522. '{db_prefix}log_spider_hits',
  523. array('id_spider' => 'int', 'log_time' => 'int', 'url' => 'string'),
  524. array($_SESSION['id_robot'], time(), $url),
  525. array()
  526. );
  527. }
  528. }
  529. /**
  530. * This function takes any unprocessed hits and turns them into stats.
  531. */
  532. function consolidateSpiderStats()
  533. {
  534. global $smcFunc;
  535. $request = $smcFunc['db_query']('consolidate_spider_stats', '
  536. SELECT id_spider, MAX(log_time) AS last_seen, COUNT(*) AS num_hits
  537. FROM {db_prefix}log_spider_hits
  538. WHERE processed = {int:not_processed}
  539. GROUP BY id_spider, MONTH(log_time), DAYOFMONTH(log_time)',
  540. array(
  541. 'not_processed' => 0,
  542. )
  543. );
  544. $spider_hits = array();
  545. while ($row = $smcFunc['db_fetch_assoc']($request))
  546. $spider_hits[] = $row;
  547. $smcFunc['db_free_result']($request);
  548. if (empty($spider_hits))
  549. return;
  550. // Attempt to update the master data.
  551. $stat_inserts = array();
  552. foreach ($spider_hits as $stat)
  553. {
  554. // We assume the max date is within the right day.
  555. $date = strftime('%Y-%m-%d', $stat['last_seen']);
  556. $smcFunc['db_query']('', '
  557. UPDATE {db_prefix}log_spider_stats
  558. SET page_hits = page_hits + ' . $stat['num_hits'] . ',
  559. last_seen = CASE WHEN last_seen > {int:last_seen} THEN last_seen ELSE {int:last_seen} END
  560. WHERE id_spider = {int:current_spider}
  561. AND stat_date = {date:last_seen_date}',
  562. array(
  563. 'last_seen_date' => $date,
  564. 'last_seen' => $stat['last_seen'],
  565. 'current_spider' => $stat['id_spider'],
  566. )
  567. );
  568. if ($smcFunc['db_affected_rows']() == 0)
  569. $stat_inserts[] = array($date, $stat['id_spider'], $stat['num_hits'], $stat['last_seen']);
  570. }
  571. // New stats?
  572. if (!empty($stat_inserts))
  573. $smcFunc['db_insert']('ignore',
  574. '{db_prefix}log_spider_stats',
  575. array('stat_date' => 'date', 'id_spider' => 'int', 'page_hits' => 'int', 'last_seen' => 'int'),
  576. $stat_inserts,
  577. array('stat_date', 'id_spider')
  578. );
  579. // All processed.
  580. $smcFunc['db_query']('', '
  581. UPDATE {db_prefix}log_spider_hits
  582. SET processed = {int:is_processed}
  583. WHERE processed = {int:not_processed}',
  584. array(
  585. 'is_processed' => 1,
  586. 'not_processed' => 0,
  587. )
  588. );
  589. }
  590. /**
  591. * See what spiders have been up to.
  592. */
  593. function SpiderLogs()
  594. {
  595. global $context, $txt, $sourcedir, $scripturl, $smcFunc, $modSettings;
  596. // Load the template and language just incase.
  597. loadLanguage('Search');
  598. loadTemplate('ManageSearch');
  599. // Did they want to delete some entries?
  600. if (!empty($_POST['delete_entries']) && isset($_POST['older']))
  601. {
  602. checkSession();
  603. validateToken('admin-sl');
  604. $deleteTime = time() - (((int) $_POST['older']) * 24 * 60 * 60);
  605. // Delete the entires.
  606. $smcFunc['db_query']('', '
  607. DELETE FROM {db_prefix}log_spider_hits
  608. WHERE log_time < {int:delete_period}',
  609. array(
  610. 'delete_period' => $deleteTime,
  611. )
  612. );
  613. }
  614. $listOptions = array(
  615. 'id' => 'spider_logs',
  616. 'items_per_page' => 20,
  617. 'title' => $txt['spider_logs'],
  618. 'no_items_label' => $txt['spider_logs_empty'],
  619. 'base_href' => $context['admin_area'] == 'sengines' ? $scripturl . '?action=admin;area=sengines;sa=logs' : $scripturl . '?action=admin;area=logs;sa=spiderlog',
  620. 'default_sort_col' => 'log_time',
  621. 'get_items' => array(
  622. 'function' => 'list_getSpiderLogs',
  623. ),
  624. 'get_count' => array(
  625. 'function' => 'list_getNumSpiderLogs',
  626. ),
  627. 'columns' => array(
  628. 'name' => array(
  629. 'header' => array(
  630. 'value' => $txt['spider'],
  631. ),
  632. 'data' => array(
  633. 'db' => 'spider_name',
  634. ),
  635. 'sort' => array(
  636. 'default' => 's.spider_name',
  637. 'reverse' => 's.spider_name DESC',
  638. ),
  639. ),
  640. 'log_time' => array(
  641. 'header' => array(
  642. 'value' => $txt['spider_time'],
  643. ),
  644. 'data' => array(
  645. 'function' => create_function('$rowData', '
  646. return timeformat($rowData[\'log_time\']);
  647. '),
  648. ),
  649. 'sort' => array(
  650. 'default' => 'sl.id_hit DESC',
  651. 'reverse' => 'sl.id_hit',
  652. ),
  653. ),
  654. 'viewing' => array(
  655. 'header' => array(
  656. 'value' => $txt['spider_viewing'],
  657. ),
  658. 'data' => array(
  659. 'db' => 'url',
  660. ),
  661. ),
  662. ),
  663. 'form' => array(
  664. 'token' => 'admin-sl',
  665. 'href' => $scripturl . '?action=admin;area=sengines;sa=logs',
  666. ),
  667. 'additional_rows' => array(
  668. array(
  669. 'position' => 'after_title',
  670. 'value' => $txt['spider_logs_info'],
  671. 'class' => 'windowbg2',
  672. ),
  673. array(
  674. 'position' => 'below_table_data',
  675. 'value' => '<input type="submit" name="removeAll" value="' . $txt['spider_log_empty_log'] . '" onclick="return confirm(\'' . $txt['spider_log_empty_log_confirm'] . '\');" class="button_submit" />',
  676. ),
  677. ),
  678. );
  679. createToken('admin-sl');
  680. require_once($sourcedir . '/Subs-List.php');
  681. createList($listOptions);
  682. // Now determine the actions of the URLs.
  683. if (!empty($context['spider_logs']['rows']))
  684. {
  685. $urls = array();
  686. // Grab the current /url.
  687. foreach ($context['spider_logs']['rows'] as $k => $row)
  688. {
  689. // Feature disabled?
  690. if (empty($row['viewing']['value']) && isset($modSettings['spider_mode']) && $modSettings['spider_mode'] < 3)
  691. $context['spider_logs']['rows'][$k]['viewing']['value'] = '<em>' . $txt['spider_disabled'] . '</em>';
  692. else
  693. $urls[$k] = array($row['viewing']['value'], -1);
  694. }
  695. // Now stick in the new URLs.
  696. require_once($sourcedir . '/Who.php');
  697. $urls = determineActions($urls, 'whospider_');
  698. foreach ($urls as $k => $new_url)
  699. {
  700. $context['spider_logs']['rows'][$k]['viewing']['value'] = $new_url;
  701. }
  702. }
  703. $context['page_title'] = $txt['spider_logs'];
  704. $context['sub_template'] = 'show_spider_logs';
  705. $context['default_list'] = 'spider_logs';
  706. }
  707. /**
  708. * Callback function for createList()
  709. *
  710. * @param int $start
  711. * @param int $items_per_page
  712. * @param string $sort
  713. * @return array
  714. */
  715. function list_getSpiderLogs($start, $items_per_page, $sort)
  716. {
  717. global $smcFunc;
  718. $request = $smcFunc['db_query']('', '
  719. SELECT sl.id_spider, sl.url, sl.log_time, s.spider_name
  720. FROM {db_prefix}log_spider_hits AS sl
  721. INNER JOIN {db_prefix}spiders AS s ON (s.id_spider = sl.id_spider)
  722. ORDER BY ' . $sort . '
  723. LIMIT ' . $start . ', ' . $items_per_page,
  724. array(
  725. )
  726. );
  727. $spider_logs = array();
  728. while ($row = $smcFunc['db_fetch_assoc']($request))
  729. $spider_logs[] = $row;
  730. $smcFunc['db_free_result']($request);
  731. return $spider_logs;
  732. }
  733. /**
  734. * Callback function for createList()
  735. * @return int
  736. */
  737. function list_getNumSpiderLogs()
  738. {
  739. global $smcFunc;
  740. $request = $smcFunc['db_query']('', '
  741. SELECT COUNT(*) AS num_logs
  742. FROM {db_prefix}log_spider_hits',
  743. array(
  744. )
  745. );
  746. list ($numLogs) = $smcFunc['db_fetch_row']($request);
  747. $smcFunc['db_free_result']($request);
  748. return $numLogs;
  749. }
  750. /**
  751. * Show the spider statistics.
  752. */
  753. function SpiderStats()
  754. {
  755. global $context, $txt, $sourcedir, $scripturl, $smcFunc;
  756. // Force an update of the stats every 60 seconds.
  757. if (!isset($_SESSION['spider_stat']) || $_SESSION['spider_stat'] < time() - 60)
  758. {
  759. consolidateSpiderStats();
  760. $_SESSION['spider_stat'] = time();
  761. }
  762. // Are we cleaning up some old stats?
  763. if (!empty($_POST['delete_entries']) && isset($_POST['older']))
  764. {
  765. checkSession();
  766. validateToken('admin-ss');
  767. $deleteTime = time() - (((int) $_POST['older']) * 24 * 60 * 60);
  768. // Delete the entires.
  769. $smcFunc['db_query']('', '
  770. DELETE FROM {db_prefix}log_spider_stats
  771. WHERE last_seen < {int:delete_period}',
  772. array(
  773. 'delete_period' => $deleteTime,
  774. )
  775. );
  776. }
  777. // Get the earliest and latest dates.
  778. $request = $smcFunc['db_query']('', '
  779. SELECT MIN(stat_date) AS first_date, MAX(stat_date) AS last_date
  780. FROM {db_prefix}log_spider_stats',
  781. array(
  782. )
  783. );
  784. list ($min_date, $max_date) = $smcFunc['db_fetch_row']($request);
  785. $smcFunc['db_free_result']($request);
  786. $min_year = (int) substr($min_date, 0, 4);
  787. $max_year = (int) substr($max_date, 0, 4);
  788. $min_month = (int) substr($min_date, 5, 2);
  789. $max_month = (int) substr($max_date, 5, 2);
  790. // Prepare the dates for the drop down.
  791. $date_choices = array();
  792. for ($y = $min_year; $y <= $max_year; $y++)
  793. for ($m = 1; $m <= 12; $m++)
  794. {
  795. // This doesn't count?
  796. if ($y == $min_year && $m < $min_month)
  797. continue;
  798. if ($y == $max_year && $m > $max_month)
  799. break;
  800. $date_choices[$y . $m] = $txt['months_short'][$m] . ' ' . $y;
  801. }
  802. // What are we currently viewing?
  803. $current_date = isset($_REQUEST['new_date']) && isset($date_choices[$_REQUEST['new_date']]) ? $_REQUEST['new_date'] : $max_date;
  804. // Prepare the HTML.
  805. $date_select = '
  806. ' . $txt['spider_stats_select_month'] . ':
  807. <select name="new_date" onchange="document.spider_stat_list.submit();">';
  808. if (empty($date_choices))
  809. $date_select .= '
  810. <option></option>';
  811. else
  812. foreach ($date_choices as $id => $text)
  813. $date_select .= '
  814. <option value="' . $id . '"' . ($current_date == $id ? ' selected="selected"' : '') . '>' . $text . '</option>';
  815. $date_select .= '
  816. </select>
  817. <noscript>
  818. <input type="submit" name="go" value="' . $txt['go'] . '" class="button_submit" />
  819. </noscript>';
  820. // If we manually jumped to a date work out the offset.
  821. if (isset($_REQUEST['new_date']))
  822. {
  823. $date_query = sprintf('%04d-%02d-01', substr($current_date, 0, 4), substr($current_date, 4));
  824. $request = $smcFunc['db_query']('', '
  825. SELECT COUNT(*) AS offset
  826. FROM {db_prefix}log_spider_stats
  827. WHERE stat_date < {date:date_being_viewed}',
  828. array(
  829. 'date_being_viewed' => $date_query,
  830. )
  831. );
  832. list ($_REQUEST['start']) = $smcFunc['db_fetch_row']($request);
  833. $smcFunc['db_free_result']($request);
  834. }
  835. $listOptions = array(
  836. 'id' => 'spider_stat_list',
  837. 'title' => $txt['spider'] . ' ' . $txt['spider_stats'],
  838. 'items_per_page' => 20,
  839. 'base_href' => $scripturl . '?action=admin;area=sengines;sa=stats',
  840. 'default_sort_col' => 'stat_date',
  841. 'get_items' => array(
  842. 'function' => 'list_getSpiderStats',
  843. ),
  844. 'get_count' => array(
  845. 'function' => 'list_getNumSpiderStats',
  846. ),
  847. 'no_items_label' => $txt['spider_stats_no_entries'],
  848. 'columns' => array(
  849. 'stat_date' => array(
  850. 'header' => array(
  851. 'value' => $txt['date'],
  852. ),
  853. 'data' => array(
  854. 'db' => 'stat_date',
  855. ),
  856. 'sort' => array(
  857. 'default' => 'stat_date',
  858. 'reverse' => 'stat_date DESC',
  859. ),
  860. ),
  861. 'name' => array(
  862. 'header' => array(
  863. 'value' => $txt['spider_name'],
  864. ),
  865. 'data' => array(
  866. 'db' => 'spider_name',
  867. ),
  868. 'sort' => array(
  869. 'default' => 's.spider_name',
  870. 'reverse' => 's.spider_name DESC',
  871. ),
  872. ),
  873. 'page_hits' => array(
  874. 'header' => array(
  875. 'value' => $txt['spider_stats_page_hits'],
  876. ),
  877. 'data' => array(
  878. 'db' => 'page_hits',
  879. ),
  880. 'sort' => array(
  881. 'default' => 'ss.page_hits',
  882. 'reverse' => 'ss.page_hits DESC',
  883. ),
  884. ),
  885. ),
  886. 'form' => array(
  887. 'href' => $scripturl . '?action=admin;area=sengines;sa=stats',
  888. 'name' => 'spider_stat_list',
  889. ),
  890. 'additional_rows' => array(
  891. array(
  892. 'position' => 'below_table_data',
  893. 'value' => $date_select,
  894. 'style' => 'text-align: right;',
  895. ),
  896. ),
  897. );
  898. createToken('admin-ss');
  899. require_once($sourcedir . '/Subs-List.php');
  900. createList($listOptions);
  901. $context['sub_template'] = 'show_spider_stats';
  902. $context['default_list'] = 'spider_stat_list';
  903. }
  904. /**
  905. * Callback function for createList()
  906. * Get a list of spider stats from the log_spider table
  907. *
  908. * @param type $start
  909. * @param type $items_per_page
  910. * @param type $sort
  911. * @return array
  912. */
  913. function list_getSpiderStats($start, $items_per_page, $sort)
  914. {
  915. global $smcFunc;
  916. $request = $smcFunc['db_query']('', '
  917. SELECT ss.id_spider, ss.stat_date, ss.page_hits, s.spider_name
  918. FROM {db_prefix}log_spider_stats AS ss
  919. INNER JOIN {db_prefix}spiders AS s ON (s.id_spider = ss.id_spider)
  920. ORDER BY ' . $sort . '
  921. LIMIT ' . $start . ', ' . $items_per_page,
  922. array(
  923. )
  924. );
  925. $spider_stats = array();
  926. while ($row = $smcFunc['db_fetch_assoc']($request))
  927. $spider_stats[] = $row;
  928. $smcFunc['db_free_result']($request);
  929. return $spider_stats;
  930. }
  931. /**
  932. * Callback function for createList()
  933. * Get the number of spider stat rows from the log spicer stats table
  934. *
  935. * @return int
  936. */
  937. function list_getNumSpiderStats()
  938. {
  939. global $smcFunc;
  940. $request = $smcFunc['db_query']('', '
  941. SELECT COUNT(*) AS num_stats
  942. FROM {db_prefix}log_spider_stats',
  943. array(
  944. )
  945. );
  946. list ($numStats) = $smcFunc['db_fetch_row']($request);
  947. $smcFunc['db_free_result']($request);
  948. return $numStats;
  949. }
  950. /**
  951. * Recache spider names?
  952. */
  953. function recacheSpiderNames()
  954. {
  955. global $smcFunc;
  956. $request = $smcFunc['db_query']('', '
  957. SELECT id_spider, spider_name
  958. FROM {db_prefix}spiders',
  959. array(
  960. )
  961. );
  962. $spiders = array();
  963. while ($row = $smcFunc['db_fetch_assoc']($request))
  964. $spiders[$row['id_spider']] = $row['spider_name'];
  965. $smcFunc['db_free_result']($request);
  966. updateSettings(array('spider_name_cache' => serialize($spiders)));
  967. }
  968. /**
  969. * Sort the search engine table by user agent name to avoid misidentification of engine.
  970. */
  971. function sortSpiderTable()
  972. {
  973. global $smcFunc;
  974. db_extend('packages');
  975. // Add a sorting column.
  976. $smcFunc['db_add_column']('{db_prefix}spiders', array('name' => 'temp_order', 'size' => 8, 'type' => 'mediumint', 'null' => false));
  977. // Set the contents of this column.
  978. $smcFunc['db_query']('set_spider_order', '
  979. UPDATE {db_prefix}spiders
  980. SET temp_order = LENGTH(user_agent)',
  981. array(
  982. )
  983. );
  984. // Order the table by this column.
  985. $smcFunc['db_query']('alter_table_spiders', '
  986. ALTER TABLE {db_prefix}spiders
  987. ORDER BY temp_order DESC',
  988. array(
  989. 'db_error_skip' => true,
  990. )
  991. );
  992. // Remove the sorting column.
  993. $smcFunc['db_remove_column']('{db_prefix}spiders', 'temp_order');
  994. }
  995. ?>