ManageSearchEngines.php 29 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139
  1. <?php
  2. /**
  3. * This file contains all the screens that relate to search engines.
  4. *
  5. * Simple Machines Forum (SMF)
  6. *
  7. * @package SMF
  8. * @author Simple Machines http://www.simplemachines.org
  9. * @copyright 2013 Simple Machines and individual contributors
  10. * @license http://www.simplemachines.org/about/smf/license.php BSD
  11. *
  12. * @version 2.1 Alpha 1
  13. */
  14. if (!defined('SMF'))
  15. die('No direct access...');
  16. /**
  17. * Entry point for this section.
  18. */
  19. function SearchEngines()
  20. {
  21. global $context, $txt, $scripturl, $modSettings;
  22. isAllowedTo('admin_forum');
  23. loadLanguage('Search');
  24. loadTemplate('ManageSearch');
  25. if (!empty($modSettings['spider_mode']))
  26. {
  27. $subActions = array(
  28. 'editspiders' => 'EditSpider',
  29. 'logs' => 'SpiderLogs',
  30. 'settings' => 'ManageSearchEngineSettings',
  31. 'spiders' => 'ViewSpiders',
  32. 'stats' => 'SpiderStats',
  33. );
  34. $default = 'stats';
  35. }
  36. else
  37. {
  38. $subActions = array(
  39. 'settings' => 'ManageSearchEngineSettings',
  40. );
  41. $default = 'settings';
  42. }
  43. call_integration_hook('integrate_manage_search_engines', array(&$subActions));
  44. // Ensure we have a valid subaction.
  45. $context['sub_action'] = isset($_REQUEST['sa']) && isset($subActions[$_REQUEST['sa']]) ? $_REQUEST['sa'] : $default;
  46. $context['page_title'] = $txt['search_engines'];
  47. // Some more tab data.
  48. $context[$context['admin_menu_name']]['tab_data'] = array(
  49. 'title' => $txt['search_engines'],
  50. 'description' => $txt['search_engines_description'],
  51. );
  52. // Call the function!
  53. $subActions[$context['sub_action']]();
  54. }
  55. /**
  56. * This is really just the settings page.
  57. *
  58. * @param bool $return_config = false
  59. */
  60. function ManageSearchEngineSettings($return_config = false)
  61. {
  62. global $context, $txt, $modSettings, $scripturl, $sourcedir, $smcFunc;
  63. $config_vars = array(
  64. // How much detail?
  65. array('select', 'spider_mode', 'subtext' => $txt['spider_mode_note'], array($txt['spider_mode_off'], $txt['spider_mode_standard'], $txt['spider_mode_high'], $txt['spider_mode_vhigh']), 'onchange' => 'disableFields();'),
  66. 'spider_group' => array('select', 'spider_group', 'subtext' => $txt['spider_group_note'], array($txt['spider_group_none'], $txt['membergroups_members'])),
  67. array('select', 'show_spider_online', array($txt['show_spider_online_no'], $txt['show_spider_online_summary'], $txt['show_spider_online_detail'], $txt['show_spider_online_detail_admin'])),
  68. );
  69. // Set up a message.
  70. $context['settings_message'] = '<span class="smalltext">' . sprintf($txt['spider_settings_desc'], $scripturl . '?action=admin;area=logs;sa=pruning;' . $context['session_var'] . '=' . $context['session_id']) . '</span>';
  71. // Do some javascript.
  72. $javascript_function = '
  73. function disableFields()
  74. {
  75. disabledState = document.getElementById(\'spider_mode\').value == 0;';
  76. foreach ($config_vars as $variable)
  77. if ($variable[1] != 'spider_mode')
  78. $javascript_function .= '
  79. if (document.getElementById(\'' . $variable[1] . '\'))
  80. document.getElementById(\'' . $variable[1] . '\').disabled = disabledState;';
  81. $javascript_function .= '
  82. }
  83. disableFields();';
  84. call_integration_hook('integrate_modify_search_engine_settings', array(&$config_vars));
  85. if ($return_config)
  86. return $config_vars;
  87. // We need to load the groups for the spider group thingy.
  88. $request = $smcFunc['db_query']('', '
  89. SELECT id_group, group_name
  90. FROM {db_prefix}membergroups
  91. WHERE id_group != {int:admin_group}
  92. AND id_group != {int:moderator_group}',
  93. array(
  94. 'admin_group' => 1,
  95. 'moderator_group' => 3,
  96. )
  97. );
  98. while ($row = $smcFunc['db_fetch_assoc']($request))
  99. $config_vars['spider_group'][2][$row['id_group']] = $row['group_name'];
  100. $smcFunc['db_free_result']($request);
  101. // Make sure it's valid - note that regular members are given id_group = 1 which is reversed in Load.php - no admins here!
  102. if (isset($_POST['spider_group']) && !isset($config_vars['spider_group'][2][$_POST['spider_group']]))
  103. $_POST['spider_group'] = 0;
  104. // We'll want this for our easy save.
  105. require_once($sourcedir . '/ManageServer.php');
  106. // Setup the template.
  107. $context['page_title'] = $txt['settings'];
  108. $context['sub_template'] = 'show_settings';
  109. // Are we saving them - are we??
  110. if (isset($_GET['save']))
  111. {
  112. checkSession();
  113. call_integration_hook('integrate_save_search_engine_settings');
  114. saveDBSettings($config_vars);
  115. recacheSpiderNames();
  116. $_SESSION['adm-save'] = true;
  117. redirectexit('action=admin;area=sengines;sa=settings');
  118. }
  119. // Final settings...
  120. $context['post_url'] = $scripturl . '?action=admin;area=sengines;save;sa=settings';
  121. $context['settings_title'] = $txt['settings'];
  122. $context['settings_insert_below'] = '
  123. <script type="text/javascript"><!-- // --><![CDATA[
  124. ' . $javascript_function . '
  125. // ]]></script>';
  126. // Prepare the settings...
  127. prepareDBSettingContext($config_vars);
  128. }
  129. /**
  130. * View a list of all the spiders we know about.
  131. */
  132. function ViewSpiders()
  133. {
  134. global $context, $txt, $sourcedir, $scripturl, $smcFunc;
  135. if (!isset($_SESSION['spider_stat']) || $_SESSION['spider_stat'] < time() - 60)
  136. {
  137. consolidateSpiderStats();
  138. $_SESSION['spider_stat'] = time();
  139. }
  140. // Are we adding a new one?
  141. if (!empty($_POST['addSpider']))
  142. return EditSpider();
  143. // User pressed the 'remove selection button'.
  144. elseif (!empty($_POST['removeSpiders']) && !empty($_POST['remove']) && is_array($_POST['remove']))
  145. {
  146. checkSession();
  147. validateToken('admin-ser');
  148. // Make sure every entry is a proper integer.
  149. foreach ($_POST['remove'] as $index => $spider_id)
  150. $_POST['remove'][(int) $index] = (int) $spider_id;
  151. // Delete them all!
  152. $smcFunc['db_query']('', '
  153. DELETE FROM {db_prefix}spiders
  154. WHERE id_spider IN ({array_int:remove_list})',
  155. array(
  156. 'remove_list' => $_POST['remove'],
  157. )
  158. );
  159. $smcFunc['db_query']('', '
  160. DELETE FROM {db_prefix}log_spider_hits
  161. WHERE id_spider IN ({array_int:remove_list})',
  162. array(
  163. 'remove_list' => $_POST['remove'],
  164. )
  165. );
  166. $smcFunc['db_query']('', '
  167. DELETE FROM {db_prefix}log_spider_stats
  168. WHERE id_spider IN ({array_int:remove_list})',
  169. array(
  170. 'remove_list' => $_POST['remove'],
  171. )
  172. );
  173. cache_put_data('spider_search', null, 300);
  174. recacheSpiderNames();
  175. }
  176. // Get the last seens.
  177. $request = $smcFunc['db_query']('', '
  178. SELECT id_spider, MAX(last_seen) AS last_seen_time
  179. FROM {db_prefix}log_spider_stats
  180. GROUP BY id_spider',
  181. array(
  182. )
  183. );
  184. $context['spider_last_seen'] = array();
  185. while ($row = $smcFunc['db_fetch_assoc']($request))
  186. $context['spider_last_seen'][$row['id_spider']] = $row['last_seen_time'];
  187. $smcFunc['db_free_result']($request);
  188. createToken('admin-ser');
  189. $listOptions = array(
  190. 'id' => 'spider_list',
  191. 'title' => $txt['spiders'],
  192. 'items_per_page' => 20,
  193. 'base_href' => $scripturl . '?action=admin;area=sengines;sa=spiders',
  194. 'default_sort_col' => 'name',
  195. 'get_items' => array(
  196. 'function' => 'list_getSpiders',
  197. ),
  198. 'get_count' => array(
  199. 'function' => 'list_getNumSpiders',
  200. ),
  201. 'no_items_label' => $txt['spiders_no_entries'],
  202. 'columns' => array(
  203. 'name' => array(
  204. 'header' => array(
  205. 'value' => $txt['spider_name'],
  206. ),
  207. 'data' => array(
  208. 'function' => create_function('$rowData', '
  209. global $scripturl, $smcFunc;
  210. return sprintf(\'<a href="%1$s?action=admin;area=sengines;sa=editspiders;sid=%2$d">%3$s</a>\', $scripturl, $rowData[\'id_spider\'], $smcFunc[\'htmlspecialchars\']($rowData[\'spider_name\']));
  211. '),
  212. ),
  213. 'sort' => array(
  214. 'default' => 'spider_name',
  215. 'reverse' => 'spider_name DESC',
  216. ),
  217. ),
  218. 'last_seen' => array(
  219. 'header' => array(
  220. 'value' => $txt['spider_last_seen'],
  221. ),
  222. 'data' => array(
  223. 'function' => create_function('$rowData', '
  224. global $context, $txt;
  225. return isset($context[\'spider_last_seen\'][$rowData[\'id_spider\']]) ? timeformat($context[\'spider_last_seen\'][$rowData[\'id_spider\']]) : $txt[\'spider_last_never\'];
  226. '),
  227. ),
  228. ),
  229. 'user_agent' => array(
  230. 'header' => array(
  231. 'value' => $txt['spider_agent'],
  232. ),
  233. 'data' => array(
  234. 'db_htmlsafe' => 'user_agent',
  235. ),
  236. 'sort' => array(
  237. 'default' => 'user_agent',
  238. 'reverse' => 'user_agent DESC',
  239. ),
  240. ),
  241. 'ip_info' => array(
  242. 'header' => array(
  243. 'value' => $txt['spider_ip_info'],
  244. ),
  245. 'data' => array(
  246. 'db_htmlsafe' => 'ip_info',
  247. 'class' => 'smalltext',
  248. ),
  249. 'sort' => array(
  250. 'default' => 'ip_info',
  251. 'reverse' => 'ip_info DESC',
  252. ),
  253. ),
  254. 'check' => array(
  255. 'header' => array(
  256. 'value' => '<input type="checkbox" onclick="invertAll(this, this.form);" class="input_check" />',
  257. 'class' => 'centercol',
  258. ),
  259. 'data' => array(
  260. 'sprintf' => array(
  261. 'format' => '<input type="checkbox" name="remove[]" value="%1$d" class="input_check" />',
  262. 'params' => array(
  263. 'id_spider' => false,
  264. ),
  265. ),
  266. 'class' => 'centercol',
  267. ),
  268. ),
  269. ),
  270. 'form' => array(
  271. 'href' => $scripturl . '?action=admin;area=sengines;sa=spiders',
  272. 'token' => 'admin-ser',
  273. ),
  274. 'additional_rows' => array(
  275. array(
  276. 'position' => 'bottom_of_list',
  277. 'value' => '
  278. <input type="submit" name="removeSpiders" value="' . $txt['spiders_remove_selected'] . '" onclick="return confirm(\'' . $txt['spider_remove_selected_confirm'] . '\');" class="button_submit" />
  279. <input type="submit" name="addSpider" value="' . $txt['spiders_add'] . '" class="button_submit" />
  280. ',
  281. ),
  282. ),
  283. );
  284. require_once($sourcedir . '/Subs-List.php');
  285. createList($listOptions);
  286. $context['sub_template'] = 'show_list';
  287. $context['default_list'] = 'spider_list';
  288. }
  289. /**
  290. * Callback function for createList()
  291. * @param int $start
  292. * @param int $items_per_page
  293. * @param string sort
  294. */
  295. function list_getSpiders($start, $items_per_page, $sort)
  296. {
  297. global $smcFunc;
  298. $request = $smcFunc['db_query']('', '
  299. SELECT id_spider, spider_name, user_agent, ip_info
  300. FROM {db_prefix}spiders
  301. ORDER BY ' . $sort . '
  302. LIMIT ' . $start . ', ' . $items_per_page,
  303. array(
  304. )
  305. );
  306. $spiders = array();
  307. while ($row = $smcFunc['db_fetch_assoc']($request))
  308. $spiders[$row['id_spider']] = $row;
  309. $smcFunc['db_free_result']($request);
  310. return $spiders;
  311. }
  312. /**
  313. * Callback function for createList()
  314. * @return int
  315. */
  316. function list_getNumSpiders()
  317. {
  318. global $smcFunc;
  319. $request = $smcFunc['db_query']('', '
  320. SELECT COUNT(*) AS num_spiders
  321. FROM {db_prefix}spiders',
  322. array(
  323. )
  324. );
  325. list ($numSpiders) = $smcFunc['db_fetch_row']($request);
  326. $smcFunc['db_free_result']($request);
  327. return $numSpiders;
  328. }
  329. /**
  330. * Here we can add, and edit, spider info!
  331. */
  332. function EditSpider()
  333. {
  334. global $context, $smcFunc, $txt;
  335. // Some standard stuff.
  336. $context['id_spider'] = !empty($_GET['sid']) ? (int) $_GET['sid'] : 0;
  337. $context['page_title'] = $context['id_spider'] ? $txt['spiders_edit'] : $txt['spiders_add'];
  338. $context['sub_template'] = 'spider_edit';
  339. // Are we saving?
  340. if (!empty($_POST['save']))
  341. {
  342. checkSession();
  343. validateToken('admin-ses');
  344. $ips = array();
  345. // Check the IP range is valid.
  346. $ip_sets = explode(',', $_POST['spider_ip']);
  347. foreach ($ip_sets as $set)
  348. {
  349. $test = ip2range(trim($set));
  350. if (!empty($test))
  351. $ips[] = $set;
  352. }
  353. $ips = implode(',', $ips);
  354. // Goes in as it is...
  355. if ($context['id_spider'])
  356. $smcFunc['db_query']('', '
  357. UPDATE {db_prefix}spiders
  358. SET spider_name = {string:spider_name}, user_agent = {string:spider_agent},
  359. ip_info = {string:ip_info}
  360. WHERE id_spider = {int:current_spider}',
  361. array(
  362. 'current_spider' => $context['id_spider'],
  363. 'spider_name' => $_POST['spider_name'],
  364. 'spider_agent' => $_POST['spider_agent'],
  365. 'ip_info' => $ips,
  366. )
  367. );
  368. else
  369. $smcFunc['db_insert']('insert',
  370. '{db_prefix}spiders',
  371. array(
  372. 'spider_name' => 'string', 'user_agent' => 'string', 'ip_info' => 'string',
  373. ),
  374. array(
  375. $_POST['spider_name'], $_POST['spider_agent'], $ips,
  376. ),
  377. array('id_spider')
  378. );
  379. // Order by user agent length.
  380. sortSpiderTable();
  381. cache_put_data('spider_search', null, 300);
  382. recacheSpiderNames();
  383. redirectexit('action=admin;area=sengines;sa=spiders');
  384. }
  385. // The default is new.
  386. $context['spider'] = array(
  387. 'id' => 0,
  388. 'name' => '',
  389. 'agent' => '',
  390. 'ip_info' => '',
  391. );
  392. // An edit?
  393. if ($context['id_spider'])
  394. {
  395. $request = $smcFunc['db_query']('', '
  396. SELECT id_spider, spider_name, user_agent, ip_info
  397. FROM {db_prefix}spiders
  398. WHERE id_spider = {int:current_spider}',
  399. array(
  400. 'current_spider' => $context['id_spider'],
  401. )
  402. );
  403. if ($row = $smcFunc['db_fetch_assoc']($request))
  404. $context['spider'] = array(
  405. 'id' => $row['id_spider'],
  406. 'name' => $row['spider_name'],
  407. 'agent' => $row['user_agent'],
  408. 'ip_info' => $row['ip_info'],
  409. );
  410. $smcFunc['db_free_result']($request);
  411. }
  412. createToken('admin-ses');
  413. }
  414. /**
  415. * Do we think the current user is a spider?
  416. *
  417. * @todo Should this not be... you know... in a different file?
  418. * @return int
  419. */
  420. function SpiderCheck()
  421. {
  422. global $modSettings, $smcFunc;
  423. if (isset($_SESSION['id_robot']))
  424. unset($_SESSION['id_robot']);
  425. $_SESSION['robot_check'] = time();
  426. // We cache the spider data for five minutes if we can.
  427. if (($spider_data = cache_get_data('spider_search', 300)) === null)
  428. {
  429. $request = $smcFunc['db_query']('spider_check', '
  430. SELECT id_spider, user_agent, ip_info
  431. FROM {db_prefix}spiders',
  432. array(
  433. )
  434. );
  435. $spider_data = array();
  436. while ($row = $smcFunc['db_fetch_assoc']($request))
  437. $spider_data[] = $row;
  438. $smcFunc['db_free_result']($request);
  439. cache_put_data('spider_search', $spider_data, 300);
  440. }
  441. if (empty($spider_data))
  442. return false;
  443. // Only do these bits once.
  444. $ci_user_agent = strtolower($_SERVER['HTTP_USER_AGENT']);
  445. // Always attempt IPv6 first.
  446. if (strpos($_SERVER['REMOTE_ADDR'], ':') !== false)
  447. $ip_parts = convertIPv6toInts($_SERVER['REMOTE_ADDR']);
  448. else
  449. preg_match('/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/', $_SERVER['REMOTE_ADDR'], $ip_parts);
  450. foreach ($spider_data as $spider)
  451. {
  452. // User agent is easy.
  453. if (!empty($spider['user_agent']) && strpos($ci_user_agent, strtolower($spider['user_agent'])) !== false)
  454. $_SESSION['id_robot'] = $spider['id_spider'];
  455. // IP stuff is harder.
  456. elseif (!empty($ip_parts))
  457. {
  458. $ips = explode(',', $spider['ip_info']);
  459. foreach ($ips as $ip)
  460. {
  461. $ip = ip2range($ip);
  462. if (!empty($ip))
  463. {
  464. foreach ($ip as $key => $value)
  465. {
  466. if ($value['low'] > $ip_parts[$key + 1] || $value['high'] < $ip_parts[$key + 1])
  467. break;
  468. elseif (($key == 7 && strpos($_SERVER['REMOTE_ADDR'], ':') !== false) || ($key == 3 && strpos($_SERVER['REMOTE_ADDR'], ':') === false))
  469. $_SESSION['id_robot'] = $spider['id_spider'];
  470. }
  471. }
  472. }
  473. }
  474. if (isset($_SESSION['id_robot']))
  475. break;
  476. }
  477. // If this is low server tracking then log the spider here as oppossed to the main logging function.
  478. if (!empty($modSettings['spider_mode']) && $modSettings['spider_mode'] == 1 && !empty($_SESSION['id_robot']))
  479. logSpider();
  480. return !empty($_SESSION['id_robot']) ? $_SESSION['id_robot'] : 0;
  481. }
  482. /**
  483. * Log the spider presence online.
  484. *
  485. * @todo Different file?
  486. */
  487. function logSpider()
  488. {
  489. global $smcFunc, $modSettings, $context;
  490. if (empty($modSettings['spider_mode']) || empty($_SESSION['id_robot']))
  491. return;
  492. // Attempt to update today's entry.
  493. if ($modSettings['spider_mode'] == 1)
  494. {
  495. $date = strftime('%Y-%m-%d', forum_time(false));
  496. $smcFunc['db_query']('', '
  497. UPDATE {db_prefix}log_spider_stats
  498. SET last_seen = {int:current_time}, page_hits = page_hits + 1
  499. WHERE id_spider = {int:current_spider}
  500. AND stat_date = {date:current_date}',
  501. array(
  502. 'current_date' => $date,
  503. 'current_time' => time(),
  504. 'current_spider' => $_SESSION['id_robot'],
  505. )
  506. );
  507. // Nothing updated?
  508. if ($smcFunc['db_affected_rows']() == 0)
  509. {
  510. $smcFunc['db_insert']('ignore',
  511. '{db_prefix}log_spider_stats',
  512. array(
  513. 'id_spider' => 'int', 'last_seen' => 'int', 'stat_date' => 'date', 'page_hits' => 'int',
  514. ),
  515. array(
  516. $_SESSION['id_robot'], time(), $date, 1,
  517. ),
  518. array('id_spider', 'stat_date')
  519. );
  520. }
  521. }
  522. // If we're tracking better stats than track, better stats - we sort out the today thing later.
  523. else
  524. {
  525. if ($modSettings['spider_mode'] > 2)
  526. {
  527. $url = $_GET + array('USER_AGENT' => $_SERVER['HTTP_USER_AGENT']);
  528. unset($url['sesc'], $url[$context['session_var']]);
  529. $url = serialize($url);
  530. }
  531. else
  532. $url = '';
  533. $smcFunc['db_insert']('insert',
  534. '{db_prefix}log_spider_hits',
  535. array('id_spider' => 'int', 'log_time' => 'int', 'url' => 'string'),
  536. array($_SESSION['id_robot'], time(), $url),
  537. array()
  538. );
  539. }
  540. }
  541. /**
  542. * This function takes any unprocessed hits and turns them into stats.
  543. */
  544. function consolidateSpiderStats()
  545. {
  546. global $smcFunc;
  547. $request = $smcFunc['db_query']('consolidate_spider_stats', '
  548. SELECT id_spider, MAX(log_time) AS last_seen, COUNT(*) AS num_hits
  549. FROM {db_prefix}log_spider_hits
  550. WHERE processed = {int:not_processed}
  551. GROUP BY id_spider, MONTH(log_time), DAYOFMONTH(log_time)',
  552. array(
  553. 'not_processed' => 0,
  554. )
  555. );
  556. $spider_hits = array();
  557. while ($row = $smcFunc['db_fetch_assoc']($request))
  558. $spider_hits[] = $row;
  559. $smcFunc['db_free_result']($request);
  560. if (empty($spider_hits))
  561. return;
  562. // Attempt to update the master data.
  563. $stat_inserts = array();
  564. foreach ($spider_hits as $stat)
  565. {
  566. // We assume the max date is within the right day.
  567. $date = strftime('%Y-%m-%d', $stat['last_seen']);
  568. $smcFunc['db_query']('', '
  569. UPDATE {db_prefix}log_spider_stats
  570. SET page_hits = page_hits + ' . $stat['num_hits'] . ',
  571. last_seen = CASE WHEN last_seen > {int:last_seen} THEN last_seen ELSE {int:last_seen} END
  572. WHERE id_spider = {int:current_spider}
  573. AND stat_date = {date:last_seen_date}',
  574. array(
  575. 'last_seen_date' => $date,
  576. 'last_seen' => $stat['last_seen'],
  577. 'current_spider' => $stat['id_spider'],
  578. )
  579. );
  580. if ($smcFunc['db_affected_rows']() == 0)
  581. $stat_inserts[] = array($date, $stat['id_spider'], $stat['num_hits'], $stat['last_seen']);
  582. }
  583. // New stats?
  584. if (!empty($stat_inserts))
  585. $smcFunc['db_insert']('ignore',
  586. '{db_prefix}log_spider_stats',
  587. array('stat_date' => 'date', 'id_spider' => 'int', 'page_hits' => 'int', 'last_seen' => 'int'),
  588. $stat_inserts,
  589. array('stat_date', 'id_spider')
  590. );
  591. // All processed.
  592. $smcFunc['db_query']('', '
  593. UPDATE {db_prefix}log_spider_hits
  594. SET processed = {int:is_processed}
  595. WHERE processed = {int:not_processed}',
  596. array(
  597. 'is_processed' => 1,
  598. 'not_processed' => 0,
  599. )
  600. );
  601. }
  602. /**
  603. * See what spiders have been up to.
  604. */
  605. function SpiderLogs()
  606. {
  607. global $context, $txt, $sourcedir, $scripturl, $smcFunc, $modSettings;
  608. // Load the template and language just incase.
  609. loadLanguage('Search');
  610. loadTemplate('ManageSearch');
  611. // Did they want to delete some entries?
  612. if (!empty($_POST['delete_entries']) && isset($_POST['older']))
  613. {
  614. checkSession();
  615. validateToken('admin-sl');
  616. $deleteTime = time() - (((int) $_POST['older']) * 24 * 60 * 60);
  617. // Delete the entires.
  618. $smcFunc['db_query']('', '
  619. DELETE FROM {db_prefix}log_spider_hits
  620. WHERE log_time < {int:delete_period}',
  621. array(
  622. 'delete_period' => $deleteTime,
  623. )
  624. );
  625. }
  626. $listOptions = array(
  627. 'id' => 'spider_logs',
  628. 'items_per_page' => 20,
  629. 'title' => $txt['spider_logs'],
  630. 'no_items_label' => $txt['spider_logs_empty'],
  631. 'base_href' => $context['admin_area'] == 'sengines' ? $scripturl . '?action=admin;area=sengines;sa=logs' : $scripturl . '?action=admin;area=logs;sa=spiderlog',
  632. 'default_sort_col' => 'log_time',
  633. 'get_items' => array(
  634. 'function' => 'list_getSpiderLogs',
  635. ),
  636. 'get_count' => array(
  637. 'function' => 'list_getNumSpiderLogs',
  638. ),
  639. 'columns' => array(
  640. 'name' => array(
  641. 'header' => array(
  642. 'value' => $txt['spider'],
  643. ),
  644. 'data' => array(
  645. 'db' => 'spider_name',
  646. ),
  647. 'sort' => array(
  648. 'default' => 's.spider_name',
  649. 'reverse' => 's.spider_name DESC',
  650. ),
  651. ),
  652. 'log_time' => array(
  653. 'header' => array(
  654. 'value' => $txt['spider_time'],
  655. ),
  656. 'data' => array(
  657. 'function' => create_function('$rowData', '
  658. return timeformat($rowData[\'log_time\']);
  659. '),
  660. ),
  661. 'sort' => array(
  662. 'default' => 'sl.id_hit DESC',
  663. 'reverse' => 'sl.id_hit',
  664. ),
  665. ),
  666. 'viewing' => array(
  667. 'header' => array(
  668. 'value' => $txt['spider_viewing'],
  669. ),
  670. 'data' => array(
  671. 'db' => 'url',
  672. ),
  673. ),
  674. ),
  675. 'form' => array(
  676. 'token' => 'admin-sl',
  677. 'href' => $scripturl . '?action=admin;area=sengines;sa=logs',
  678. ),
  679. 'additional_rows' => array(
  680. array(
  681. 'position' => 'after_title',
  682. 'value' => $txt['spider_logs_info'],
  683. 'class' => 'windowbg2',
  684. ),
  685. array(
  686. 'position' => 'below_table_data',
  687. 'value' => '<input type="submit" name="removeAll" value="' . $txt['spider_log_empty_log'] . '" onclick="return confirm(\'' . $txt['spider_log_empty_log_confirm'] . '\');" class="button_submit" />',
  688. ),
  689. ),
  690. );
  691. createToken('admin-sl');
  692. require_once($sourcedir . '/Subs-List.php');
  693. createList($listOptions);
  694. // Now determine the actions of the URLs.
  695. if (!empty($context['spider_logs']['rows']))
  696. {
  697. $urls = array();
  698. // Grab the current /url.
  699. foreach ($context['spider_logs']['rows'] as $k => $row)
  700. {
  701. // Feature disabled?
  702. if (empty($row['viewing']['value']) && isset($modSettings['spider_mode']) && $modSettings['spider_mode'] < 3)
  703. $context['spider_logs']['rows'][$k]['viewing']['value'] = '<em>' . $txt['spider_disabled'] . '</em>';
  704. else
  705. $urls[$k] = array($row['viewing']['value'], -1);
  706. }
  707. // Now stick in the new URLs.
  708. require_once($sourcedir . '/Who.php');
  709. $urls = determineActions($urls, 'whospider_');
  710. foreach ($urls as $k => $new_url)
  711. {
  712. $context['spider_logs']['rows'][$k]['viewing']['value'] = $new_url;
  713. }
  714. }
  715. $context['page_title'] = $txt['spider_logs'];
  716. $context['sub_template'] = 'show_spider_logs';
  717. $context['default_list'] = 'spider_logs';
  718. }
  719. /**
  720. * Callback function for createList()
  721. *
  722. * @param int $start
  723. * @param int $items_per_page
  724. * @param string $sort
  725. * @return array
  726. */
  727. function list_getSpiderLogs($start, $items_per_page, $sort)
  728. {
  729. global $smcFunc;
  730. $request = $smcFunc['db_query']('', '
  731. SELECT sl.id_spider, sl.url, sl.log_time, s.spider_name
  732. FROM {db_prefix}log_spider_hits AS sl
  733. INNER JOIN {db_prefix}spiders AS s ON (s.id_spider = sl.id_spider)
  734. ORDER BY ' . $sort . '
  735. LIMIT ' . $start . ', ' . $items_per_page,
  736. array(
  737. )
  738. );
  739. $spider_logs = array();
  740. while ($row = $smcFunc['db_fetch_assoc']($request))
  741. $spider_logs[] = $row;
  742. $smcFunc['db_free_result']($request);
  743. return $spider_logs;
  744. }
  745. /**
  746. * Callback function for createList()
  747. * @return int
  748. */
  749. function list_getNumSpiderLogs()
  750. {
  751. global $smcFunc;
  752. $request = $smcFunc['db_query']('', '
  753. SELECT COUNT(*) AS num_logs
  754. FROM {db_prefix}log_spider_hits',
  755. array(
  756. )
  757. );
  758. list ($numLogs) = $smcFunc['db_fetch_row']($request);
  759. $smcFunc['db_free_result']($request);
  760. return $numLogs;
  761. }
  762. /**
  763. * Show the spider statistics.
  764. */
  765. function SpiderStats()
  766. {
  767. global $context, $txt, $sourcedir, $scripturl, $smcFunc;
  768. // Force an update of the stats every 60 seconds.
  769. if (!isset($_SESSION['spider_stat']) || $_SESSION['spider_stat'] < time() - 60)
  770. {
  771. consolidateSpiderStats();
  772. $_SESSION['spider_stat'] = time();
  773. }
  774. // Are we cleaning up some old stats?
  775. if (!empty($_POST['delete_entries']) && isset($_POST['older']))
  776. {
  777. checkSession();
  778. validateToken('admin-ss');
  779. $deleteTime = time() - (((int) $_POST['older']) * 24 * 60 * 60);
  780. // Delete the entires.
  781. $smcFunc['db_query']('', '
  782. DELETE FROM {db_prefix}log_spider_stats
  783. WHERE last_seen < {int:delete_period}',
  784. array(
  785. 'delete_period' => $deleteTime,
  786. )
  787. );
  788. }
  789. // Get the earliest and latest dates.
  790. $request = $smcFunc['db_query']('', '
  791. SELECT MIN(stat_date) AS first_date, MAX(stat_date) AS last_date
  792. FROM {db_prefix}log_spider_stats',
  793. array(
  794. )
  795. );
  796. list ($min_date, $max_date) = $smcFunc['db_fetch_row']($request);
  797. $smcFunc['db_free_result']($request);
  798. $min_year = (int) substr($min_date, 0, 4);
  799. $max_year = (int) substr($max_date, 0, 4);
  800. $min_month = (int) substr($min_date, 5, 2);
  801. $max_month = (int) substr($max_date, 5, 2);
  802. // Prepare the dates for the drop down.
  803. $date_choices = array();
  804. for ($y = $min_year; $y <= $max_year; $y++)
  805. for ($m = 1; $m <= 12; $m++)
  806. {
  807. // This doesn't count?
  808. if ($y == $min_year && $m < $min_month)
  809. continue;
  810. if ($y == $max_year && $m > $max_month)
  811. break;
  812. $date_choices[$y . $m] = $txt['months_short'][$m] . ' ' . $y;
  813. }
  814. // What are we currently viewing?
  815. $current_date = isset($_REQUEST['new_date']) && isset($date_choices[$_REQUEST['new_date']]) ? $_REQUEST['new_date'] : $max_date;
  816. // Prepare the HTML.
  817. $date_select = '
  818. ' . $txt['spider_stats_select_month'] . ':
  819. <select name="new_date" onchange="document.spider_stat_list.submit();">';
  820. if (empty($date_choices))
  821. $date_select .= '
  822. <option></option>';
  823. else
  824. foreach ($date_choices as $id => $text)
  825. $date_select .= '
  826. <option value="' . $id . '"' . ($current_date == $id ? ' selected="selected"' : '') . '>' . $text . '</option>';
  827. $date_select .= '
  828. </select>
  829. <noscript>
  830. <input type="submit" name="go" value="' . $txt['go'] . '" class="button_submit" />
  831. </noscript>';
  832. // If we manually jumped to a date work out the offset.
  833. if (isset($_REQUEST['new_date']))
  834. {
  835. $date_query = sprintf('%04d-%02d-01', substr($current_date, 0, 4), substr($current_date, 4));
  836. $request = $smcFunc['db_query']('', '
  837. SELECT COUNT(*) AS offset
  838. FROM {db_prefix}log_spider_stats
  839. WHERE stat_date < {date:date_being_viewed}',
  840. array(
  841. 'date_being_viewed' => $date_query,
  842. )
  843. );
  844. list ($_REQUEST['start']) = $smcFunc['db_fetch_row']($request);
  845. $smcFunc['db_free_result']($request);
  846. }
  847. $listOptions = array(
  848. 'id' => 'spider_stat_list',
  849. 'title' => $txt['spider'] . ' ' . $txt['spider_stats'],
  850. 'items_per_page' => 20,
  851. 'base_href' => $scripturl . '?action=admin;area=sengines;sa=stats',
  852. 'default_sort_col' => 'stat_date',
  853. 'get_items' => array(
  854. 'function' => 'list_getSpiderStats',
  855. ),
  856. 'get_count' => array(
  857. 'function' => 'list_getNumSpiderStats',
  858. ),
  859. 'no_items_label' => $txt['spider_stats_no_entries'],
  860. 'columns' => array(
  861. 'stat_date' => array(
  862. 'header' => array(
  863. 'value' => $txt['date'],
  864. ),
  865. 'data' => array(
  866. 'db' => 'stat_date',
  867. ),
  868. 'sort' => array(
  869. 'default' => 'stat_date',
  870. 'reverse' => 'stat_date DESC',
  871. ),
  872. ),
  873. 'name' => array(
  874. 'header' => array(
  875. 'value' => $txt['spider_name'],
  876. ),
  877. 'data' => array(
  878. 'db' => 'spider_name',
  879. ),
  880. 'sort' => array(
  881. 'default' => 's.spider_name',
  882. 'reverse' => 's.spider_name DESC',
  883. ),
  884. ),
  885. 'page_hits' => array(
  886. 'header' => array(
  887. 'value' => $txt['spider_stats_page_hits'],
  888. ),
  889. 'data' => array(
  890. 'db' => 'page_hits',
  891. ),
  892. 'sort' => array(
  893. 'default' => 'ss.page_hits',
  894. 'reverse' => 'ss.page_hits DESC',
  895. ),
  896. ),
  897. ),
  898. 'form' => array(
  899. 'href' => $scripturl . '?action=admin;area=sengines;sa=stats',
  900. 'name' => 'spider_stat_list',
  901. ),
  902. 'additional_rows' => array(
  903. array(
  904. 'position' => 'below_table_data',
  905. 'value' => $date_select,
  906. 'style' => 'text-align: right;',
  907. ),
  908. ),
  909. );
  910. createToken('admin-ss');
  911. require_once($sourcedir . '/Subs-List.php');
  912. createList($listOptions);
  913. $context['sub_template'] = 'show_spider_stats';
  914. $context['default_list'] = 'spider_stat_list';
  915. }
  916. /**
  917. * Callback function for createList()
  918. * Get a list of spider stats from the log_spider table
  919. *
  920. * @param type $start
  921. * @param type $items_per_page
  922. * @param type $sort
  923. * @return array
  924. */
  925. function list_getSpiderStats($start, $items_per_page, $sort)
  926. {
  927. global $smcFunc;
  928. $request = $smcFunc['db_query']('', '
  929. SELECT ss.id_spider, ss.stat_date, ss.page_hits, s.spider_name
  930. FROM {db_prefix}log_spider_stats AS ss
  931. INNER JOIN {db_prefix}spiders AS s ON (s.id_spider = ss.id_spider)
  932. ORDER BY ' . $sort . '
  933. LIMIT ' . $start . ', ' . $items_per_page,
  934. array(
  935. )
  936. );
  937. $spider_stats = array();
  938. while ($row = $smcFunc['db_fetch_assoc']($request))
  939. $spider_stats[] = $row;
  940. $smcFunc['db_free_result']($request);
  941. return $spider_stats;
  942. }
  943. /**
  944. * Callback function for createList()
  945. * Get the number of spider stat rows from the log spicer stats table
  946. *
  947. * @return int
  948. */
  949. function list_getNumSpiderStats()
  950. {
  951. global $smcFunc;
  952. $request = $smcFunc['db_query']('', '
  953. SELECT COUNT(*) AS num_stats
  954. FROM {db_prefix}log_spider_stats',
  955. array(
  956. )
  957. );
  958. list ($numStats) = $smcFunc['db_fetch_row']($request);
  959. $smcFunc['db_free_result']($request);
  960. return $numStats;
  961. }
  962. /**
  963. * Recache spider names?
  964. */
  965. function recacheSpiderNames()
  966. {
  967. global $smcFunc;
  968. $request = $smcFunc['db_query']('', '
  969. SELECT id_spider, spider_name
  970. FROM {db_prefix}spiders',
  971. array(
  972. )
  973. );
  974. $spiders = array();
  975. while ($row = $smcFunc['db_fetch_assoc']($request))
  976. $spiders[$row['id_spider']] = $row['spider_name'];
  977. $smcFunc['db_free_result']($request);
  978. updateSettings(array('spider_name_cache' => serialize($spiders)));
  979. }
  980. /**
  981. * Sort the search engine table by user agent name to avoid misidentification of engine.
  982. */
  983. function sortSpiderTable()
  984. {
  985. global $smcFunc;
  986. db_extend('packages');
  987. // Add a sorting column.
  988. $smcFunc['db_add_column']('{db_prefix}spiders', array('name' => 'temp_order', 'size' => 8, 'type' => 'mediumint', 'null' => false));
  989. // Set the contents of this column.
  990. $smcFunc['db_query']('set_spider_order', '
  991. UPDATE {db_prefix}spiders
  992. SET temp_order = LENGTH(user_agent)',
  993. array(
  994. )
  995. );
  996. // Order the table by this column.
  997. $smcFunc['db_query']('alter_table_spiders', '
  998. ALTER TABLE {db_prefix}spiders
  999. ORDER BY temp_order DESC',
  1000. array(
  1001. 'db_error_skip' => true,
  1002. )
  1003. );
  1004. // Remove the sorting column.
  1005. $smcFunc['db_remove_column']('{db_prefix}spiders', 'temp_order');
  1006. }
  1007. ?>