ManageSearchEngines.php 28 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114
  1. <?php
  2. /**
  3. * This file contains all the screens that relate to search engines.
  4. *
  5. * Simple Machines Forum (SMF)
  6. *
  7. * @package SMF
  8. * @author Simple Machines http://www.simplemachines.org
  9. * @copyright 2011 Simple Machines
  10. * @license http://www.simplemachines.org/about/smf/license.php BSD
  11. *
  12. * @version 2.1 Alpha 1
  13. */
  14. if (!defined('SMF'))
  15. die('Hacking attempt...');
  16. /**
  17. * Entry point for this section.
  18. */
  19. function SearchEngines()
  20. {
  21. global $context, $txt, $scripturl;
  22. isAllowedTo('admin_forum');
  23. loadLanguage('Search');
  24. loadTemplate('ManageSearch');
  25. $subActions = array(
  26. 'editspiders' => 'EditSpider',
  27. 'logs' => 'SpiderLogs',
  28. 'settings' => 'ManageSearchEngineSettings',
  29. 'spiders' => 'ViewSpiders',
  30. 'stats' => 'SpiderStats',
  31. );
  32. call_integration_hook('integrate_manage_search_engines', array(&$subActions));
  33. // Ensure we have a valid subaction.
  34. $context['sub_action'] = isset($_REQUEST['sa']) && isset($subActions[$_REQUEST['sa']]) ? $_REQUEST['sa'] : 'stats';
  35. $context['page_title'] = $txt['search_engines'];
  36. // Some more tab data.
  37. $context[$context['admin_menu_name']]['tab_data'] = array(
  38. 'title' => $txt['search_engines'],
  39. 'description' => $txt['search_engines_description'],
  40. );
  41. // Call the function!
  42. $subActions[$context['sub_action']]();
  43. }
  44. /**
  45. * This is really just the settings page.
  46. *
  47. * @param bool $return_config = false
  48. */
  49. function ManageSearchEngineSettings($return_config = false)
  50. {
  51. global $context, $txt, $modSettings, $scripturl, $sourcedir, $smcFunc;
  52. $config_vars = array(
  53. // How much detail?
  54. array('select', 'spider_mode', array($txt['spider_mode_off'], $txt['spider_mode_standard'], $txt['spider_mode_high'], $txt['spider_mode_vhigh']), 'onchange' => 'disableFields();'),
  55. 'spider_group' => array('select', 'spider_group', array($txt['spider_group_none'], $txt['membergroups_members'])),
  56. array('select', 'show_spider_online', array($txt['show_spider_online_no'], $txt['show_spider_online_summary'], $txt['show_spider_online_detail'], $txt['show_spider_online_detail_admin'])),
  57. );
  58. // Set up a message.
  59. $context['settings_message'] = '<span class="smalltext">' . sprintf($txt['spider_settings_desc'], $scripturl . '?action=admin;area=logs;sa=pruning;' . $context['session_var'] . '=' . $context['session_id']) . '</span>';
  60. // Do some javascript.
  61. $javascript_function = '
  62. function disableFields()
  63. {
  64. disabledState = document.getElementById(\'spider_mode\').value == 0;';
  65. foreach ($config_vars as $variable)
  66. if ($variable[1] != 'spider_mode')
  67. $javascript_function .= '
  68. if (document.getElementById(\'' . $variable[1] . '\'))
  69. document.getElementById(\'' . $variable[1] . '\').disabled = disabledState;';
  70. $javascript_function .= '
  71. }
  72. disableFields();';
  73. call_integration_hook('integrate_modify_search_engine_settings', array(&$config_vars));
  74. if ($return_config)
  75. return $config_vars;
  76. // We need to load the groups for the spider group thingy.
  77. $request = $smcFunc['db_query']('', '
  78. SELECT id_group, group_name
  79. FROM {db_prefix}membergroups
  80. WHERE id_group != {int:admin_group}
  81. AND id_group != {int:moderator_group}',
  82. array(
  83. 'admin_group' => 1,
  84. 'moderator_group' => 3,
  85. )
  86. );
  87. while ($row = $smcFunc['db_fetch_assoc']($request))
  88. $config_vars['spider_group'][2][$row['id_group']] = $row['group_name'];
  89. $smcFunc['db_free_result']($request);
  90. // Make sure it's valid - note that regular members are given id_group = 1 which is reversed in Load.php - no admins here!
  91. if (isset($_POST['spider_group']) && !isset($config_vars['spider_group'][2][$_POST['spider_group']]))
  92. $_POST['spider_group'] = 0;
  93. // We'll want this for our easy save.
  94. require_once($sourcedir . '/ManageServer.php');
  95. // Setup the template.
  96. $context['page_title'] = $txt['settings'];
  97. $context['sub_template'] = 'show_settings';
  98. // Are we saving them - are we??
  99. if (isset($_GET['save']))
  100. {
  101. checkSession();
  102. call_integration_hook('integrate_save_search_engine_settings');
  103. saveDBSettings($config_vars);
  104. recacheSpiderNames();
  105. redirectexit('action=admin;area=sengines;sa=settings');
  106. }
  107. // Final settings...
  108. $context['post_url'] = $scripturl . '?action=admin;area=sengines;save;sa=settings';
  109. $context['settings_title'] = $txt['settings'];
  110. $context['settings_insert_below'] = '
  111. <script type="text/javascript"><!-- // --><![CDATA[
  112. ' . $javascript_function . '
  113. // ]]></script>';
  114. // Prepare the settings...
  115. prepareDBSettingContext($config_vars);
  116. }
  117. /**
  118. * View a list of all the spiders we know about.
  119. */
  120. function ViewSpiders()
  121. {
  122. global $context, $txt, $sourcedir, $scripturl, $smcFunc;
  123. if (!isset($_SESSION['spider_stat']) || $_SESSION['spider_stat'] < time() - 60)
  124. {
  125. consolidateSpiderStats();
  126. $_SESSION['spider_stat'] = time();
  127. }
  128. // Are we adding a new one?
  129. if (!empty($_POST['addSpider']))
  130. return EditSpider();
  131. // User pressed the 'remove selection button'.
  132. elseif (!empty($_POST['removeSpiders']) && !empty($_POST['remove']) && is_array($_POST['remove']))
  133. {
  134. checkSession();
  135. validateToken('admin-ser');
  136. // Make sure every entry is a proper integer.
  137. foreach ($_POST['remove'] as $index => $spider_id)
  138. $_POST['remove'][(int) $index] = (int) $spider_id;
  139. // Delete them all!
  140. $smcFunc['db_query']('', '
  141. DELETE FROM {db_prefix}spiders
  142. WHERE id_spider IN ({array_int:remove_list})',
  143. array(
  144. 'remove_list' => $_POST['remove'],
  145. )
  146. );
  147. $smcFunc['db_query']('', '
  148. DELETE FROM {db_prefix}log_spider_hits
  149. WHERE id_spider IN ({array_int:remove_list})',
  150. array(
  151. 'remove_list' => $_POST['remove'],
  152. )
  153. );
  154. $smcFunc['db_query']('', '
  155. DELETE FROM {db_prefix}log_spider_stats
  156. WHERE id_spider IN ({array_int:remove_list})',
  157. array(
  158. 'remove_list' => $_POST['remove'],
  159. )
  160. );
  161. cache_put_data('spider_search', null, 300);
  162. recacheSpiderNames();
  163. }
  164. // Get the last seens.
  165. $request = $smcFunc['db_query']('', '
  166. SELECT id_spider, MAX(last_seen) AS last_seen_time
  167. FROM {db_prefix}log_spider_stats
  168. GROUP BY id_spider',
  169. array(
  170. )
  171. );
  172. $context['spider_last_seen'] = array();
  173. while ($row = $smcFunc['db_fetch_assoc']($request))
  174. $context['spider_last_seen'][$row['id_spider']] = $row['last_seen_time'];
  175. $smcFunc['db_free_result']($request);
  176. createToken('admin-ser');
  177. $listOptions = array(
  178. 'id' => 'spider_list',
  179. 'items_per_page' => 20,
  180. 'base_href' => $scripturl . '?action=admin;area=sengines;sa=spiders',
  181. 'default_sort_col' => 'name',
  182. 'get_items' => array(
  183. 'function' => 'list_getSpiders',
  184. ),
  185. 'get_count' => array(
  186. 'function' => 'list_getNumSpiders',
  187. ),
  188. 'no_items_label' => $txt['spiders_no_entries'],
  189. 'columns' => array(
  190. 'name' => array(
  191. 'header' => array(
  192. 'value' => $txt['spider_name'],
  193. ),
  194. 'data' => array(
  195. 'function' => create_function('$rowData', '
  196. global $scripturl;
  197. return sprintf(\'<a href="%1$s?action=admin;area=sengines;sa=editspiders;sid=%2$d">%3$s</a>\', $scripturl, $rowData[\'id_spider\'], htmlspecialchars($rowData[\'spider_name\']));
  198. '),
  199. ),
  200. 'sort' => array(
  201. 'default' => 'spider_name',
  202. 'reverse' => 'spider_name DESC',
  203. ),
  204. ),
  205. 'last_seen' => array(
  206. 'header' => array(
  207. 'value' => $txt['spider_last_seen'],
  208. ),
  209. 'data' => array(
  210. 'function' => create_function('$rowData', '
  211. global $context, $txt;
  212. return isset($context[\'spider_last_seen\'][$rowData[\'id_spider\']]) ? timeformat($context[\'spider_last_seen\'][$rowData[\'id_spider\']]) : $txt[\'spider_last_never\'];
  213. '),
  214. ),
  215. ),
  216. 'user_agent' => array(
  217. 'header' => array(
  218. 'value' => $txt['spider_agent'],
  219. ),
  220. 'data' => array(
  221. 'db_htmlsafe' => 'user_agent',
  222. ),
  223. 'sort' => array(
  224. 'default' => 'user_agent',
  225. 'reverse' => 'user_agent DESC',
  226. ),
  227. ),
  228. 'ip_info' => array(
  229. 'header' => array(
  230. 'value' => $txt['spider_ip_info'],
  231. ),
  232. 'data' => array(
  233. 'db_htmlsafe' => 'ip_info',
  234. 'class' => 'smalltext',
  235. ),
  236. 'sort' => array(
  237. 'default' => 'ip_info',
  238. 'reverse' => 'ip_info DESC',
  239. ),
  240. ),
  241. 'check' => array(
  242. 'header' => array(
  243. 'value' => '<input type="checkbox" onclick="invertAll(this, this.form);" class="input_check" />',
  244. ),
  245. 'data' => array(
  246. 'sprintf' => array(
  247. 'format' => '<input type="checkbox" name="remove[]" value="%1$d" class="input_check" />',
  248. 'params' => array(
  249. 'id_spider' => false,
  250. ),
  251. ),
  252. 'style' => 'text-align: center',
  253. ),
  254. ),
  255. ),
  256. 'form' => array(
  257. 'href' => $scripturl . '?action=admin;area=sengines;sa=spiders',
  258. 'token' => 'admin-ser',
  259. ),
  260. 'additional_rows' => array(
  261. array(
  262. 'position' => 'below_table_data',
  263. 'value' => '
  264. <input type="submit" name="addSpider" value="' . $txt['spiders_add'] . '" class="button_submit" />
  265. <input type="submit" name="removeSpiders" value="' . $txt['spiders_remove_selected'] . '" onclick="return confirm(\'' . $txt['spider_remove_selected_confirm'] . '\');" class="button_submit" />
  266. ',
  267. 'style' => 'text-align: right;',
  268. ),
  269. ),
  270. );
  271. require_once($sourcedir . '/Subs-List.php');
  272. createList($listOptions);
  273. $context['sub_template'] = 'show_list';
  274. $context['default_list'] = 'spider_list';
  275. }
  276. /**
  277. * Callback function for createList()
  278. * @param int $start
  279. * @param int $items_per_page
  280. * @param string sort
  281. */
  282. function list_getSpiders($start, $items_per_page, $sort)
  283. {
  284. global $smcFunc;
  285. $request = $smcFunc['db_query']('', '
  286. SELECT id_spider, spider_name, user_agent, ip_info
  287. FROM {db_prefix}spiders
  288. ORDER BY ' . $sort . '
  289. LIMIT ' . $start . ', ' . $items_per_page,
  290. array(
  291. )
  292. );
  293. $spiders = array();
  294. while ($row = $smcFunc['db_fetch_assoc']($request))
  295. $spiders[$row['id_spider']] = $row;
  296. $smcFunc['db_free_result']($request);
  297. return $spiders;
  298. }
  299. /**
  300. * Callback function for createList()
  301. * @return int
  302. */
  303. function list_getNumSpiders()
  304. {
  305. global $smcFunc;
  306. $request = $smcFunc['db_query']('', '
  307. SELECT COUNT(*) AS num_spiders
  308. FROM {db_prefix}spiders',
  309. array(
  310. )
  311. );
  312. list ($numSpiders) = $smcFunc['db_fetch_row']($request);
  313. $smcFunc['db_free_result']($request);
  314. return $numSpiders;
  315. }
  316. /**
  317. * Here we can add, and edit, spider info!
  318. */
  319. function EditSpider()
  320. {
  321. global $context, $smcFunc, $txt;
  322. // Some standard stuff.
  323. $context['id_spider'] = !empty($_GET['sid']) ? (int) $_GET['sid'] : 0;
  324. $context['page_title'] = $context['id_spider'] ? $txt['spiders_edit'] : $txt['spiders_add'];
  325. $context['sub_template'] = 'spider_edit';
  326. // Are we saving?
  327. if (!empty($_POST['save']))
  328. {
  329. checkSession();
  330. validateToken('admin-ses');
  331. $ips = array();
  332. // Check the IP range is valid.
  333. $ip_sets = explode(',', $_POST['spider_ip']);
  334. foreach ($ip_sets as $set)
  335. {
  336. $test = ip2range(trim($set));
  337. if (!empty($test))
  338. $ips[] = $set;
  339. }
  340. $ips = implode(',', $ips);
  341. // Goes in as it is...
  342. if ($context['id_spider'])
  343. $smcFunc['db_query']('', '
  344. UPDATE {db_prefix}spiders
  345. SET spider_name = {string:spider_name}, user_agent = {string:spider_agent},
  346. ip_info = {string:ip_info}
  347. WHERE id_spider = {int:current_spider}',
  348. array(
  349. 'current_spider' => $context['id_spider'],
  350. 'spider_name' => $_POST['spider_name'],
  351. 'spider_agent' => $_POST['spider_agent'],
  352. 'ip_info' => $ips,
  353. )
  354. );
  355. else
  356. $smcFunc['db_insert']('insert',
  357. '{db_prefix}spiders',
  358. array(
  359. 'spider_name' => 'string', 'user_agent' => 'string', 'ip_info' => 'string',
  360. ),
  361. array(
  362. $_POST['spider_name'], $_POST['spider_agent'], $ips,
  363. ),
  364. array('id_spider')
  365. );
  366. // Order by user agent length.
  367. sortSpiderTable();
  368. cache_put_data('spider_search', null, 300);
  369. recacheSpiderNames();
  370. redirectexit('action=admin;area=sengines;sa=spiders');
  371. }
  372. // The default is new.
  373. $context['spider'] = array(
  374. 'id' => 0,
  375. 'name' => '',
  376. 'agent' => '',
  377. 'ip_info' => '',
  378. );
  379. // An edit?
  380. if ($context['id_spider'])
  381. {
  382. $request = $smcFunc['db_query']('', '
  383. SELECT id_spider, spider_name, user_agent, ip_info
  384. FROM {db_prefix}spiders
  385. WHERE id_spider = {int:current_spider}',
  386. array(
  387. 'current_spider' => $context['id_spider'],
  388. )
  389. );
  390. if ($row = $smcFunc['db_fetch_assoc']($request))
  391. $context['spider'] = array(
  392. 'id' => $row['id_spider'],
  393. 'name' => $row['spider_name'],
  394. 'agent' => $row['user_agent'],
  395. 'ip_info' => $row['ip_info'],
  396. );
  397. $smcFunc['db_free_result']($request);
  398. }
  399. createToken('admin-ses');
  400. }
  401. /**
  402. * Do we think the current user is a spider?
  403. *
  404. * @todo Should this not be... you know... in a different file?
  405. * @return int
  406. */
  407. function SpiderCheck()
  408. {
  409. global $modSettings, $smcFunc;
  410. if (isset($_SESSION['id_robot']))
  411. unset($_SESSION['id_robot']);
  412. $_SESSION['robot_check'] = time();
  413. // We cache the spider data for five minutes if we can.
  414. if (($spider_data = cache_get_data('spider_search', 300)) === null)
  415. {
  416. $request = $smcFunc['db_query']('spider_check', '
  417. SELECT id_spider, user_agent, ip_info
  418. FROM {db_prefix}spiders',
  419. array(
  420. )
  421. );
  422. $spider_data = array();
  423. while ($row = $smcFunc['db_fetch_assoc']($request))
  424. $spider_data[] = $row;
  425. $smcFunc['db_free_result']($request);
  426. if (!empty($modSettings['cache_enable']))
  427. cache_put_data('spider_search', $spider_data, 300);
  428. }
  429. if (empty($spider_data))
  430. return false;
  431. // Only do these bits once.
  432. $ci_user_agent = strtolower($_SERVER['HTTP_USER_AGENT']);
  433. // Always attempt IPv6 first.
  434. if (empty($ip_parts) && strpos($_SERVER['REMOTE_ADDR'], ':') !== false)
  435. $ip_parts = convertIPv6toInts($_SERVER['REMOTE_ADDR']);
  436. else
  437. preg_match('/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/', $_SERVER['REMOTE_ADDR'], $ip_parts);
  438. foreach ($spider_data as $spider)
  439. {
  440. // User agent is easy.
  441. if (!empty($spider['user_agent']) && strpos($ci_user_agent, strtolower($spider['user_agent'])) !== false)
  442. $_SESSION['id_robot'] = $spider['id_spider'];
  443. // IP stuff is harder.
  444. elseif (!empty($ip_parts))
  445. {
  446. $ips = explode(',', $spider['ip_info']);
  447. foreach ($ips as $ip)
  448. {
  449. $ip = ip2range($ip);
  450. if (!empty($ip))
  451. {
  452. foreach ($ip as $key => $value)
  453. {
  454. if ($value['low'] > $ip_parts[$key + 1] || $value['high'] < $ip_parts[$key + 1])
  455. break;
  456. elseif ($key == 3)
  457. $_SESSION['id_robot'] = $spider['id_spider'];
  458. }
  459. }
  460. }
  461. }
  462. if (isset($_SESSION['id_robot']))
  463. break;
  464. }
  465. // If this is low server tracking then log the spider here as oppossed to the main logging function.
  466. if (!empty($modSettings['spider_mode']) && $modSettings['spider_mode'] == 1 && !empty($_SESSION['id_robot']))
  467. logSpider();
  468. return !empty($_SESSION['id_robot']) ? $_SESSION['id_robot'] : 0;
  469. }
  470. /**
  471. * Log the spider presence online.
  472. *
  473. * @todo Different file?
  474. */
  475. function logSpider()
  476. {
  477. global $smcFunc, $modSettings, $context;
  478. if (empty($modSettings['spider_mode']) || empty($_SESSION['id_robot']))
  479. return;
  480. // Attempt to update today's entry.
  481. if ($modSettings['spider_mode'] == 1)
  482. {
  483. $date = strftime('%Y-%m-%d', forum_time(false));
  484. $smcFunc['db_query']('', '
  485. UPDATE {db_prefix}log_spider_stats
  486. SET last_seen = {int:current_time}, page_hits = page_hits + 1
  487. WHERE id_spider = {int:current_spider}
  488. AND stat_date = {date:current_date}',
  489. array(
  490. 'current_date' => $date,
  491. 'current_time' => time(),
  492. 'current_spider' => $_SESSION['id_robot'],
  493. )
  494. );
  495. // Nothing updated?
  496. if ($smcFunc['db_affected_rows']() == 0)
  497. {
  498. $smcFunc['db_insert']('ignore',
  499. '{db_prefix}log_spider_stats',
  500. array(
  501. 'id_spider' => 'int', 'last_seen' => 'int', 'stat_date' => 'date', 'page_hits' => 'int',
  502. ),
  503. array(
  504. $_SESSION['id_robot'], time(), $date, 1,
  505. ),
  506. array('id_spider', 'stat_date')
  507. );
  508. }
  509. }
  510. // If we're tracking better stats than track, better stats - we sort out the today thing later.
  511. else
  512. {
  513. if ($modSettings['spider_mode'] > 2)
  514. {
  515. $url = $_GET + array('USER_AGENT' => $_SERVER['HTTP_USER_AGENT']);
  516. unset($url['sesc'], $url[$context['session_var']]);
  517. $url = serialize($url);
  518. }
  519. else
  520. $url = '';
  521. $smcFunc['db_insert']('insert',
  522. '{db_prefix}log_spider_hits',
  523. array('id_spider' => 'int', 'log_time' => 'int', 'url' => 'string'),
  524. array($_SESSION['id_robot'], time(), $url),
  525. array()
  526. );
  527. }
  528. }
  529. /**
  530. * This function takes any unprocessed hits and turns them into stats.
  531. */
  532. function consolidateSpiderStats()
  533. {
  534. global $smcFunc;
  535. $request = $smcFunc['db_query']('consolidate_spider_stats', '
  536. SELECT id_spider, MAX(log_time) AS last_seen, COUNT(*) AS num_hits
  537. FROM {db_prefix}log_spider_hits
  538. WHERE processed = {int:not_processed}
  539. GROUP BY id_spider, MONTH(log_time), DAYOFMONTH(log_time)',
  540. array(
  541. 'not_processed' => 0,
  542. )
  543. );
  544. $spider_hits = array();
  545. while ($row = $smcFunc['db_fetch_assoc']($request))
  546. $spider_hits[] = $row;
  547. $smcFunc['db_free_result']($request);
  548. if (empty($spider_hits))
  549. return;
  550. // Attempt to update the master data.
  551. $stat_inserts = array();
  552. foreach ($spider_hits as $stat)
  553. {
  554. // We assume the max date is within the right day.
  555. $date = strftime('%Y-%m-%d', $stat['last_seen']);
  556. $smcFunc['db_query']('', '
  557. UPDATE {db_prefix}log_spider_stats
  558. SET page_hits = page_hits + ' . $stat['num_hits'] . ',
  559. last_seen = CASE WHEN last_seen > {int:last_seen} THEN last_seen ELSE {int:last_seen} END
  560. WHERE id_spider = {int:current_spider}
  561. AND stat_date = {date:last_seen_date}',
  562. array(
  563. 'last_seen_date' => $date,
  564. 'last_seen' => $stat['last_seen'],
  565. 'current_spider' => $stat['id_spider'],
  566. )
  567. );
  568. if ($smcFunc['db_affected_rows']() == 0)
  569. $stat_inserts[] = array($date, $stat['id_spider'], $stat['num_hits'], $stat['last_seen']);
  570. }
  571. // New stats?
  572. if (!empty($stat_inserts))
  573. $smcFunc['db_insert']('ignore',
  574. '{db_prefix}log_spider_stats',
  575. array('stat_date' => 'date', 'id_spider' => 'int', 'page_hits' => 'int', 'last_seen' => 'int'),
  576. $stat_inserts,
  577. array('stat_date', 'id_spider')
  578. );
  579. // All processed.
  580. $smcFunc['db_query']('', '
  581. UPDATE {db_prefix}log_spider_hits
  582. SET processed = {int:is_processed}
  583. WHERE processed = {int:not_processed}',
  584. array(
  585. 'is_processed' => 1,
  586. 'not_processed' => 0,
  587. )
  588. );
  589. }
  590. /**
  591. * See what spiders have been up to.
  592. */
  593. function SpiderLogs()
  594. {
  595. global $context, $txt, $sourcedir, $scripturl, $smcFunc, $modSettings;
  596. // Load the template and language just incase.
  597. loadLanguage('Search');
  598. loadTemplate('ManageSearch');
  599. // Did they want to delete some entries?
  600. if (!empty($_POST['delete_entries']) && isset($_POST['older']))
  601. {
  602. checkSession();
  603. validateToken('admin-sl');
  604. $deleteTime = time() - (((int) $_POST['older']) * 24 * 60 * 60);
  605. // Delete the entires.
  606. $smcFunc['db_query']('', '
  607. DELETE FROM {db_prefix}log_spider_hits
  608. WHERE log_time < {int:delete_period}',
  609. array(
  610. 'delete_period' => $deleteTime,
  611. )
  612. );
  613. }
  614. $listOptions = array(
  615. 'id' => 'spider_logs',
  616. 'items_per_page' => 20,
  617. 'title' => $txt['spider_logs'],
  618. 'no_items_label' => $txt['spider_logs_empty'],
  619. 'base_href' => $context['admin_area'] == 'sengines' ? $scripturl . '?action=admin;area=sengines;sa=logs' : $scripturl . '?action=admin;area=logs;sa=spiderlog',
  620. 'default_sort_col' => 'log_time',
  621. 'get_items' => array(
  622. 'function' => 'list_getSpiderLogs',
  623. ),
  624. 'get_count' => array(
  625. 'function' => 'list_getNumSpiderLogs',
  626. ),
  627. 'columns' => array(
  628. 'name' => array(
  629. 'header' => array(
  630. 'value' => $txt['spider'],
  631. ),
  632. 'data' => array(
  633. 'db' => 'spider_name',
  634. ),
  635. 'sort' => array(
  636. 'default' => 's.spider_name',
  637. 'reverse' => 's.spider_name DESC',
  638. ),
  639. ),
  640. 'log_time' => array(
  641. 'header' => array(
  642. 'value' => $txt['spider_time'],
  643. ),
  644. 'data' => array(
  645. 'function' => create_function('$rowData', '
  646. return timeformat($rowData[\'log_time\']);
  647. '),
  648. ),
  649. 'sort' => array(
  650. 'default' => 'sl.id_hit DESC',
  651. 'reverse' => 'sl.id_hit',
  652. ),
  653. ),
  654. 'viewing' => array(
  655. 'header' => array(
  656. 'value' => $txt['spider_viewing'],
  657. ),
  658. 'data' => array(
  659. 'db' => 'url',
  660. ),
  661. ),
  662. ),
  663. 'form' => array(
  664. 'token' => 'admin-sl',
  665. ),
  666. 'additional_rows' => array(
  667. array(
  668. 'position' => 'after_title',
  669. 'value' => $txt['spider_logs_info'],
  670. 'class' => 'smalltext',
  671. ),
  672. ),
  673. );
  674. createToken('admin-sl');
  675. require_once($sourcedir . '/Subs-List.php');
  676. createList($listOptions);
  677. // Now determine the actions of the URLs.
  678. if (!empty($context['spider_logs']['rows']))
  679. {
  680. $urls = array();
  681. // Grab the current /url.
  682. foreach ($context['spider_logs']['rows'] as $k => $row)
  683. {
  684. // Feature disabled?
  685. if (empty($row['viewing']['value']) && isset($modSettings['spider_mode']) && $modSettings['spider_mode'] < 3)
  686. $context['spider_logs']['rows'][$k]['viewing']['value'] = '<em>' . $txt['spider_disabled'] . '</em>';
  687. else
  688. $urls[$k] = array($row['viewing']['value'], -1);
  689. }
  690. // Now stick in the new URLs.
  691. require_once($sourcedir . '/Who.php');
  692. $urls = determineActions($urls, 'whospider_');
  693. foreach ($urls as $k => $new_url)
  694. {
  695. $context['spider_logs']['rows'][$k]['viewing']['value'] = $new_url;
  696. }
  697. }
  698. $context['page_title'] = $txt['spider_logs'];
  699. $context['sub_template'] = 'show_spider_logs';
  700. $context['default_list'] = 'spider_logs';
  701. }
  702. /**
  703. * Callback function for createList()
  704. *
  705. * @param int $start
  706. * @param int $items_per_page
  707. * @param string $sort
  708. * @return array
  709. */
  710. function list_getSpiderLogs($start, $items_per_page, $sort)
  711. {
  712. global $smcFunc;
  713. $request = $smcFunc['db_query']('', '
  714. SELECT sl.id_spider, sl.url, sl.log_time, s.spider_name
  715. FROM {db_prefix}log_spider_hits AS sl
  716. INNER JOIN {db_prefix}spiders AS s ON (s.id_spider = sl.id_spider)
  717. ORDER BY ' . $sort . '
  718. LIMIT ' . $start . ', ' . $items_per_page,
  719. array(
  720. )
  721. );
  722. $spider_logs = array();
  723. while ($row = $smcFunc['db_fetch_assoc']($request))
  724. $spider_logs[] = $row;
  725. $smcFunc['db_free_result']($request);
  726. return $spider_logs;
  727. }
  728. /**
  729. * Callback function for createList()
  730. * @return int
  731. */
  732. function list_getNumSpiderLogs()
  733. {
  734. global $smcFunc;
  735. $request = $smcFunc['db_query']('', '
  736. SELECT COUNT(*) AS num_logs
  737. FROM {db_prefix}log_spider_hits',
  738. array(
  739. )
  740. );
  741. list ($numLogs) = $smcFunc['db_fetch_row']($request);
  742. $smcFunc['db_free_result']($request);
  743. return $numLogs;
  744. }
  745. /**
  746. * Show the spider statistics.
  747. */
  748. function SpiderStats()
  749. {
  750. global $context, $txt, $sourcedir, $scripturl, $smcFunc;
  751. // Force an update of the stats every 60 seconds.
  752. if (!isset($_SESSION['spider_stat']) || $_SESSION['spider_stat'] < time() - 60)
  753. {
  754. consolidateSpiderStats();
  755. $_SESSION['spider_stat'] = time();
  756. }
  757. // Are we cleaning up some old stats?
  758. if (!empty($_POST['delete_entries']) && isset($_POST['older']))
  759. {
  760. checkSession();
  761. validateToken('admin-ss');
  762. $deleteTime = time() - (((int) $_POST['older']) * 24 * 60 * 60);
  763. // Delete the entires.
  764. $smcFunc['db_query']('', '
  765. DELETE FROM {db_prefix}log_spider_stats
  766. WHERE last_seen < {int:delete_period}',
  767. array(
  768. 'delete_period' => $deleteTime,
  769. )
  770. );
  771. }
  772. // Get the earliest and latest dates.
  773. $request = $smcFunc['db_query']('', '
  774. SELECT MIN(stat_date) AS first_date, MAX(stat_date) AS last_date
  775. FROM {db_prefix}log_spider_stats',
  776. array(
  777. )
  778. );
  779. list ($min_date, $max_date) = $smcFunc['db_fetch_row']($request);
  780. $smcFunc['db_free_result']($request);
  781. $min_year = (int) substr($min_date, 0, 4);
  782. $max_year = (int) substr($max_date, 0, 4);
  783. $min_month = (int) substr($min_date, 5, 2);
  784. $max_month = (int) substr($max_date, 5, 2);
  785. // Prepare the dates for the drop down.
  786. $date_choices = array();
  787. for ($y = $min_year; $y <= $max_year; $y++)
  788. for ($m = 1; $m <= 12; $m++)
  789. {
  790. // This doesn't count?
  791. if ($y == $min_year && $m < $min_month)
  792. continue;
  793. if ($y == $max_year && $m > $max_month)
  794. break;
  795. $date_choices[$y . $m] = $txt['months_short'][$m] . ' ' . $y;
  796. }
  797. // What are we currently viewing?
  798. $current_date = isset($_REQUEST['new_date']) && isset($date_choices[$_REQUEST['new_date']]) ? $_REQUEST['new_date'] : $max_date;
  799. // Prepare the HTML.
  800. $date_select = '
  801. ' . $txt['spider_stats_select_month'] . ':
  802. <select name="new_date" onchange="document.spider_stat_list.submit();">';
  803. if (empty($date_choices))
  804. $date_select .= '
  805. <option></option>';
  806. else
  807. foreach ($date_choices as $id => $text)
  808. $date_select .= '
  809. <option value="' . $id . '"' . ($current_date == $id ? ' selected="selected"' : '') . '>' . $text . '</option>';
  810. $date_select .= '
  811. </select>
  812. <noscript>
  813. <input type="submit" name="go" value="' . $txt['go'] . '" class="button_submit" />
  814. </noscript>';
  815. // If we manually jumped to a date work out the offset.
  816. if (isset($_REQUEST['new_date']))
  817. {
  818. $date_query = sprintf('%04d-%02d-01', substr($current_date, 0, 4), substr($current_date, 4));
  819. $request = $smcFunc['db_query']('', '
  820. SELECT COUNT(*) AS offset
  821. FROM {db_prefix}log_spider_stats
  822. WHERE stat_date < {date:date_being_viewed}',
  823. array(
  824. 'date_being_viewed' => $date_query,
  825. )
  826. );
  827. list ($_REQUEST['start']) = $smcFunc['db_fetch_row']($request);
  828. $smcFunc['db_free_result']($request);
  829. }
  830. $listOptions = array(
  831. 'id' => 'spider_stat_list',
  832. 'items_per_page' => 20,
  833. 'base_href' => $scripturl . '?action=admin;area=sengines;sa=stats',
  834. 'default_sort_col' => 'stat_date',
  835. 'get_items' => array(
  836. 'function' => 'list_getSpiderStats',
  837. ),
  838. 'get_count' => array(
  839. 'function' => 'list_getNumSpiderStats',
  840. ),
  841. 'no_items_label' => $txt['spider_stats_no_entries'],
  842. 'columns' => array(
  843. 'stat_date' => array(
  844. 'header' => array(
  845. 'value' => $txt['date'],
  846. ),
  847. 'data' => array(
  848. 'db' => 'stat_date',
  849. ),
  850. 'sort' => array(
  851. 'default' => 'stat_date',
  852. 'reverse' => 'stat_date DESC',
  853. ),
  854. ),
  855. 'name' => array(
  856. 'header' => array(
  857. 'value' => $txt['spider_name'],
  858. ),
  859. 'data' => array(
  860. 'db' => 'spider_name',
  861. ),
  862. 'sort' => array(
  863. 'default' => 's.spider_name',
  864. 'reverse' => 's.spider_name DESC',
  865. ),
  866. ),
  867. 'page_hits' => array(
  868. 'header' => array(
  869. 'value' => $txt['spider_stats_page_hits'],
  870. ),
  871. 'data' => array(
  872. 'db' => 'page_hits',
  873. ),
  874. 'sort' => array(
  875. 'default' => 'ss.page_hits',
  876. 'reverse' => 'ss.page_hits DESC',
  877. ),
  878. ),
  879. ),
  880. 'form' => array(
  881. 'href' => $scripturl . '?action=admin;area=sengines;sa=stats',
  882. 'name' => 'spider_stat_list',
  883. ),
  884. 'additional_rows' => array(
  885. array(
  886. 'position' => 'below_table_data',
  887. 'value' => $date_select,
  888. 'style' => 'text-align: right;',
  889. ),
  890. ),
  891. );
  892. createToken('admin-ss');
  893. require_once($sourcedir . '/Subs-List.php');
  894. createList($listOptions);
  895. $context['sub_template'] = 'show_spider_stats';
  896. $context['default_list'] = 'spider_stat_list';
  897. }
  898. /**
  899. * Callback function for createList()
  900. * @return array
  901. */
  902. function list_getSpiderStats($start, $items_per_page, $sort)
  903. {
  904. global $smcFunc;
  905. $request = $smcFunc['db_query']('', '
  906. SELECT ss.id_spider, ss.stat_date, ss.page_hits, s.spider_name
  907. FROM {db_prefix}log_spider_stats AS ss
  908. INNER JOIN {db_prefix}spiders AS s ON (s.id_spider = ss.id_spider)
  909. ORDER BY ' . $sort . '
  910. LIMIT ' . $start . ', ' . $items_per_page,
  911. array(
  912. )
  913. );
  914. $spider_stats = array();
  915. while ($row = $smcFunc['db_fetch_assoc']($request))
  916. $spider_stats[] = $row;
  917. $smcFunc['db_free_result']($request);
  918. return $spider_stats;
  919. }
  920. /**
  921. * Callback function for createList()
  922. * @return int
  923. */
  924. function list_getNumSpiderStats()
  925. {
  926. global $smcFunc;
  927. $request = $smcFunc['db_query']('', '
  928. SELECT COUNT(*) AS num_stats
  929. FROM {db_prefix}log_spider_stats',
  930. array(
  931. )
  932. );
  933. list ($numStats) = $smcFunc['db_fetch_row']($request);
  934. $smcFunc['db_free_result']($request);
  935. return $numStats;
  936. }
  937. /**
  938. * Recache spider names?
  939. */
  940. function recacheSpiderNames()
  941. {
  942. global $smcFunc;
  943. $request = $smcFunc['db_query']('', '
  944. SELECT id_spider, spider_name
  945. FROM {db_prefix}spiders',
  946. array(
  947. )
  948. );
  949. $spiders = array();
  950. while ($row = $smcFunc['db_fetch_assoc']($request))
  951. $spiders[$row['id_spider']] = $row['spider_name'];
  952. $smcFunc['db_free_result']($request);
  953. updateSettings(array('spider_name_cache' => serialize($spiders)));
  954. }
  955. /**
  956. * Sort the search engine table by user agent name to avoid misidentification of engine.
  957. */
  958. function sortSpiderTable()
  959. {
  960. global $smcFunc;
  961. db_extend('packages');
  962. // Add a sorting column.
  963. $smcFunc['db_add_column']('{db_prefix}spiders', array('name' => 'temp_order', 'size' => 8, 'type' => 'mediumint', 'null' => false));
  964. // Set the contents of this column.
  965. $smcFunc['db_query']('set_spider_order', '
  966. UPDATE {db_prefix}spiders
  967. SET temp_order = LENGTH(user_agent)',
  968. array(
  969. )
  970. );
  971. // Order the table by this column.
  972. $smcFunc['db_query']('alter_table_spiders', '
  973. ALTER TABLE {db_prefix}spiders
  974. ORDER BY temp_order DESC',
  975. array(
  976. 'db_error_skip' => true,
  977. )
  978. );
  979. // Remove the sorting column.
  980. $smcFunc['db_remove_column']('{db_prefix}spiders', 'temp_order');
  981. }
  982. ?>