ManageSearchEngines.php 28 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102
  1. <?php
  2. /**
  3. * This file contains all the screens that relate to search engines.
  4. *
  5. * Simple Machines Forum (SMF)
  6. *
  7. * @package SMF
  8. * @author Simple Machines http://www.simplemachines.org
  9. * @copyright 2014 Simple Machines and individual contributors
  10. * @license http://www.simplemachines.org/about/smf/license.php BSD
  11. *
  12. * @version 2.1 Alpha 1
  13. */
  14. if (!defined('SMF'))
  15. die('No direct access...');
  16. /**
  17. * Entry point for this section.
  18. */
  19. function SearchEngines()
  20. {
  21. global $context, $txt, $scripturl, $modSettings;
  22. isAllowedTo('admin_forum');
  23. loadLanguage('Search');
  24. loadTemplate('ManageSearch');
  25. if (!empty($modSettings['spider_mode']))
  26. {
  27. $subActions = array(
  28. 'editspiders' => 'EditSpider',
  29. 'logs' => 'SpiderLogs',
  30. 'settings' => 'ManageSearchEngineSettings',
  31. 'spiders' => 'ViewSpiders',
  32. 'stats' => 'SpiderStats',
  33. );
  34. $default = 'stats';
  35. }
  36. else
  37. {
  38. $subActions = array(
  39. 'settings' => 'ManageSearchEngineSettings',
  40. );
  41. $default = 'settings';
  42. }
  43. call_integration_hook('integrate_manage_search_engines', array(&$subActions));
  44. // Ensure we have a valid subaction.
  45. $context['sub_action'] = isset($_REQUEST['sa']) && isset($subActions[$_REQUEST['sa']]) ? $_REQUEST['sa'] : $default;
  46. $context['page_title'] = $txt['search_engines'];
  47. // Some more tab data.
  48. $context[$context['admin_menu_name']]['tab_data'] = array(
  49. 'title' => $txt['search_engines'],
  50. 'description' => $txt['search_engines_description'],
  51. );
  52. // Call the function!
  53. $subActions[$context['sub_action']]();
  54. }
  55. /**
  56. * This is really just the settings page.
  57. *
  58. * @param bool $return_config = false
  59. */
  60. function ManageSearchEngineSettings($return_config = false)
  61. {
  62. global $context, $txt, $modSettings, $scripturl, $sourcedir, $smcFunc;
  63. $config_vars = array(
  64. // How much detail?
  65. array('select', 'spider_mode', 'subtext' => $txt['spider_mode_note'], array($txt['spider_mode_off'], $txt['spider_mode_standard'], $txt['spider_mode_high'], $txt['spider_mode_vhigh']), 'onchange' => 'disableFields();'),
  66. 'spider_group' => array('select', 'spider_group', 'subtext' => $txt['spider_group_note'], array($txt['spider_group_none'], $txt['membergroups_members'])),
  67. array('select', 'show_spider_online', array($txt['show_spider_online_no'], $txt['show_spider_online_summary'], $txt['show_spider_online_detail'], $txt['show_spider_online_detail_admin'])),
  68. );
  69. // Set up a message.
  70. $context['settings_message'] = '<span class="smalltext">' . sprintf($txt['spider_settings_desc'], $scripturl . '?action=admin;area=logs;sa=pruning;' . $context['session_var'] . '=' . $context['session_id']) . '</span>';
  71. // Do some javascript.
  72. $javascript_function = '
  73. function disableFields()
  74. {
  75. disabledState = document.getElementById(\'spider_mode\').value == 0;';
  76. foreach ($config_vars as $variable)
  77. if ($variable[1] != 'spider_mode')
  78. $javascript_function .= '
  79. if (document.getElementById(\'' . $variable[1] . '\'))
  80. document.getElementById(\'' . $variable[1] . '\').disabled = disabledState;';
  81. $javascript_function .= '
  82. }
  83. disableFields();';
  84. call_integration_hook('integrate_modify_search_engine_settings', array(&$config_vars));
  85. if ($return_config)
  86. return $config_vars;
  87. // We need to load the groups for the spider group thingy.
  88. $request = $smcFunc['db_query']('', '
  89. SELECT id_group, group_name
  90. FROM {db_prefix}membergroups
  91. WHERE id_group != {int:admin_group}
  92. AND id_group != {int:moderator_group}',
  93. array(
  94. 'admin_group' => 1,
  95. 'moderator_group' => 3,
  96. )
  97. );
  98. while ($row = $smcFunc['db_fetch_assoc']($request))
  99. $config_vars['spider_group'][2][$row['id_group']] = $row['group_name'];
  100. $smcFunc['db_free_result']($request);
  101. // Make sure it's valid - note that regular members are given id_group = 1 which is reversed in Load.php - no admins here!
  102. if (isset($_POST['spider_group']) && !isset($config_vars['spider_group'][2][$_POST['spider_group']]))
  103. $_POST['spider_group'] = 0;
  104. // We'll want this for our easy save.
  105. require_once($sourcedir . '/ManageServer.php');
  106. // Setup the template.
  107. $context['page_title'] = $txt['settings'];
  108. $context['sub_template'] = 'show_settings';
  109. // Are we saving them - are we??
  110. if (isset($_GET['save']))
  111. {
  112. checkSession();
  113. call_integration_hook('integrate_save_search_engine_settings');
  114. saveDBSettings($config_vars);
  115. recacheSpiderNames();
  116. $_SESSION['adm-save'] = true;
  117. redirectexit('action=admin;area=sengines;sa=settings');
  118. }
  119. // Final settings...
  120. $context['post_url'] = $scripturl . '?action=admin;area=sengines;save;sa=settings';
  121. $context['settings_title'] = $txt['settings'];
  122. addInlineJavascript($javascript_function, true);
  123. // Prepare the settings...
  124. prepareDBSettingContext($config_vars);
  125. }
  126. /**
  127. * View a list of all the spiders we know about.
  128. */
  129. function ViewSpiders()
  130. {
  131. global $context, $txt, $sourcedir, $scripturl, $smcFunc;
  132. if (!isset($_SESSION['spider_stat']) || $_SESSION['spider_stat'] < time() - 60)
  133. {
  134. consolidateSpiderStats();
  135. $_SESSION['spider_stat'] = time();
  136. }
  137. // Are we adding a new one?
  138. if (!empty($_POST['addSpider']))
  139. return EditSpider();
  140. // User pressed the 'remove selection button'.
  141. elseif (!empty($_POST['removeSpiders']) && !empty($_POST['remove']) && is_array($_POST['remove']))
  142. {
  143. checkSession();
  144. validateToken('admin-ser');
  145. // Make sure every entry is a proper integer.
  146. foreach ($_POST['remove'] as $index => $spider_id)
  147. $_POST['remove'][(int) $index] = (int) $spider_id;
  148. // Delete them all!
  149. $smcFunc['db_query']('', '
  150. DELETE FROM {db_prefix}spiders
  151. WHERE id_spider IN ({array_int:remove_list})',
  152. array(
  153. 'remove_list' => $_POST['remove'],
  154. )
  155. );
  156. $smcFunc['db_query']('', '
  157. DELETE FROM {db_prefix}log_spider_hits
  158. WHERE id_spider IN ({array_int:remove_list})',
  159. array(
  160. 'remove_list' => $_POST['remove'],
  161. )
  162. );
  163. $smcFunc['db_query']('', '
  164. DELETE FROM {db_prefix}log_spider_stats
  165. WHERE id_spider IN ({array_int:remove_list})',
  166. array(
  167. 'remove_list' => $_POST['remove'],
  168. )
  169. );
  170. cache_put_data('spider_search', null, 300);
  171. recacheSpiderNames();
  172. }
  173. // Get the last seens.
  174. $request = $smcFunc['db_query']('', '
  175. SELECT id_spider, MAX(last_seen) AS last_seen_time
  176. FROM {db_prefix}log_spider_stats
  177. GROUP BY id_spider',
  178. array(
  179. )
  180. );
  181. $context['spider_last_seen'] = array();
  182. while ($row = $smcFunc['db_fetch_assoc']($request))
  183. $context['spider_last_seen'][$row['id_spider']] = $row['last_seen_time'];
  184. $smcFunc['db_free_result']($request);
  185. createToken('admin-ser');
  186. $listOptions = array(
  187. 'id' => 'spider_list',
  188. 'title' => $txt['spiders'],
  189. 'items_per_page' => 20,
  190. 'base_href' => $scripturl . '?action=admin;area=sengines;sa=spiders',
  191. 'default_sort_col' => 'name',
  192. 'get_items' => array(
  193. 'function' => 'list_getSpiders',
  194. ),
  195. 'get_count' => array(
  196. 'function' => 'list_getNumSpiders',
  197. ),
  198. 'no_items_label' => $txt['spiders_no_entries'],
  199. 'columns' => array(
  200. 'name' => array(
  201. 'header' => array(
  202. 'value' => $txt['spider_name'],
  203. ),
  204. 'data' => array(
  205. 'function' => create_function('$rowData', '
  206. global $scripturl, $smcFunc;
  207. return sprintf(\'<a href="%1$s?action=admin;area=sengines;sa=editspiders;sid=%2$d">%3$s</a>\', $scripturl, $rowData[\'id_spider\'], $smcFunc[\'htmlspecialchars\']($rowData[\'spider_name\']));
  208. '),
  209. ),
  210. 'sort' => array(
  211. 'default' => 'spider_name',
  212. 'reverse' => 'spider_name DESC',
  213. ),
  214. ),
  215. 'last_seen' => array(
  216. 'header' => array(
  217. 'value' => $txt['spider_last_seen'],
  218. ),
  219. 'data' => array(
  220. 'function' => create_function('$rowData', '
  221. global $context, $txt;
  222. return isset($context[\'spider_last_seen\'][$rowData[\'id_spider\']]) ? timeformat($context[\'spider_last_seen\'][$rowData[\'id_spider\']]) : $txt[\'spider_last_never\'];
  223. '),
  224. ),
  225. ),
  226. 'user_agent' => array(
  227. 'header' => array(
  228. 'value' => $txt['spider_agent'],
  229. ),
  230. 'data' => array(
  231. 'db_htmlsafe' => 'user_agent',
  232. ),
  233. 'sort' => array(
  234. 'default' => 'user_agent',
  235. 'reverse' => 'user_agent DESC',
  236. ),
  237. ),
  238. 'ip_info' => array(
  239. 'header' => array(
  240. 'value' => $txt['spider_ip_info'],
  241. ),
  242. 'data' => array(
  243. 'db_htmlsafe' => 'ip_info',
  244. 'class' => 'smalltext',
  245. ),
  246. 'sort' => array(
  247. 'default' => 'ip_info',
  248. 'reverse' => 'ip_info DESC',
  249. ),
  250. ),
  251. 'check' => array(
  252. 'header' => array(
  253. 'value' => '<input type="checkbox" onclick="invertAll(this, this.form);" class="input_check" />',
  254. 'class' => 'centercol',
  255. ),
  256. 'data' => array(
  257. 'sprintf' => array(
  258. 'format' => '<input type="checkbox" name="remove[]" value="%1$d" class="input_check" />',
  259. 'params' => array(
  260. 'id_spider' => false,
  261. ),
  262. ),
  263. 'class' => 'centercol',
  264. ),
  265. ),
  266. ),
  267. 'form' => array(
  268. 'href' => $scripturl . '?action=admin;area=sengines;sa=spiders',
  269. 'token' => 'admin-ser',
  270. ),
  271. 'additional_rows' => array(
  272. array(
  273. 'position' => 'bottom_of_list',
  274. 'value' => '
  275. <input type="submit" name="removeSpiders" value="' . $txt['spiders_remove_selected'] . '" onclick="return confirm(\'' . $txt['spider_remove_selected_confirm'] . '\');" class="button_submit" />
  276. <input type="submit" name="addSpider" value="' . $txt['spiders_add'] . '" class="button_submit" />
  277. ',
  278. ),
  279. ),
  280. );
  281. require_once($sourcedir . '/Subs-List.php');
  282. createList($listOptions);
  283. $context['sub_template'] = 'show_list';
  284. $context['default_list'] = 'spider_list';
  285. }
  286. /**
  287. * Callback function for createList()
  288. * @param int $start
  289. * @param int $items_per_page
  290. * @param string sort
  291. */
  292. function list_getSpiders($start, $items_per_page, $sort)
  293. {
  294. global $smcFunc;
  295. $request = $smcFunc['db_query']('', '
  296. SELECT id_spider, spider_name, user_agent, ip_info
  297. FROM {db_prefix}spiders
  298. ORDER BY ' . $sort . '
  299. LIMIT ' . $start . ', ' . $items_per_page,
  300. array(
  301. )
  302. );
  303. $spiders = array();
  304. while ($row = $smcFunc['db_fetch_assoc']($request))
  305. $spiders[$row['id_spider']] = $row;
  306. $smcFunc['db_free_result']($request);
  307. return $spiders;
  308. }
  309. /**
  310. * Callback function for createList()
  311. * @return int
  312. */
  313. function list_getNumSpiders()
  314. {
  315. global $smcFunc;
  316. $request = $smcFunc['db_query']('', '
  317. SELECT COUNT(*) AS num_spiders
  318. FROM {db_prefix}spiders',
  319. array(
  320. )
  321. );
  322. list ($numSpiders) = $smcFunc['db_fetch_row']($request);
  323. $smcFunc['db_free_result']($request);
  324. return $numSpiders;
  325. }
  326. /**
  327. * Here we can add, and edit, spider info!
  328. */
  329. function EditSpider()
  330. {
  331. global $context, $smcFunc, $txt;
  332. // Some standard stuff.
  333. $context['id_spider'] = !empty($_GET['sid']) ? (int) $_GET['sid'] : 0;
  334. $context['page_title'] = $context['id_spider'] ? $txt['spiders_edit'] : $txt['spiders_add'];
  335. $context['sub_template'] = 'spider_edit';
  336. // Are we saving?
  337. if (!empty($_POST['save']))
  338. {
  339. checkSession();
  340. validateToken('admin-ses');
  341. $ips = array();
  342. // Check the IP range is valid.
  343. $ip_sets = explode(',', $_POST['spider_ip']);
  344. foreach ($ip_sets as $set)
  345. {
  346. $test = ip2range(trim($set));
  347. if (!empty($test))
  348. $ips[] = $set;
  349. }
  350. $ips = implode(',', $ips);
  351. // Goes in as it is...
  352. if ($context['id_spider'])
  353. $smcFunc['db_query']('', '
  354. UPDATE {db_prefix}spiders
  355. SET spider_name = {string:spider_name}, user_agent = {string:spider_agent},
  356. ip_info = {string:ip_info}
  357. WHERE id_spider = {int:current_spider}',
  358. array(
  359. 'current_spider' => $context['id_spider'],
  360. 'spider_name' => $_POST['spider_name'],
  361. 'spider_agent' => $_POST['spider_agent'],
  362. 'ip_info' => $ips,
  363. )
  364. );
  365. else
  366. $smcFunc['db_insert']('insert',
  367. '{db_prefix}spiders',
  368. array(
  369. 'spider_name' => 'string', 'user_agent' => 'string', 'ip_info' => 'string',
  370. ),
  371. array(
  372. $_POST['spider_name'], $_POST['spider_agent'], $ips,
  373. ),
  374. array('id_spider')
  375. );
  376. cache_put_data('spider_search', null);
  377. recacheSpiderNames();
  378. redirectexit('action=admin;area=sengines;sa=spiders');
  379. }
  380. // The default is new.
  381. $context['spider'] = array(
  382. 'id' => 0,
  383. 'name' => '',
  384. 'agent' => '',
  385. 'ip_info' => '',
  386. );
  387. // An edit?
  388. if ($context['id_spider'])
  389. {
  390. $request = $smcFunc['db_query']('', '
  391. SELECT id_spider, spider_name, user_agent, ip_info
  392. FROM {db_prefix}spiders
  393. WHERE id_spider = {int:current_spider}',
  394. array(
  395. 'current_spider' => $context['id_spider'],
  396. )
  397. );
  398. if ($row = $smcFunc['db_fetch_assoc']($request))
  399. $context['spider'] = array(
  400. 'id' => $row['id_spider'],
  401. 'name' => $row['spider_name'],
  402. 'agent' => $row['user_agent'],
  403. 'ip_info' => $row['ip_info'],
  404. );
  405. $smcFunc['db_free_result']($request);
  406. }
  407. createToken('admin-ses');
  408. }
  409. /**
  410. * Do we think the current user is a spider?
  411. *
  412. * @todo Should this not be... you know... in a different file?
  413. * @return int
  414. */
  415. function SpiderCheck()
  416. {
  417. global $modSettings, $smcFunc;
  418. if (isset($_SESSION['id_robot']))
  419. unset($_SESSION['id_robot']);
  420. $_SESSION['robot_check'] = time();
  421. // We cache the spider data for ten minutes if we can.
  422. if (($spider_data = cache_get_data('spider_search', 600)) === null)
  423. {
  424. $request = $smcFunc['db_query']('', '
  425. SELECT id_spider, user_agent, ip_info
  426. FROM {db_prefix}spiders
  427. ORDER BY LENGTH(user_agent) DESC',
  428. array(
  429. )
  430. );
  431. $spider_data = array();
  432. while ($row = $smcFunc['db_fetch_assoc']($request))
  433. $spider_data[] = $row;
  434. $smcFunc['db_free_result']($request);
  435. cache_put_data('spider_search', $spider_data, 600);
  436. }
  437. if (empty($spider_data))
  438. return false;
  439. // Only do these bits once.
  440. $ci_user_agent = strtolower($_SERVER['HTTP_USER_AGENT']);
  441. // Always attempt IPv6 first.
  442. if (strpos($_SERVER['REMOTE_ADDR'], ':') !== false)
  443. $ip_parts = convertIPv6toInts($_SERVER['REMOTE_ADDR']);
  444. else
  445. preg_match('/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/', $_SERVER['REMOTE_ADDR'], $ip_parts);
  446. foreach ($spider_data as $spider)
  447. {
  448. // User agent is easy.
  449. if (!empty($spider['user_agent']) && strpos($ci_user_agent, strtolower($spider['user_agent'])) !== false)
  450. $_SESSION['id_robot'] = $spider['id_spider'];
  451. // IP stuff is harder.
  452. elseif (!empty($ip_parts))
  453. {
  454. $ips = explode(',', $spider['ip_info']);
  455. foreach ($ips as $ip)
  456. {
  457. $ip = ip2range($ip);
  458. if (!empty($ip))
  459. {
  460. foreach ($ip as $key => $value)
  461. {
  462. if ($value['low'] > $ip_parts[$key + 1] || $value['high'] < $ip_parts[$key + 1])
  463. break;
  464. elseif (($key == 7 && strpos($_SERVER['REMOTE_ADDR'], ':') !== false) || ($key == 3 && strpos($_SERVER['REMOTE_ADDR'], ':') === false))
  465. $_SESSION['id_robot'] = $spider['id_spider'];
  466. }
  467. }
  468. }
  469. }
  470. if (isset($_SESSION['id_robot']))
  471. break;
  472. }
  473. // If this is low server tracking then log the spider here as oppossed to the main logging function.
  474. if (!empty($modSettings['spider_mode']) && $modSettings['spider_mode'] == 1 && !empty($_SESSION['id_robot']))
  475. logSpider();
  476. return !empty($_SESSION['id_robot']) ? $_SESSION['id_robot'] : 0;
  477. }
  478. /**
  479. * Log the spider presence online.
  480. *
  481. * @todo Different file?
  482. */
  483. function logSpider()
  484. {
  485. global $smcFunc, $modSettings, $context;
  486. if (empty($modSettings['spider_mode']) || empty($_SESSION['id_robot']))
  487. return;
  488. // Attempt to update today's entry.
  489. if ($modSettings['spider_mode'] == 1)
  490. {
  491. $date = strftime('%Y-%m-%d', forum_time(false));
  492. $smcFunc['db_query']('', '
  493. UPDATE {db_prefix}log_spider_stats
  494. SET last_seen = {int:current_time}, page_hits = page_hits + 1
  495. WHERE id_spider = {int:current_spider}
  496. AND stat_date = {date:current_date}',
  497. array(
  498. 'current_date' => $date,
  499. 'current_time' => time(),
  500. 'current_spider' => $_SESSION['id_robot'],
  501. )
  502. );
  503. // Nothing updated?
  504. if ($smcFunc['db_affected_rows']() == 0)
  505. {
  506. $smcFunc['db_insert']('ignore',
  507. '{db_prefix}log_spider_stats',
  508. array(
  509. 'id_spider' => 'int', 'last_seen' => 'int', 'stat_date' => 'date', 'page_hits' => 'int',
  510. ),
  511. array(
  512. $_SESSION['id_robot'], time(), $date, 1,
  513. ),
  514. array('id_spider', 'stat_date')
  515. );
  516. }
  517. }
  518. // If we're tracking better stats than track, better stats - we sort out the today thing later.
  519. else
  520. {
  521. if ($modSettings['spider_mode'] > 2)
  522. {
  523. $url = $_GET + array('USER_AGENT' => $_SERVER['HTTP_USER_AGENT']);
  524. unset($url['sesc'], $url[$context['session_var']]);
  525. $url = serialize($url);
  526. }
  527. else
  528. $url = '';
  529. $smcFunc['db_insert']('insert',
  530. '{db_prefix}log_spider_hits',
  531. array('id_spider' => 'int', 'log_time' => 'int', 'url' => 'string'),
  532. array($_SESSION['id_robot'], time(), $url),
  533. array()
  534. );
  535. }
  536. }
  537. /**
  538. * This function takes any unprocessed hits and turns them into stats.
  539. */
  540. function consolidateSpiderStats()
  541. {
  542. global $smcFunc;
  543. $request = $smcFunc['db_query']('consolidate_spider_stats', '
  544. SELECT id_spider, MAX(log_time) AS last_seen, COUNT(*) AS num_hits
  545. FROM {db_prefix}log_spider_hits
  546. WHERE processed = {int:not_processed}
  547. GROUP BY id_spider, MONTH(log_time), DAYOFMONTH(log_time)',
  548. array(
  549. 'not_processed' => 0,
  550. )
  551. );
  552. $spider_hits = array();
  553. while ($row = $smcFunc['db_fetch_assoc']($request))
  554. $spider_hits[] = $row;
  555. $smcFunc['db_free_result']($request);
  556. if (empty($spider_hits))
  557. return;
  558. // Attempt to update the master data.
  559. $stat_inserts = array();
  560. foreach ($spider_hits as $stat)
  561. {
  562. // We assume the max date is within the right day.
  563. $date = strftime('%Y-%m-%d', $stat['last_seen']);
  564. $smcFunc['db_query']('', '
  565. UPDATE {db_prefix}log_spider_stats
  566. SET page_hits = page_hits + ' . $stat['num_hits'] . ',
  567. last_seen = CASE WHEN last_seen > {int:last_seen} THEN last_seen ELSE {int:last_seen} END
  568. WHERE id_spider = {int:current_spider}
  569. AND stat_date = {date:last_seen_date}',
  570. array(
  571. 'last_seen_date' => $date,
  572. 'last_seen' => $stat['last_seen'],
  573. 'current_spider' => $stat['id_spider'],
  574. )
  575. );
  576. if ($smcFunc['db_affected_rows']() == 0)
  577. $stat_inserts[] = array($date, $stat['id_spider'], $stat['num_hits'], $stat['last_seen']);
  578. }
  579. // New stats?
  580. if (!empty($stat_inserts))
  581. $smcFunc['db_insert']('ignore',
  582. '{db_prefix}log_spider_stats',
  583. array('stat_date' => 'date', 'id_spider' => 'int', 'page_hits' => 'int', 'last_seen' => 'int'),
  584. $stat_inserts,
  585. array('stat_date', 'id_spider')
  586. );
  587. // All processed.
  588. $smcFunc['db_query']('', '
  589. UPDATE {db_prefix}log_spider_hits
  590. SET processed = {int:is_processed}
  591. WHERE processed = {int:not_processed}',
  592. array(
  593. 'is_processed' => 1,
  594. 'not_processed' => 0,
  595. )
  596. );
  597. }
  598. /**
  599. * See what spiders have been up to.
  600. */
  601. function SpiderLogs()
  602. {
  603. global $context, $txt, $sourcedir, $scripturl, $smcFunc, $modSettings;
  604. // Load the template and language just incase.
  605. loadLanguage('Search');
  606. loadTemplate('ManageSearch');
  607. // Did they want to delete some entries?
  608. if (!empty($_POST['delete_entries']) && isset($_POST['older']))
  609. {
  610. checkSession();
  611. validateToken('admin-sl');
  612. $deleteTime = time() - (((int) $_POST['older']) * 24 * 60 * 60);
  613. // Delete the entires.
  614. $smcFunc['db_query']('', '
  615. DELETE FROM {db_prefix}log_spider_hits
  616. WHERE log_time < {int:delete_period}',
  617. array(
  618. 'delete_period' => $deleteTime,
  619. )
  620. );
  621. }
  622. $listOptions = array(
  623. 'id' => 'spider_logs',
  624. 'items_per_page' => 20,
  625. 'title' => $txt['spider_logs'],
  626. 'no_items_label' => $txt['spider_logs_empty'],
  627. 'base_href' => $context['admin_area'] == 'sengines' ? $scripturl . '?action=admin;area=sengines;sa=logs' : $scripturl . '?action=admin;area=logs;sa=spiderlog',
  628. 'default_sort_col' => 'log_time',
  629. 'get_items' => array(
  630. 'function' => 'list_getSpiderLogs',
  631. ),
  632. 'get_count' => array(
  633. 'function' => 'list_getNumSpiderLogs',
  634. ),
  635. 'columns' => array(
  636. 'name' => array(
  637. 'header' => array(
  638. 'value' => $txt['spider'],
  639. ),
  640. 'data' => array(
  641. 'db' => 'spider_name',
  642. ),
  643. 'sort' => array(
  644. 'default' => 's.spider_name',
  645. 'reverse' => 's.spider_name DESC',
  646. ),
  647. ),
  648. 'log_time' => array(
  649. 'header' => array(
  650. 'value' => $txt['spider_time'],
  651. ),
  652. 'data' => array(
  653. 'function' => create_function('$rowData', '
  654. return timeformat($rowData[\'log_time\']);
  655. '),
  656. ),
  657. 'sort' => array(
  658. 'default' => 'sl.id_hit DESC',
  659. 'reverse' => 'sl.id_hit',
  660. ),
  661. ),
  662. 'viewing' => array(
  663. 'header' => array(
  664. 'value' => $txt['spider_viewing'],
  665. ),
  666. 'data' => array(
  667. 'db' => 'url',
  668. ),
  669. ),
  670. ),
  671. 'form' => array(
  672. 'token' => 'admin-sl',
  673. 'href' => $scripturl . '?action=admin;area=sengines;sa=logs',
  674. ),
  675. 'additional_rows' => array(
  676. array(
  677. 'position' => 'after_title',
  678. 'value' => $txt['spider_logs_info'],
  679. 'class' => 'windowbg2',
  680. ),
  681. array(
  682. 'position' => 'below_table_data',
  683. 'value' => '<input type="submit" name="removeAll" value="' . $txt['spider_log_empty_log'] . '" onclick="return confirm(\'' . $txt['spider_log_empty_log_confirm'] . '\');" class="button_submit" />',
  684. ),
  685. ),
  686. );
  687. createToken('admin-sl');
  688. require_once($sourcedir . '/Subs-List.php');
  689. createList($listOptions);
  690. // Now determine the actions of the URLs.
  691. if (!empty($context['spider_logs']['rows']))
  692. {
  693. $urls = array();
  694. // Grab the current /url.
  695. foreach ($context['spider_logs']['rows'] as $k => $row)
  696. {
  697. // Feature disabled?
  698. if (empty($row['viewing']['value']) && isset($modSettings['spider_mode']) && $modSettings['spider_mode'] < 3)
  699. $context['spider_logs']['rows'][$k]['viewing']['value'] = '<em>' . $txt['spider_disabled'] . '</em>';
  700. else
  701. $urls[$k] = array($row['viewing']['value'], -1);
  702. }
  703. // Now stick in the new URLs.
  704. require_once($sourcedir . '/Who.php');
  705. $urls = determineActions($urls, 'whospider_');
  706. foreach ($urls as $k => $new_url)
  707. {
  708. $context['spider_logs']['rows'][$k]['viewing']['value'] = $new_url;
  709. }
  710. }
  711. $context['page_title'] = $txt['spider_logs'];
  712. $context['sub_template'] = 'show_spider_logs';
  713. $context['default_list'] = 'spider_logs';
  714. }
  715. /**
  716. * Callback function for createList()
  717. *
  718. * @param int $start
  719. * @param int $items_per_page
  720. * @param string $sort
  721. * @return array
  722. */
  723. function list_getSpiderLogs($start, $items_per_page, $sort)
  724. {
  725. global $smcFunc;
  726. $request = $smcFunc['db_query']('', '
  727. SELECT sl.id_spider, sl.url, sl.log_time, s.spider_name
  728. FROM {db_prefix}log_spider_hits AS sl
  729. INNER JOIN {db_prefix}spiders AS s ON (s.id_spider = sl.id_spider)
  730. ORDER BY ' . $sort . '
  731. LIMIT ' . $start . ', ' . $items_per_page,
  732. array(
  733. )
  734. );
  735. $spider_logs = array();
  736. while ($row = $smcFunc['db_fetch_assoc']($request))
  737. $spider_logs[] = $row;
  738. $smcFunc['db_free_result']($request);
  739. return $spider_logs;
  740. }
  741. /**
  742. * Callback function for createList()
  743. * @return int
  744. */
  745. function list_getNumSpiderLogs()
  746. {
  747. global $smcFunc;
  748. $request = $smcFunc['db_query']('', '
  749. SELECT COUNT(*) AS num_logs
  750. FROM {db_prefix}log_spider_hits',
  751. array(
  752. )
  753. );
  754. list ($numLogs) = $smcFunc['db_fetch_row']($request);
  755. $smcFunc['db_free_result']($request);
  756. return $numLogs;
  757. }
  758. /**
  759. * Show the spider statistics.
  760. */
  761. function SpiderStats()
  762. {
  763. global $context, $txt, $sourcedir, $scripturl, $smcFunc;
  764. // Force an update of the stats every 60 seconds.
  765. if (!isset($_SESSION['spider_stat']) || $_SESSION['spider_stat'] < time() - 60)
  766. {
  767. consolidateSpiderStats();
  768. $_SESSION['spider_stat'] = time();
  769. }
  770. // Are we cleaning up some old stats?
  771. if (!empty($_POST['delete_entries']) && isset($_POST['older']))
  772. {
  773. checkSession();
  774. validateToken('admin-ss');
  775. $deleteTime = time() - (((int) $_POST['older']) * 24 * 60 * 60);
  776. // Delete the entires.
  777. $smcFunc['db_query']('', '
  778. DELETE FROM {db_prefix}log_spider_stats
  779. WHERE last_seen < {int:delete_period}',
  780. array(
  781. 'delete_period' => $deleteTime,
  782. )
  783. );
  784. }
  785. // Get the earliest and latest dates.
  786. $request = $smcFunc['db_query']('', '
  787. SELECT MIN(stat_date) AS first_date, MAX(stat_date) AS last_date
  788. FROM {db_prefix}log_spider_stats',
  789. array(
  790. )
  791. );
  792. list ($min_date, $max_date) = $smcFunc['db_fetch_row']($request);
  793. $smcFunc['db_free_result']($request);
  794. $min_year = (int) substr($min_date, 0, 4);
  795. $max_year = (int) substr($max_date, 0, 4);
  796. $min_month = (int) substr($min_date, 5, 2);
  797. $max_month = (int) substr($max_date, 5, 2);
  798. // Prepare the dates for the drop down.
  799. $date_choices = array();
  800. for ($y = $min_year; $y <= $max_year; $y++)
  801. for ($m = 1; $m <= 12; $m++)
  802. {
  803. // This doesn't count?
  804. if ($y == $min_year && $m < $min_month)
  805. continue;
  806. if ($y == $max_year && $m > $max_month)
  807. break;
  808. $date_choices[$y . $m] = $txt['months_short'][$m] . ' ' . $y;
  809. }
  810. // What are we currently viewing?
  811. $current_date = isset($_REQUEST['new_date']) && isset($date_choices[$_REQUEST['new_date']]) ? $_REQUEST['new_date'] : $max_date;
  812. // Prepare the HTML.
  813. $date_select = '
  814. ' . $txt['spider_stats_select_month'] . ':
  815. <select name="new_date" onchange="document.spider_stat_list.submit();">';
  816. if (empty($date_choices))
  817. $date_select .= '
  818. <option></option>';
  819. else
  820. foreach ($date_choices as $id => $text)
  821. $date_select .= '
  822. <option value="' . $id . '"' . ($current_date == $id ? ' selected="selected"' : '') . '>' . $text . '</option>';
  823. $date_select .= '
  824. </select>
  825. <noscript>
  826. <input type="submit" name="go" value="' . $txt['go'] . '" class="button_submit" />
  827. </noscript>';
  828. // If we manually jumped to a date work out the offset.
  829. if (isset($_REQUEST['new_date']))
  830. {
  831. $date_query = sprintf('%04d-%02d-01', substr($current_date, 0, 4), substr($current_date, 4));
  832. $request = $smcFunc['db_query']('', '
  833. SELECT COUNT(*) AS offset
  834. FROM {db_prefix}log_spider_stats
  835. WHERE stat_date < {date:date_being_viewed}',
  836. array(
  837. 'date_being_viewed' => $date_query,
  838. )
  839. );
  840. list ($_REQUEST['start']) = $smcFunc['db_fetch_row']($request);
  841. $smcFunc['db_free_result']($request);
  842. }
  843. $listOptions = array(
  844. 'id' => 'spider_stat_list',
  845. 'title' => $txt['spider'] . ' ' . $txt['spider_stats'],
  846. 'items_per_page' => 20,
  847. 'base_href' => $scripturl . '?action=admin;area=sengines;sa=stats',
  848. 'default_sort_col' => 'stat_date',
  849. 'get_items' => array(
  850. 'function' => 'list_getSpiderStats',
  851. ),
  852. 'get_count' => array(
  853. 'function' => 'list_getNumSpiderStats',
  854. ),
  855. 'no_items_label' => $txt['spider_stats_no_entries'],
  856. 'columns' => array(
  857. 'stat_date' => array(
  858. 'header' => array(
  859. 'value' => $txt['date'],
  860. ),
  861. 'data' => array(
  862. 'db' => 'stat_date',
  863. ),
  864. 'sort' => array(
  865. 'default' => 'stat_date',
  866. 'reverse' => 'stat_date DESC',
  867. ),
  868. ),
  869. 'name' => array(
  870. 'header' => array(
  871. 'value' => $txt['spider_name'],
  872. ),
  873. 'data' => array(
  874. 'db' => 'spider_name',
  875. ),
  876. 'sort' => array(
  877. 'default' => 's.spider_name',
  878. 'reverse' => 's.spider_name DESC',
  879. ),
  880. ),
  881. 'page_hits' => array(
  882. 'header' => array(
  883. 'value' => $txt['spider_stats_page_hits'],
  884. ),
  885. 'data' => array(
  886. 'db' => 'page_hits',
  887. ),
  888. 'sort' => array(
  889. 'default' => 'ss.page_hits',
  890. 'reverse' => 'ss.page_hits DESC',
  891. ),
  892. ),
  893. ),
  894. 'form' => array(
  895. 'href' => $scripturl . '?action=admin;area=sengines;sa=stats',
  896. 'name' => 'spider_stat_list',
  897. ),
  898. 'additional_rows' => array(
  899. array(
  900. 'position' => 'below_table_data',
  901. 'value' => $date_select,
  902. 'style' => 'text-align: right;',
  903. ),
  904. ),
  905. );
  906. createToken('admin-ss');
  907. require_once($sourcedir . '/Subs-List.php');
  908. createList($listOptions);
  909. $context['sub_template'] = 'show_spider_stats';
  910. $context['default_list'] = 'spider_stat_list';
  911. }
  912. /**
  913. * Callback function for createList()
  914. * Get a list of spider stats from the log_spider table
  915. *
  916. * @param type $start
  917. * @param type $items_per_page
  918. * @param type $sort
  919. * @return array
  920. */
  921. function list_getSpiderStats($start, $items_per_page, $sort)
  922. {
  923. global $smcFunc;
  924. $request = $smcFunc['db_query']('', '
  925. SELECT ss.id_spider, ss.stat_date, ss.page_hits, s.spider_name
  926. FROM {db_prefix}log_spider_stats AS ss
  927. INNER JOIN {db_prefix}spiders AS s ON (s.id_spider = ss.id_spider)
  928. ORDER BY ' . $sort . '
  929. LIMIT ' . $start . ', ' . $items_per_page,
  930. array(
  931. )
  932. );
  933. $spider_stats = array();
  934. while ($row = $smcFunc['db_fetch_assoc']($request))
  935. $spider_stats[] = $row;
  936. $smcFunc['db_free_result']($request);
  937. return $spider_stats;
  938. }
  939. /**
  940. * Callback function for createList()
  941. * Get the number of spider stat rows from the log spicer stats table
  942. *
  943. * @return int
  944. */
  945. function list_getNumSpiderStats()
  946. {
  947. global $smcFunc;
  948. $request = $smcFunc['db_query']('', '
  949. SELECT COUNT(*) AS num_stats
  950. FROM {db_prefix}log_spider_stats',
  951. array(
  952. )
  953. );
  954. list ($numStats) = $smcFunc['db_fetch_row']($request);
  955. $smcFunc['db_free_result']($request);
  956. return $numStats;
  957. }
  958. /**
  959. * Recache spider names?
  960. */
  961. function recacheSpiderNames()
  962. {
  963. global $smcFunc;
  964. $request = $smcFunc['db_query']('', '
  965. SELECT id_spider, spider_name
  966. FROM {db_prefix}spiders',
  967. array(
  968. )
  969. );
  970. $spiders = array();
  971. while ($row = $smcFunc['db_fetch_assoc']($request))
  972. $spiders[$row['id_spider']] = $row['spider_name'];
  973. $smcFunc['db_free_result']($request);
  974. updateSettings(array('spider_name_cache' => serialize($spiders)));
  975. }
  976. ?>