ManageSearchEngines.php 29 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138
  1. <?php
  2. /**
  3. * This file contains all the screens that relate to search engines.
  4. *
  5. * Simple Machines Forum (SMF)
  6. *
  7. * @package SMF
  8. * @author Simple Machines http://www.simplemachines.org
  9. * @copyright 2013 Simple Machines and individual contributors
  10. * @license http://www.simplemachines.org/about/smf/license.php BSD
  11. *
  12. * @version 2.1 Alpha 1
  13. */
  14. if (!defined('SMF'))
  15. die('No direct access...');
  16. /**
  17. * Entry point for this section.
  18. */
  19. function SearchEngines()
  20. {
  21. global $context, $txt, $scripturl, $modSettings;
  22. isAllowedTo('admin_forum');
  23. loadLanguage('Search');
  24. loadTemplate('ManageSearch');
  25. if (!empty($modSettings['spider_mode']))
  26. {
  27. $subActions = array(
  28. 'editspiders' => 'EditSpider',
  29. 'logs' => 'SpiderLogs',
  30. 'settings' => 'ManageSearchEngineSettings',
  31. 'spiders' => 'ViewSpiders',
  32. 'stats' => 'SpiderStats',
  33. );
  34. $default = 'stats';
  35. }
  36. else
  37. {
  38. $subActions = array(
  39. 'settings' => 'ManageSearchEngineSettings',
  40. );
  41. $default = 'settings';
  42. }
  43. call_integration_hook('integrate_manage_search_engines', array(&$subActions));
  44. // Ensure we have a valid subaction.
  45. $context['sub_action'] = isset($_REQUEST['sa']) && isset($subActions[$_REQUEST['sa']]) ? $_REQUEST['sa'] : $default;
  46. $context['page_title'] = $txt['search_engines'];
  47. // Some more tab data.
  48. $context[$context['admin_menu_name']]['tab_data'] = array(
  49. 'title' => $txt['search_engines'],
  50. 'description' => $txt['search_engines_description'],
  51. );
  52. // Call the function!
  53. $subActions[$context['sub_action']]();
  54. }
  55. /**
  56. * This is really just the settings page.
  57. *
  58. * @param bool $return_config = false
  59. */
  60. function ManageSearchEngineSettings($return_config = false)
  61. {
  62. global $context, $txt, $modSettings, $scripturl, $sourcedir, $smcFunc;
  63. $config_vars = array(
  64. // How much detail?
  65. array('select', 'spider_mode', 'subtext' => $txt['spider_mode_note'], array($txt['spider_mode_off'], $txt['spider_mode_standard'], $txt['spider_mode_high'], $txt['spider_mode_vhigh']), 'onchange' => 'disableFields();'),
  66. 'spider_group' => array('select', 'spider_group', 'subtext' => $txt['spider_group_note'], array($txt['spider_group_none'], $txt['membergroups_members'])),
  67. array('select', 'show_spider_online', array($txt['show_spider_online_no'], $txt['show_spider_online_summary'], $txt['show_spider_online_detail'], $txt['show_spider_online_detail_admin'])),
  68. );
  69. // Set up a message.
  70. $context['settings_message'] = '<span class="smalltext">' . sprintf($txt['spider_settings_desc'], $scripturl . '?action=admin;area=logs;sa=pruning;' . $context['session_var'] . '=' . $context['session_id']) . '</span>';
  71. // Do some javascript.
  72. $javascript_function = '
  73. function disableFields()
  74. {
  75. disabledState = document.getElementById(\'spider_mode\').value == 0;';
  76. foreach ($config_vars as $variable)
  77. if ($variable[1] != 'spider_mode')
  78. $javascript_function .= '
  79. if (document.getElementById(\'' . $variable[1] . '\'))
  80. document.getElementById(\'' . $variable[1] . '\').disabled = disabledState;';
  81. $javascript_function .= '
  82. }
  83. disableFields();';
  84. call_integration_hook('integrate_modify_search_engine_settings', array(&$config_vars));
  85. if ($return_config)
  86. return $config_vars;
  87. // We need to load the groups for the spider group thingy.
  88. $request = $smcFunc['db_query']('', '
  89. SELECT id_group, group_name
  90. FROM {db_prefix}membergroups
  91. WHERE id_group != {int:admin_group}
  92. AND id_group != {int:moderator_group}',
  93. array(
  94. 'admin_group' => 1,
  95. 'moderator_group' => 3,
  96. )
  97. );
  98. while ($row = $smcFunc['db_fetch_assoc']($request))
  99. $config_vars['spider_group'][2][$row['id_group']] = $row['group_name'];
  100. $smcFunc['db_free_result']($request);
  101. // Make sure it's valid - note that regular members are given id_group = 1 which is reversed in Load.php - no admins here!
  102. if (isset($_POST['spider_group']) && !isset($config_vars['spider_group'][2][$_POST['spider_group']]))
  103. $_POST['spider_group'] = 0;
  104. // We'll want this for our easy save.
  105. require_once($sourcedir . '/ManageServer.php');
  106. // Setup the template.
  107. $context['page_title'] = $txt['settings'];
  108. $context['sub_template'] = 'show_settings';
  109. // Are we saving them - are we??
  110. if (isset($_GET['save']))
  111. {
  112. checkSession();
  113. call_integration_hook('integrate_save_search_engine_settings');
  114. saveDBSettings($config_vars);
  115. recacheSpiderNames();
  116. redirectexit('action=admin;area=sengines;sa=settings');
  117. }
  118. // Final settings...
  119. $context['post_url'] = $scripturl . '?action=admin;area=sengines;save;sa=settings';
  120. $context['settings_title'] = $txt['settings'];
  121. $context['settings_insert_below'] = '
  122. <script type="text/javascript"><!-- // --><![CDATA[
  123. ' . $javascript_function . '
  124. // ]]></script>';
  125. // Prepare the settings...
  126. prepareDBSettingContext($config_vars);
  127. }
  128. /**
  129. * View a list of all the spiders we know about.
  130. */
  131. function ViewSpiders()
  132. {
  133. global $context, $txt, $sourcedir, $scripturl, $smcFunc;
  134. if (!isset($_SESSION['spider_stat']) || $_SESSION['spider_stat'] < time() - 60)
  135. {
  136. consolidateSpiderStats();
  137. $_SESSION['spider_stat'] = time();
  138. }
  139. // Are we adding a new one?
  140. if (!empty($_POST['addSpider']))
  141. return EditSpider();
  142. // User pressed the 'remove selection button'.
  143. elseif (!empty($_POST['removeSpiders']) && !empty($_POST['remove']) && is_array($_POST['remove']))
  144. {
  145. checkSession();
  146. validateToken('admin-ser');
  147. // Make sure every entry is a proper integer.
  148. foreach ($_POST['remove'] as $index => $spider_id)
  149. $_POST['remove'][(int) $index] = (int) $spider_id;
  150. // Delete them all!
  151. $smcFunc['db_query']('', '
  152. DELETE FROM {db_prefix}spiders
  153. WHERE id_spider IN ({array_int:remove_list})',
  154. array(
  155. 'remove_list' => $_POST['remove'],
  156. )
  157. );
  158. $smcFunc['db_query']('', '
  159. DELETE FROM {db_prefix}log_spider_hits
  160. WHERE id_spider IN ({array_int:remove_list})',
  161. array(
  162. 'remove_list' => $_POST['remove'],
  163. )
  164. );
  165. $smcFunc['db_query']('', '
  166. DELETE FROM {db_prefix}log_spider_stats
  167. WHERE id_spider IN ({array_int:remove_list})',
  168. array(
  169. 'remove_list' => $_POST['remove'],
  170. )
  171. );
  172. cache_put_data('spider_search', null, 300);
  173. recacheSpiderNames();
  174. }
  175. // Get the last seens.
  176. $request = $smcFunc['db_query']('', '
  177. SELECT id_spider, MAX(last_seen) AS last_seen_time
  178. FROM {db_prefix}log_spider_stats
  179. GROUP BY id_spider',
  180. array(
  181. )
  182. );
  183. $context['spider_last_seen'] = array();
  184. while ($row = $smcFunc['db_fetch_assoc']($request))
  185. $context['spider_last_seen'][$row['id_spider']] = $row['last_seen_time'];
  186. $smcFunc['db_free_result']($request);
  187. createToken('admin-ser');
  188. $listOptions = array(
  189. 'id' => 'spider_list',
  190. 'title' => $txt['spiders'],
  191. 'items_per_page' => 20,
  192. 'base_href' => $scripturl . '?action=admin;area=sengines;sa=spiders',
  193. 'default_sort_col' => 'name',
  194. 'get_items' => array(
  195. 'function' => 'list_getSpiders',
  196. ),
  197. 'get_count' => array(
  198. 'function' => 'list_getNumSpiders',
  199. ),
  200. 'no_items_label' => $txt['spiders_no_entries'],
  201. 'columns' => array(
  202. 'name' => array(
  203. 'header' => array(
  204. 'value' => $txt['spider_name'],
  205. ),
  206. 'data' => array(
  207. 'function' => create_function('$rowData', '
  208. global $scripturl;
  209. return sprintf(\'<a href="%1$s?action=admin;area=sengines;sa=editspiders;sid=%2$d">%3$s</a>\', $scripturl, $rowData[\'id_spider\'], htmlspecialchars($rowData[\'spider_name\']));
  210. '),
  211. ),
  212. 'sort' => array(
  213. 'default' => 'spider_name',
  214. 'reverse' => 'spider_name DESC',
  215. ),
  216. ),
  217. 'last_seen' => array(
  218. 'header' => array(
  219. 'value' => $txt['spider_last_seen'],
  220. ),
  221. 'data' => array(
  222. 'function' => create_function('$rowData', '
  223. global $context, $txt;
  224. return isset($context[\'spider_last_seen\'][$rowData[\'id_spider\']]) ? timeformat($context[\'spider_last_seen\'][$rowData[\'id_spider\']]) : $txt[\'spider_last_never\'];
  225. '),
  226. ),
  227. ),
  228. 'user_agent' => array(
  229. 'header' => array(
  230. 'value' => $txt['spider_agent'],
  231. ),
  232. 'data' => array(
  233. 'db_htmlsafe' => 'user_agent',
  234. ),
  235. 'sort' => array(
  236. 'default' => 'user_agent',
  237. 'reverse' => 'user_agent DESC',
  238. ),
  239. ),
  240. 'ip_info' => array(
  241. 'header' => array(
  242. 'value' => $txt['spider_ip_info'],
  243. ),
  244. 'data' => array(
  245. 'db_htmlsafe' => 'ip_info',
  246. 'class' => 'smalltext',
  247. ),
  248. 'sort' => array(
  249. 'default' => 'ip_info',
  250. 'reverse' => 'ip_info DESC',
  251. ),
  252. ),
  253. 'check' => array(
  254. 'header' => array(
  255. 'value' => '<input type="checkbox" onclick="invertAll(this, this.form);" class="input_check" />',
  256. 'class' => 'centercol',
  257. ),
  258. 'data' => array(
  259. 'sprintf' => array(
  260. 'format' => '<input type="checkbox" name="remove[]" value="%1$d" class="input_check" />',
  261. 'params' => array(
  262. 'id_spider' => false,
  263. ),
  264. ),
  265. 'class' => 'centercol',
  266. ),
  267. ),
  268. ),
  269. 'form' => array(
  270. 'href' => $scripturl . '?action=admin;area=sengines;sa=spiders',
  271. 'token' => 'admin-ser',
  272. ),
  273. 'additional_rows' => array(
  274. array(
  275. 'position' => 'bottom_of_list',
  276. 'value' => '
  277. <input type="submit" name="removeSpiders" value="' . $txt['spiders_remove_selected'] . '" onclick="return confirm(\'' . $txt['spider_remove_selected_confirm'] . '\');" class="button_submit" />
  278. <input type="submit" name="addSpider" value="' . $txt['spiders_add'] . '" class="button_submit" />
  279. ',
  280. ),
  281. ),
  282. );
  283. require_once($sourcedir . '/Subs-List.php');
  284. createList($listOptions);
  285. $context['sub_template'] = 'show_list';
  286. $context['default_list'] = 'spider_list';
  287. }
  288. /**
  289. * Callback function for createList()
  290. * @param int $start
  291. * @param int $items_per_page
  292. * @param string sort
  293. */
  294. function list_getSpiders($start, $items_per_page, $sort)
  295. {
  296. global $smcFunc;
  297. $request = $smcFunc['db_query']('', '
  298. SELECT id_spider, spider_name, user_agent, ip_info
  299. FROM {db_prefix}spiders
  300. ORDER BY ' . $sort . '
  301. LIMIT ' . $start . ', ' . $items_per_page,
  302. array(
  303. )
  304. );
  305. $spiders = array();
  306. while ($row = $smcFunc['db_fetch_assoc']($request))
  307. $spiders[$row['id_spider']] = $row;
  308. $smcFunc['db_free_result']($request);
  309. return $spiders;
  310. }
  311. /**
  312. * Callback function for createList()
  313. * @return int
  314. */
  315. function list_getNumSpiders()
  316. {
  317. global $smcFunc;
  318. $request = $smcFunc['db_query']('', '
  319. SELECT COUNT(*) AS num_spiders
  320. FROM {db_prefix}spiders',
  321. array(
  322. )
  323. );
  324. list ($numSpiders) = $smcFunc['db_fetch_row']($request);
  325. $smcFunc['db_free_result']($request);
  326. return $numSpiders;
  327. }
  328. /**
  329. * Here we can add, and edit, spider info!
  330. */
  331. function EditSpider()
  332. {
  333. global $context, $smcFunc, $txt;
  334. // Some standard stuff.
  335. $context['id_spider'] = !empty($_GET['sid']) ? (int) $_GET['sid'] : 0;
  336. $context['page_title'] = $context['id_spider'] ? $txt['spiders_edit'] : $txt['spiders_add'];
  337. $context['sub_template'] = 'spider_edit';
  338. // Are we saving?
  339. if (!empty($_POST['save']))
  340. {
  341. checkSession();
  342. validateToken('admin-ses');
  343. $ips = array();
  344. // Check the IP range is valid.
  345. $ip_sets = explode(',', $_POST['spider_ip']);
  346. foreach ($ip_sets as $set)
  347. {
  348. $test = ip2range(trim($set));
  349. if (!empty($test))
  350. $ips[] = $set;
  351. }
  352. $ips = implode(',', $ips);
  353. // Goes in as it is...
  354. if ($context['id_spider'])
  355. $smcFunc['db_query']('', '
  356. UPDATE {db_prefix}spiders
  357. SET spider_name = {string:spider_name}, user_agent = {string:spider_agent},
  358. ip_info = {string:ip_info}
  359. WHERE id_spider = {int:current_spider}',
  360. array(
  361. 'current_spider' => $context['id_spider'],
  362. 'spider_name' => $_POST['spider_name'],
  363. 'spider_agent' => $_POST['spider_agent'],
  364. 'ip_info' => $ips,
  365. )
  366. );
  367. else
  368. $smcFunc['db_insert']('insert',
  369. '{db_prefix}spiders',
  370. array(
  371. 'spider_name' => 'string', 'user_agent' => 'string', 'ip_info' => 'string',
  372. ),
  373. array(
  374. $_POST['spider_name'], $_POST['spider_agent'], $ips,
  375. ),
  376. array('id_spider')
  377. );
  378. // Order by user agent length.
  379. sortSpiderTable();
  380. cache_put_data('spider_search', null, 300);
  381. recacheSpiderNames();
  382. redirectexit('action=admin;area=sengines;sa=spiders');
  383. }
  384. // The default is new.
  385. $context['spider'] = array(
  386. 'id' => 0,
  387. 'name' => '',
  388. 'agent' => '',
  389. 'ip_info' => '',
  390. );
  391. // An edit?
  392. if ($context['id_spider'])
  393. {
  394. $request = $smcFunc['db_query']('', '
  395. SELECT id_spider, spider_name, user_agent, ip_info
  396. FROM {db_prefix}spiders
  397. WHERE id_spider = {int:current_spider}',
  398. array(
  399. 'current_spider' => $context['id_spider'],
  400. )
  401. );
  402. if ($row = $smcFunc['db_fetch_assoc']($request))
  403. $context['spider'] = array(
  404. 'id' => $row['id_spider'],
  405. 'name' => $row['spider_name'],
  406. 'agent' => $row['user_agent'],
  407. 'ip_info' => $row['ip_info'],
  408. );
  409. $smcFunc['db_free_result']($request);
  410. }
  411. createToken('admin-ses');
  412. }
  413. /**
  414. * Do we think the current user is a spider?
  415. *
  416. * @todo Should this not be... you know... in a different file?
  417. * @return int
  418. */
  419. function SpiderCheck()
  420. {
  421. global $modSettings, $smcFunc;
  422. if (isset($_SESSION['id_robot']))
  423. unset($_SESSION['id_robot']);
  424. $_SESSION['robot_check'] = time();
  425. // We cache the spider data for five minutes if we can.
  426. if (($spider_data = cache_get_data('spider_search', 300)) === null)
  427. {
  428. $request = $smcFunc['db_query']('spider_check', '
  429. SELECT id_spider, user_agent, ip_info
  430. FROM {db_prefix}spiders',
  431. array(
  432. )
  433. );
  434. $spider_data = array();
  435. while ($row = $smcFunc['db_fetch_assoc']($request))
  436. $spider_data[] = $row;
  437. $smcFunc['db_free_result']($request);
  438. cache_put_data('spider_search', $spider_data, 300);
  439. }
  440. if (empty($spider_data))
  441. return false;
  442. // Only do these bits once.
  443. $ci_user_agent = strtolower($_SERVER['HTTP_USER_AGENT']);
  444. // Always attempt IPv6 first.
  445. if (strpos($_SERVER['REMOTE_ADDR'], ':') !== false)
  446. $ip_parts = convertIPv6toInts($_SERVER['REMOTE_ADDR']);
  447. else
  448. preg_match('/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/', $_SERVER['REMOTE_ADDR'], $ip_parts);
  449. foreach ($spider_data as $spider)
  450. {
  451. // User agent is easy.
  452. if (!empty($spider['user_agent']) && strpos($ci_user_agent, strtolower($spider['user_agent'])) !== false)
  453. $_SESSION['id_robot'] = $spider['id_spider'];
  454. // IP stuff is harder.
  455. elseif (!empty($ip_parts))
  456. {
  457. $ips = explode(',', $spider['ip_info']);
  458. foreach ($ips as $ip)
  459. {
  460. $ip = ip2range($ip);
  461. if (!empty($ip))
  462. {
  463. foreach ($ip as $key => $value)
  464. {
  465. if ($value['low'] > $ip_parts[$key + 1] || $value['high'] < $ip_parts[$key + 1])
  466. break;
  467. elseif (($key == 7 && strpos($_SERVER['REMOTE_ADDR'], ':') !== false) || ($key == 3 && strpos($_SERVER['REMOTE_ADDR'], ':') === false))
  468. $_SESSION['id_robot'] = $spider['id_spider'];
  469. }
  470. }
  471. }
  472. }
  473. if (isset($_SESSION['id_robot']))
  474. break;
  475. }
  476. // If this is low server tracking then log the spider here as oppossed to the main logging function.
  477. if (!empty($modSettings['spider_mode']) && $modSettings['spider_mode'] == 1 && !empty($_SESSION['id_robot']))
  478. logSpider();
  479. return !empty($_SESSION['id_robot']) ? $_SESSION['id_robot'] : 0;
  480. }
  481. /**
  482. * Log the spider presence online.
  483. *
  484. * @todo Different file?
  485. */
  486. function logSpider()
  487. {
  488. global $smcFunc, $modSettings, $context;
  489. if (empty($modSettings['spider_mode']) || empty($_SESSION['id_robot']))
  490. return;
  491. // Attempt to update today's entry.
  492. if ($modSettings['spider_mode'] == 1)
  493. {
  494. $date = strftime('%Y-%m-%d', forum_time(false));
  495. $smcFunc['db_query']('', '
  496. UPDATE {db_prefix}log_spider_stats
  497. SET last_seen = {int:current_time}, page_hits = page_hits + 1
  498. WHERE id_spider = {int:current_spider}
  499. AND stat_date = {date:current_date}',
  500. array(
  501. 'current_date' => $date,
  502. 'current_time' => time(),
  503. 'current_spider' => $_SESSION['id_robot'],
  504. )
  505. );
  506. // Nothing updated?
  507. if ($smcFunc['db_affected_rows']() == 0)
  508. {
  509. $smcFunc['db_insert']('ignore',
  510. '{db_prefix}log_spider_stats',
  511. array(
  512. 'id_spider' => 'int', 'last_seen' => 'int', 'stat_date' => 'date', 'page_hits' => 'int',
  513. ),
  514. array(
  515. $_SESSION['id_robot'], time(), $date, 1,
  516. ),
  517. array('id_spider', 'stat_date')
  518. );
  519. }
  520. }
  521. // If we're tracking better stats than track, better stats - we sort out the today thing later.
  522. else
  523. {
  524. if ($modSettings['spider_mode'] > 2)
  525. {
  526. $url = $_GET + array('USER_AGENT' => $_SERVER['HTTP_USER_AGENT']);
  527. unset($url['sesc'], $url[$context['session_var']]);
  528. $url = serialize($url);
  529. }
  530. else
  531. $url = '';
  532. $smcFunc['db_insert']('insert',
  533. '{db_prefix}log_spider_hits',
  534. array('id_spider' => 'int', 'log_time' => 'int', 'url' => 'string'),
  535. array($_SESSION['id_robot'], time(), $url),
  536. array()
  537. );
  538. }
  539. }
  540. /**
  541. * This function takes any unprocessed hits and turns them into stats.
  542. */
  543. function consolidateSpiderStats()
  544. {
  545. global $smcFunc;
  546. $request = $smcFunc['db_query']('consolidate_spider_stats', '
  547. SELECT id_spider, MAX(log_time) AS last_seen, COUNT(*) AS num_hits
  548. FROM {db_prefix}log_spider_hits
  549. WHERE processed = {int:not_processed}
  550. GROUP BY id_spider, MONTH(log_time), DAYOFMONTH(log_time)',
  551. array(
  552. 'not_processed' => 0,
  553. )
  554. );
  555. $spider_hits = array();
  556. while ($row = $smcFunc['db_fetch_assoc']($request))
  557. $spider_hits[] = $row;
  558. $smcFunc['db_free_result']($request);
  559. if (empty($spider_hits))
  560. return;
  561. // Attempt to update the master data.
  562. $stat_inserts = array();
  563. foreach ($spider_hits as $stat)
  564. {
  565. // We assume the max date is within the right day.
  566. $date = strftime('%Y-%m-%d', $stat['last_seen']);
  567. $smcFunc['db_query']('', '
  568. UPDATE {db_prefix}log_spider_stats
  569. SET page_hits = page_hits + ' . $stat['num_hits'] . ',
  570. last_seen = CASE WHEN last_seen > {int:last_seen} THEN last_seen ELSE {int:last_seen} END
  571. WHERE id_spider = {int:current_spider}
  572. AND stat_date = {date:last_seen_date}',
  573. array(
  574. 'last_seen_date' => $date,
  575. 'last_seen' => $stat['last_seen'],
  576. 'current_spider' => $stat['id_spider'],
  577. )
  578. );
  579. if ($smcFunc['db_affected_rows']() == 0)
  580. $stat_inserts[] = array($date, $stat['id_spider'], $stat['num_hits'], $stat['last_seen']);
  581. }
  582. // New stats?
  583. if (!empty($stat_inserts))
  584. $smcFunc['db_insert']('ignore',
  585. '{db_prefix}log_spider_stats',
  586. array('stat_date' => 'date', 'id_spider' => 'int', 'page_hits' => 'int', 'last_seen' => 'int'),
  587. $stat_inserts,
  588. array('stat_date', 'id_spider')
  589. );
  590. // All processed.
  591. $smcFunc['db_query']('', '
  592. UPDATE {db_prefix}log_spider_hits
  593. SET processed = {int:is_processed}
  594. WHERE processed = {int:not_processed}',
  595. array(
  596. 'is_processed' => 1,
  597. 'not_processed' => 0,
  598. )
  599. );
  600. }
  601. /**
  602. * See what spiders have been up to.
  603. */
  604. function SpiderLogs()
  605. {
  606. global $context, $txt, $sourcedir, $scripturl, $smcFunc, $modSettings;
  607. // Load the template and language just incase.
  608. loadLanguage('Search');
  609. loadTemplate('ManageSearch');
  610. // Did they want to delete some entries?
  611. if (!empty($_POST['delete_entries']) && isset($_POST['older']))
  612. {
  613. checkSession();
  614. validateToken('admin-sl');
  615. $deleteTime = time() - (((int) $_POST['older']) * 24 * 60 * 60);
  616. // Delete the entires.
  617. $smcFunc['db_query']('', '
  618. DELETE FROM {db_prefix}log_spider_hits
  619. WHERE log_time < {int:delete_period}',
  620. array(
  621. 'delete_period' => $deleteTime,
  622. )
  623. );
  624. }
  625. $listOptions = array(
  626. 'id' => 'spider_logs',
  627. 'items_per_page' => 20,
  628. 'title' => $txt['spider_logs'],
  629. 'no_items_label' => $txt['spider_logs_empty'],
  630. 'base_href' => $context['admin_area'] == 'sengines' ? $scripturl . '?action=admin;area=sengines;sa=logs' : $scripturl . '?action=admin;area=logs;sa=spiderlog',
  631. 'default_sort_col' => 'log_time',
  632. 'get_items' => array(
  633. 'function' => 'list_getSpiderLogs',
  634. ),
  635. 'get_count' => array(
  636. 'function' => 'list_getNumSpiderLogs',
  637. ),
  638. 'columns' => array(
  639. 'name' => array(
  640. 'header' => array(
  641. 'value' => $txt['spider'],
  642. ),
  643. 'data' => array(
  644. 'db' => 'spider_name',
  645. ),
  646. 'sort' => array(
  647. 'default' => 's.spider_name',
  648. 'reverse' => 's.spider_name DESC',
  649. ),
  650. ),
  651. 'log_time' => array(
  652. 'header' => array(
  653. 'value' => $txt['spider_time'],
  654. ),
  655. 'data' => array(
  656. 'function' => create_function('$rowData', '
  657. return timeformat($rowData[\'log_time\']);
  658. '),
  659. ),
  660. 'sort' => array(
  661. 'default' => 'sl.id_hit DESC',
  662. 'reverse' => 'sl.id_hit',
  663. ),
  664. ),
  665. 'viewing' => array(
  666. 'header' => array(
  667. 'value' => $txt['spider_viewing'],
  668. ),
  669. 'data' => array(
  670. 'db' => 'url',
  671. ),
  672. ),
  673. ),
  674. 'form' => array(
  675. 'token' => 'admin-sl',
  676. 'href' => $scripturl . '?action=admin;area=sengines;sa=logs',
  677. ),
  678. 'additional_rows' => array(
  679. array(
  680. 'position' => 'after_title',
  681. 'value' => $txt['spider_logs_info'],
  682. 'class' => 'windowbg2',
  683. ),
  684. array(
  685. 'position' => 'below_table_data',
  686. 'value' => '<input type="submit" name="removeAll" value="' . $txt['spider_log_empty_log'] . '" onclick="return confirm(\'' . $txt['spider_log_empty_log_confirm'] . '\');" class="button_submit" />',
  687. ),
  688. ),
  689. );
  690. createToken('admin-sl');
  691. require_once($sourcedir . '/Subs-List.php');
  692. createList($listOptions);
  693. // Now determine the actions of the URLs.
  694. if (!empty($context['spider_logs']['rows']))
  695. {
  696. $urls = array();
  697. // Grab the current /url.
  698. foreach ($context['spider_logs']['rows'] as $k => $row)
  699. {
  700. // Feature disabled?
  701. if (empty($row['viewing']['value']) && isset($modSettings['spider_mode']) && $modSettings['spider_mode'] < 3)
  702. $context['spider_logs']['rows'][$k]['viewing']['value'] = '<em>' . $txt['spider_disabled'] . '</em>';
  703. else
  704. $urls[$k] = array($row['viewing']['value'], -1);
  705. }
  706. // Now stick in the new URLs.
  707. require_once($sourcedir . '/Who.php');
  708. $urls = determineActions($urls, 'whospider_');
  709. foreach ($urls as $k => $new_url)
  710. {
  711. $context['spider_logs']['rows'][$k]['viewing']['value'] = $new_url;
  712. }
  713. }
  714. $context['page_title'] = $txt['spider_logs'];
  715. $context['sub_template'] = 'show_spider_logs';
  716. $context['default_list'] = 'spider_logs';
  717. }
  718. /**
  719. * Callback function for createList()
  720. *
  721. * @param int $start
  722. * @param int $items_per_page
  723. * @param string $sort
  724. * @return array
  725. */
  726. function list_getSpiderLogs($start, $items_per_page, $sort)
  727. {
  728. global $smcFunc;
  729. $request = $smcFunc['db_query']('', '
  730. SELECT sl.id_spider, sl.url, sl.log_time, s.spider_name
  731. FROM {db_prefix}log_spider_hits AS sl
  732. INNER JOIN {db_prefix}spiders AS s ON (s.id_spider = sl.id_spider)
  733. ORDER BY ' . $sort . '
  734. LIMIT ' . $start . ', ' . $items_per_page,
  735. array(
  736. )
  737. );
  738. $spider_logs = array();
  739. while ($row = $smcFunc['db_fetch_assoc']($request))
  740. $spider_logs[] = $row;
  741. $smcFunc['db_free_result']($request);
  742. return $spider_logs;
  743. }
  744. /**
  745. * Callback function for createList()
  746. * @return int
  747. */
  748. function list_getNumSpiderLogs()
  749. {
  750. global $smcFunc;
  751. $request = $smcFunc['db_query']('', '
  752. SELECT COUNT(*) AS num_logs
  753. FROM {db_prefix}log_spider_hits',
  754. array(
  755. )
  756. );
  757. list ($numLogs) = $smcFunc['db_fetch_row']($request);
  758. $smcFunc['db_free_result']($request);
  759. return $numLogs;
  760. }
  761. /**
  762. * Show the spider statistics.
  763. */
  764. function SpiderStats()
  765. {
  766. global $context, $txt, $sourcedir, $scripturl, $smcFunc;
  767. // Force an update of the stats every 60 seconds.
  768. if (!isset($_SESSION['spider_stat']) || $_SESSION['spider_stat'] < time() - 60)
  769. {
  770. consolidateSpiderStats();
  771. $_SESSION['spider_stat'] = time();
  772. }
  773. // Are we cleaning up some old stats?
  774. if (!empty($_POST['delete_entries']) && isset($_POST['older']))
  775. {
  776. checkSession();
  777. validateToken('admin-ss');
  778. $deleteTime = time() - (((int) $_POST['older']) * 24 * 60 * 60);
  779. // Delete the entires.
  780. $smcFunc['db_query']('', '
  781. DELETE FROM {db_prefix}log_spider_stats
  782. WHERE last_seen < {int:delete_period}',
  783. array(
  784. 'delete_period' => $deleteTime,
  785. )
  786. );
  787. }
  788. // Get the earliest and latest dates.
  789. $request = $smcFunc['db_query']('', '
  790. SELECT MIN(stat_date) AS first_date, MAX(stat_date) AS last_date
  791. FROM {db_prefix}log_spider_stats',
  792. array(
  793. )
  794. );
  795. list ($min_date, $max_date) = $smcFunc['db_fetch_row']($request);
  796. $smcFunc['db_free_result']($request);
  797. $min_year = (int) substr($min_date, 0, 4);
  798. $max_year = (int) substr($max_date, 0, 4);
  799. $min_month = (int) substr($min_date, 5, 2);
  800. $max_month = (int) substr($max_date, 5, 2);
  801. // Prepare the dates for the drop down.
  802. $date_choices = array();
  803. for ($y = $min_year; $y <= $max_year; $y++)
  804. for ($m = 1; $m <= 12; $m++)
  805. {
  806. // This doesn't count?
  807. if ($y == $min_year && $m < $min_month)
  808. continue;
  809. if ($y == $max_year && $m > $max_month)
  810. break;
  811. $date_choices[$y . $m] = $txt['months_short'][$m] . ' ' . $y;
  812. }
  813. // What are we currently viewing?
  814. $current_date = isset($_REQUEST['new_date']) && isset($date_choices[$_REQUEST['new_date']]) ? $_REQUEST['new_date'] : $max_date;
  815. // Prepare the HTML.
  816. $date_select = '
  817. ' . $txt['spider_stats_select_month'] . ':
  818. <select name="new_date" onchange="document.spider_stat_list.submit();">';
  819. if (empty($date_choices))
  820. $date_select .= '
  821. <option></option>';
  822. else
  823. foreach ($date_choices as $id => $text)
  824. $date_select .= '
  825. <option value="' . $id . '"' . ($current_date == $id ? ' selected="selected"' : '') . '>' . $text . '</option>';
  826. $date_select .= '
  827. </select>
  828. <noscript>
  829. <input type="submit" name="go" value="' . $txt['go'] . '" class="button_submit" />
  830. </noscript>';
  831. // If we manually jumped to a date work out the offset.
  832. if (isset($_REQUEST['new_date']))
  833. {
  834. $date_query = sprintf('%04d-%02d-01', substr($current_date, 0, 4), substr($current_date, 4));
  835. $request = $smcFunc['db_query']('', '
  836. SELECT COUNT(*) AS offset
  837. FROM {db_prefix}log_spider_stats
  838. WHERE stat_date < {date:date_being_viewed}',
  839. array(
  840. 'date_being_viewed' => $date_query,
  841. )
  842. );
  843. list ($_REQUEST['start']) = $smcFunc['db_fetch_row']($request);
  844. $smcFunc['db_free_result']($request);
  845. }
  846. $listOptions = array(
  847. 'id' => 'spider_stat_list',
  848. 'title' => $txt['spider'] . ' ' . $txt['spider_stats'],
  849. 'items_per_page' => 20,
  850. 'base_href' => $scripturl . '?action=admin;area=sengines;sa=stats',
  851. 'default_sort_col' => 'stat_date',
  852. 'get_items' => array(
  853. 'function' => 'list_getSpiderStats',
  854. ),
  855. 'get_count' => array(
  856. 'function' => 'list_getNumSpiderStats',
  857. ),
  858. 'no_items_label' => $txt['spider_stats_no_entries'],
  859. 'columns' => array(
  860. 'stat_date' => array(
  861. 'header' => array(
  862. 'value' => $txt['date'],
  863. ),
  864. 'data' => array(
  865. 'db' => 'stat_date',
  866. ),
  867. 'sort' => array(
  868. 'default' => 'stat_date',
  869. 'reverse' => 'stat_date DESC',
  870. ),
  871. ),
  872. 'name' => array(
  873. 'header' => array(
  874. 'value' => $txt['spider_name'],
  875. ),
  876. 'data' => array(
  877. 'db' => 'spider_name',
  878. ),
  879. 'sort' => array(
  880. 'default' => 's.spider_name',
  881. 'reverse' => 's.spider_name DESC',
  882. ),
  883. ),
  884. 'page_hits' => array(
  885. 'header' => array(
  886. 'value' => $txt['spider_stats_page_hits'],
  887. ),
  888. 'data' => array(
  889. 'db' => 'page_hits',
  890. ),
  891. 'sort' => array(
  892. 'default' => 'ss.page_hits',
  893. 'reverse' => 'ss.page_hits DESC',
  894. ),
  895. ),
  896. ),
  897. 'form' => array(
  898. 'href' => $scripturl . '?action=admin;area=sengines;sa=stats',
  899. 'name' => 'spider_stat_list',
  900. ),
  901. 'additional_rows' => array(
  902. array(
  903. 'position' => 'below_table_data',
  904. 'value' => $date_select,
  905. 'style' => 'text-align: right;',
  906. ),
  907. ),
  908. );
  909. createToken('admin-ss');
  910. require_once($sourcedir . '/Subs-List.php');
  911. createList($listOptions);
  912. $context['sub_template'] = 'show_spider_stats';
  913. $context['default_list'] = 'spider_stat_list';
  914. }
  915. /**
  916. * Callback function for createList()
  917. * Get a list of spider stats from the log_spider table
  918. *
  919. * @param type $start
  920. * @param type $items_per_page
  921. * @param type $sort
  922. * @return array
  923. */
  924. function list_getSpiderStats($start, $items_per_page, $sort)
  925. {
  926. global $smcFunc;
  927. $request = $smcFunc['db_query']('', '
  928. SELECT ss.id_spider, ss.stat_date, ss.page_hits, s.spider_name
  929. FROM {db_prefix}log_spider_stats AS ss
  930. INNER JOIN {db_prefix}spiders AS s ON (s.id_spider = ss.id_spider)
  931. ORDER BY ' . $sort . '
  932. LIMIT ' . $start . ', ' . $items_per_page,
  933. array(
  934. )
  935. );
  936. $spider_stats = array();
  937. while ($row = $smcFunc['db_fetch_assoc']($request))
  938. $spider_stats[] = $row;
  939. $smcFunc['db_free_result']($request);
  940. return $spider_stats;
  941. }
  942. /**
  943. * Callback function for createList()
  944. * Get the number of spider stat rows from the log spicer stats table
  945. *
  946. * @return int
  947. */
  948. function list_getNumSpiderStats()
  949. {
  950. global $smcFunc;
  951. $request = $smcFunc['db_query']('', '
  952. SELECT COUNT(*) AS num_stats
  953. FROM {db_prefix}log_spider_stats',
  954. array(
  955. )
  956. );
  957. list ($numStats) = $smcFunc['db_fetch_row']($request);
  958. $smcFunc['db_free_result']($request);
  959. return $numStats;
  960. }
  961. /**
  962. * Recache spider names?
  963. */
  964. function recacheSpiderNames()
  965. {
  966. global $smcFunc;
  967. $request = $smcFunc['db_query']('', '
  968. SELECT id_spider, spider_name
  969. FROM {db_prefix}spiders',
  970. array(
  971. )
  972. );
  973. $spiders = array();
  974. while ($row = $smcFunc['db_fetch_assoc']($request))
  975. $spiders[$row['id_spider']] = $row['spider_name'];
  976. $smcFunc['db_free_result']($request);
  977. updateSettings(array('spider_name_cache' => serialize($spiders)));
  978. }
  979. /**
  980. * Sort the search engine table by user agent name to avoid misidentification of engine.
  981. */
  982. function sortSpiderTable()
  983. {
  984. global $smcFunc;
  985. db_extend('packages');
  986. // Add a sorting column.
  987. $smcFunc['db_add_column']('{db_prefix}spiders', array('name' => 'temp_order', 'size' => 8, 'type' => 'mediumint', 'null' => false));
  988. // Set the contents of this column.
  989. $smcFunc['db_query']('set_spider_order', '
  990. UPDATE {db_prefix}spiders
  991. SET temp_order = LENGTH(user_agent)',
  992. array(
  993. )
  994. );
  995. // Order the table by this column.
  996. $smcFunc['db_query']('alter_table_spiders', '
  997. ALTER TABLE {db_prefix}spiders
  998. ORDER BY temp_order DESC',
  999. array(
  1000. 'db_error_skip' => true,
  1001. )
  1002. );
  1003. // Remove the sorting column.
  1004. $smcFunc['db_remove_column']('{db_prefix}spiders', 'temp_order');
  1005. }
  1006. ?>