ManageSearchEngines.php 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119
  1. <?php
  2. /**
  3. * This file contains all the screens that relate to search engines.
  4. *
  5. * Simple Machines Forum (SMF)
  6. *
  7. * @package SMF
  8. * @author Simple Machines http://www.simplemachines.org
  9. * @copyright 2012 Simple Machines
  10. * @license http://www.simplemachines.org/about/smf/license.php BSD
  11. *
  12. * @version 2.1 Alpha 1
  13. */
  14. if (!defined('SMF'))
  15. die('Hacking attempt...');
  16. /**
  17. * Entry point for this section.
  18. */
  19. function SearchEngines()
  20. {
  21. global $context, $txt, $scripturl;
  22. isAllowedTo('admin_forum');
  23. loadLanguage('Search');
  24. loadTemplate('ManageSearch');
  25. $subActions = array(
  26. 'editspiders' => 'EditSpider',
  27. 'logs' => 'SpiderLogs',
  28. 'settings' => 'ManageSearchEngineSettings',
  29. 'spiders' => 'ViewSpiders',
  30. 'stats' => 'SpiderStats',
  31. );
  32. call_integration_hook('integrate_manage_search_engines', array(&$subActions));
  33. // Ensure we have a valid subaction.
  34. $context['sub_action'] = isset($_REQUEST['sa']) && isset($subActions[$_REQUEST['sa']]) ? $_REQUEST['sa'] : 'stats';
  35. $context['page_title'] = $txt['search_engines'];
  36. // Some more tab data.
  37. $context[$context['admin_menu_name']]['tab_data'] = array(
  38. 'title' => $txt['search_engines'],
  39. 'description' => $txt['search_engines_description'],
  40. );
  41. // Call the function!
  42. $subActions[$context['sub_action']]();
  43. }
  44. /**
  45. * This is really just the settings page.
  46. *
  47. * @param bool $return_config = false
  48. */
  49. function ManageSearchEngineSettings($return_config = false)
  50. {
  51. global $context, $txt, $modSettings, $scripturl, $sourcedir, $smcFunc;
  52. $config_vars = array(
  53. // How much detail?
  54. array('select', 'spider_mode', 'subtext' => $txt['spider_mode_note'], array($txt['spider_mode_off'], $txt['spider_mode_standard'], $txt['spider_mode_high'], $txt['spider_mode_vhigh']), 'onchange' => 'disableFields();'),
  55. 'spider_group' => array('select', 'spider_group', 'subtext' => $txt['spider_group_note'], array($txt['spider_group_none'], $txt['membergroups_members'])),
  56. array('select', 'show_spider_online', array($txt['show_spider_online_no'], $txt['show_spider_online_summary'], $txt['show_spider_online_detail'], $txt['show_spider_online_detail_admin'])),
  57. );
  58. // Set up a message.
  59. $context['settings_message'] = '<span class="smalltext">' . sprintf($txt['spider_settings_desc'], $scripturl . '?action=admin;area=logs;sa=pruning;' . $context['session_var'] . '=' . $context['session_id']) . '</span>';
  60. // Do some javascript.
  61. $javascript_function = '
  62. function disableFields()
  63. {
  64. disabledState = document.getElementById(\'spider_mode\').value == 0;';
  65. foreach ($config_vars as $variable)
  66. if ($variable[1] != 'spider_mode')
  67. $javascript_function .= '
  68. if (document.getElementById(\'' . $variable[1] . '\'))
  69. document.getElementById(\'' . $variable[1] . '\').disabled = disabledState;';
  70. $javascript_function .= '
  71. }
  72. disableFields();';
  73. call_integration_hook('integrate_modify_search_engine_settings', array(&$config_vars));
  74. if ($return_config)
  75. return $config_vars;
  76. // We need to load the groups for the spider group thingy.
  77. $request = $smcFunc['db_query']('', '
  78. SELECT id_group, group_name
  79. FROM {db_prefix}membergroups
  80. WHERE id_group != {int:admin_group}
  81. AND id_group != {int:moderator_group}',
  82. array(
  83. 'admin_group' => 1,
  84. 'moderator_group' => 3,
  85. )
  86. );
  87. while ($row = $smcFunc['db_fetch_assoc']($request))
  88. $config_vars['spider_group'][2][$row['id_group']] = $row['group_name'];
  89. $smcFunc['db_free_result']($request);
  90. // Make sure it's valid - note that regular members are given id_group = 1 which is reversed in Load.php - no admins here!
  91. if (isset($_POST['spider_group']) && !isset($config_vars['spider_group'][2][$_POST['spider_group']]))
  92. $_POST['spider_group'] = 0;
  93. // We'll want this for our easy save.
  94. require_once($sourcedir . '/ManageServer.php');
  95. // Setup the template.
  96. $context['page_title'] = $txt['settings'];
  97. $context['sub_template'] = 'show_settings';
  98. // Are we saving them - are we??
  99. if (isset($_GET['save']))
  100. {
  101. checkSession();
  102. call_integration_hook('integrate_save_search_engine_settings');
  103. saveDBSettings($config_vars);
  104. recacheSpiderNames();
  105. redirectexit('action=admin;area=sengines;sa=settings');
  106. }
  107. // Final settings...
  108. $context['post_url'] = $scripturl . '?action=admin;area=sengines;save;sa=settings';
  109. $context['settings_title'] = $txt['settings'];
  110. $context['settings_insert_below'] = '
  111. <script type="text/javascript"><!-- // --><![CDATA[
  112. ' . $javascript_function . '
  113. // ]]></script>';
  114. // Prepare the settings...
  115. prepareDBSettingContext($config_vars);
  116. }
  117. /**
  118. * View a list of all the spiders we know about.
  119. */
  120. function ViewSpiders()
  121. {
  122. global $context, $txt, $sourcedir, $scripturl, $smcFunc;
  123. if (!isset($_SESSION['spider_stat']) || $_SESSION['spider_stat'] < time() - 60)
  124. {
  125. consolidateSpiderStats();
  126. $_SESSION['spider_stat'] = time();
  127. }
  128. // Are we adding a new one?
  129. if (!empty($_POST['addSpider']))
  130. return EditSpider();
  131. // User pressed the 'remove selection button'.
  132. elseif (!empty($_POST['removeSpiders']) && !empty($_POST['remove']) && is_array($_POST['remove']))
  133. {
  134. checkSession();
  135. validateToken('admin-ser');
  136. // Make sure every entry is a proper integer.
  137. foreach ($_POST['remove'] as $index => $spider_id)
  138. $_POST['remove'][(int) $index] = (int) $spider_id;
  139. // Delete them all!
  140. $smcFunc['db_query']('', '
  141. DELETE FROM {db_prefix}spiders
  142. WHERE id_spider IN ({array_int:remove_list})',
  143. array(
  144. 'remove_list' => $_POST['remove'],
  145. )
  146. );
  147. $smcFunc['db_query']('', '
  148. DELETE FROM {db_prefix}log_spider_hits
  149. WHERE id_spider IN ({array_int:remove_list})',
  150. array(
  151. 'remove_list' => $_POST['remove'],
  152. )
  153. );
  154. $smcFunc['db_query']('', '
  155. DELETE FROM {db_prefix}log_spider_stats
  156. WHERE id_spider IN ({array_int:remove_list})',
  157. array(
  158. 'remove_list' => $_POST['remove'],
  159. )
  160. );
  161. cache_put_data('spider_search', null, 300);
  162. recacheSpiderNames();
  163. }
  164. // Get the last seens.
  165. $request = $smcFunc['db_query']('', '
  166. SELECT id_spider, MAX(last_seen) AS last_seen_time
  167. FROM {db_prefix}log_spider_stats
  168. GROUP BY id_spider',
  169. array(
  170. )
  171. );
  172. $context['spider_last_seen'] = array();
  173. while ($row = $smcFunc['db_fetch_assoc']($request))
  174. $context['spider_last_seen'][$row['id_spider']] = $row['last_seen_time'];
  175. $smcFunc['db_free_result']($request);
  176. createToken('admin-ser');
  177. $listOptions = array(
  178. 'id' => 'spider_list',
  179. 'title' => $txt['spiders'],
  180. 'items_per_page' => 20,
  181. 'base_href' => $scripturl . '?action=admin;area=sengines;sa=spiders',
  182. 'default_sort_col' => 'name',
  183. 'get_items' => array(
  184. 'function' => 'list_getSpiders',
  185. ),
  186. 'get_count' => array(
  187. 'function' => 'list_getNumSpiders',
  188. ),
  189. 'no_items_label' => $txt['spiders_no_entries'],
  190. 'columns' => array(
  191. 'name' => array(
  192. 'header' => array(
  193. 'value' => $txt['spider_name'],
  194. ),
  195. 'data' => array(
  196. 'function' => create_function('$rowData', '
  197. global $scripturl;
  198. return sprintf(\'<a href="%1$s?action=admin;area=sengines;sa=editspiders;sid=%2$d">%3$s</a>\', $scripturl, $rowData[\'id_spider\'], htmlspecialchars($rowData[\'spider_name\']));
  199. '),
  200. ),
  201. 'sort' => array(
  202. 'default' => 'spider_name',
  203. 'reverse' => 'spider_name DESC',
  204. ),
  205. ),
  206. 'last_seen' => array(
  207. 'header' => array(
  208. 'value' => $txt['spider_last_seen'],
  209. ),
  210. 'data' => array(
  211. 'function' => create_function('$rowData', '
  212. global $context, $txt;
  213. return isset($context[\'spider_last_seen\'][$rowData[\'id_spider\']]) ? timeformat($context[\'spider_last_seen\'][$rowData[\'id_spider\']]) : $txt[\'spider_last_never\'];
  214. '),
  215. ),
  216. ),
  217. 'user_agent' => array(
  218. 'header' => array(
  219. 'value' => $txt['spider_agent'],
  220. ),
  221. 'data' => array(
  222. 'db_htmlsafe' => 'user_agent',
  223. ),
  224. 'sort' => array(
  225. 'default' => 'user_agent',
  226. 'reverse' => 'user_agent DESC',
  227. ),
  228. ),
  229. 'ip_info' => array(
  230. 'header' => array(
  231. 'value' => $txt['spider_ip_info'],
  232. ),
  233. 'data' => array(
  234. 'db_htmlsafe' => 'ip_info',
  235. 'class' => 'smalltext',
  236. ),
  237. 'sort' => array(
  238. 'default' => 'ip_info',
  239. 'reverse' => 'ip_info DESC',
  240. ),
  241. ),
  242. 'check' => array(
  243. 'header' => array(
  244. 'value' => '<input type="checkbox" onclick="invertAll(this, this.form);" class="input_check" />',
  245. ),
  246. 'data' => array(
  247. 'sprintf' => array(
  248. 'format' => '<input type="checkbox" name="remove[]" value="%1$d" class="input_check" />',
  249. 'params' => array(
  250. 'id_spider' => false,
  251. ),
  252. ),
  253. 'style' => 'text-align: center',
  254. ),
  255. ),
  256. ),
  257. 'form' => array(
  258. 'href' => $scripturl . '?action=admin;area=sengines;sa=spiders',
  259. 'token' => 'admin-ser',
  260. ),
  261. 'additional_rows' => array(
  262. array(
  263. 'position' => 'below_table_data',
  264. 'value' => '
  265. <input type="submit" name="removeSpiders" value="' . $txt['spiders_remove_selected'] . '" onclick="return confirm(\'' . $txt['spider_remove_selected_confirm'] . '\');" class="button_submit" />
  266. <input type="submit" name="addSpider" value="' . $txt['spiders_add'] . '" class="button_submit" />
  267. ',
  268. 'style' => 'text-align: right;',
  269. ),
  270. ),
  271. );
  272. require_once($sourcedir . '/Subs-List.php');
  273. createList($listOptions);
  274. $context['sub_template'] = 'show_list';
  275. $context['default_list'] = 'spider_list';
  276. }
  277. /**
  278. * Callback function for createList()
  279. * @param int $start
  280. * @param int $items_per_page
  281. * @param string sort
  282. */
  283. function list_getSpiders($start, $items_per_page, $sort)
  284. {
  285. global $smcFunc;
  286. $request = $smcFunc['db_query']('', '
  287. SELECT id_spider, spider_name, user_agent, ip_info
  288. FROM {db_prefix}spiders
  289. ORDER BY ' . $sort . '
  290. LIMIT ' . $start . ', ' . $items_per_page,
  291. array(
  292. )
  293. );
  294. $spiders = array();
  295. while ($row = $smcFunc['db_fetch_assoc']($request))
  296. $spiders[$row['id_spider']] = $row;
  297. $smcFunc['db_free_result']($request);
  298. return $spiders;
  299. }
  300. /**
  301. * Callback function for createList()
  302. * @return int
  303. */
  304. function list_getNumSpiders()
  305. {
  306. global $smcFunc;
  307. $request = $smcFunc['db_query']('', '
  308. SELECT COUNT(*) AS num_spiders
  309. FROM {db_prefix}spiders',
  310. array(
  311. )
  312. );
  313. list ($numSpiders) = $smcFunc['db_fetch_row']($request);
  314. $smcFunc['db_free_result']($request);
  315. return $numSpiders;
  316. }
  317. /**
  318. * Here we can add, and edit, spider info!
  319. */
  320. function EditSpider()
  321. {
  322. global $context, $smcFunc, $txt;
  323. // Some standard stuff.
  324. $context['id_spider'] = !empty($_GET['sid']) ? (int) $_GET['sid'] : 0;
  325. $context['page_title'] = $context['id_spider'] ? $txt['spiders_edit'] : $txt['spiders_add'];
  326. $context['sub_template'] = 'spider_edit';
  327. // Are we saving?
  328. if (!empty($_POST['save']))
  329. {
  330. checkSession();
  331. validateToken('admin-ses');
  332. $ips = array();
  333. // Check the IP range is valid.
  334. $ip_sets = explode(',', $_POST['spider_ip']);
  335. foreach ($ip_sets as $set)
  336. {
  337. $test = ip2range(trim($set));
  338. if (!empty($test))
  339. $ips[] = $set;
  340. }
  341. $ips = implode(',', $ips);
  342. // Goes in as it is...
  343. if ($context['id_spider'])
  344. $smcFunc['db_query']('', '
  345. UPDATE {db_prefix}spiders
  346. SET spider_name = {string:spider_name}, user_agent = {string:spider_agent},
  347. ip_info = {string:ip_info}
  348. WHERE id_spider = {int:current_spider}',
  349. array(
  350. 'current_spider' => $context['id_spider'],
  351. 'spider_name' => $_POST['spider_name'],
  352. 'spider_agent' => $_POST['spider_agent'],
  353. 'ip_info' => $ips,
  354. )
  355. );
  356. else
  357. $smcFunc['db_insert']('insert',
  358. '{db_prefix}spiders',
  359. array(
  360. 'spider_name' => 'string', 'user_agent' => 'string', 'ip_info' => 'string',
  361. ),
  362. array(
  363. $_POST['spider_name'], $_POST['spider_agent'], $ips,
  364. ),
  365. array('id_spider')
  366. );
  367. // Order by user agent length.
  368. sortSpiderTable();
  369. cache_put_data('spider_search', null, 300);
  370. recacheSpiderNames();
  371. redirectexit('action=admin;area=sengines;sa=spiders');
  372. }
  373. // The default is new.
  374. $context['spider'] = array(
  375. 'id' => 0,
  376. 'name' => '',
  377. 'agent' => '',
  378. 'ip_info' => '',
  379. );
  380. // An edit?
  381. if ($context['id_spider'])
  382. {
  383. $request = $smcFunc['db_query']('', '
  384. SELECT id_spider, spider_name, user_agent, ip_info
  385. FROM {db_prefix}spiders
  386. WHERE id_spider = {int:current_spider}',
  387. array(
  388. 'current_spider' => $context['id_spider'],
  389. )
  390. );
  391. if ($row = $smcFunc['db_fetch_assoc']($request))
  392. $context['spider'] = array(
  393. 'id' => $row['id_spider'],
  394. 'name' => $row['spider_name'],
  395. 'agent' => $row['user_agent'],
  396. 'ip_info' => $row['ip_info'],
  397. );
  398. $smcFunc['db_free_result']($request);
  399. }
  400. createToken('admin-ses');
  401. }
  402. /**
  403. * Do we think the current user is a spider?
  404. *
  405. * @todo Should this not be... you know... in a different file?
  406. * @return int
  407. */
  408. function SpiderCheck()
  409. {
  410. global $modSettings, $smcFunc;
  411. if (isset($_SESSION['id_robot']))
  412. unset($_SESSION['id_robot']);
  413. $_SESSION['robot_check'] = time();
  414. // We cache the spider data for five minutes if we can.
  415. if (($spider_data = cache_get_data('spider_search', 300)) === null)
  416. {
  417. $request = $smcFunc['db_query']('spider_check', '
  418. SELECT id_spider, user_agent, ip_info
  419. FROM {db_prefix}spiders',
  420. array(
  421. )
  422. );
  423. $spider_data = array();
  424. while ($row = $smcFunc['db_fetch_assoc']($request))
  425. $spider_data[] = $row;
  426. $smcFunc['db_free_result']($request);
  427. cache_put_data('spider_search', $spider_data, 300);
  428. }
  429. if (empty($spider_data))
  430. return false;
  431. // Only do these bits once.
  432. $ci_user_agent = strtolower($_SERVER['HTTP_USER_AGENT']);
  433. // Always attempt IPv6 first.
  434. if (strpos($_SERVER['REMOTE_ADDR'], ':') !== false)
  435. $ip_parts = convertIPv6toInts($_SERVER['REMOTE_ADDR']);
  436. else
  437. preg_match('/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/', $_SERVER['REMOTE_ADDR'], $ip_parts);
  438. foreach ($spider_data as $spider)
  439. {
  440. // User agent is easy.
  441. if (!empty($spider['user_agent']) && strpos($ci_user_agent, strtolower($spider['user_agent'])) !== false)
  442. $_SESSION['id_robot'] = $spider['id_spider'];
  443. // IP stuff is harder.
  444. elseif (!empty($ip_parts))
  445. {
  446. $ips = explode(',', $spider['ip_info']);
  447. foreach ($ips as $ip)
  448. {
  449. $ip = ip2range($ip);
  450. if (!empty($ip))
  451. {
  452. foreach ($ip as $key => $value)
  453. {
  454. if ($value['low'] > $ip_parts[$key + 1] || $value['high'] < $ip_parts[$key + 1])
  455. break;
  456. elseif (($key == 7 && strpos($_SERVER['REMOTE_ADDR'], ':') !== false) || ($key == 3 && strpos($_SERVER['REMOTE_ADDR'], ':') === false))
  457. $_SESSION['id_robot'] = $spider['id_spider'];
  458. }
  459. }
  460. }
  461. }
  462. if (isset($_SESSION['id_robot']))
  463. break;
  464. }
  465. // If this is low server tracking then log the spider here as oppossed to the main logging function.
  466. if (!empty($modSettings['spider_mode']) && $modSettings['spider_mode'] == 1 && !empty($_SESSION['id_robot']))
  467. logSpider();
  468. return !empty($_SESSION['id_robot']) ? $_SESSION['id_robot'] : 0;
  469. }
  470. /**
  471. * Log the spider presence online.
  472. *
  473. * @todo Different file?
  474. */
  475. function logSpider()
  476. {
  477. global $smcFunc, $modSettings, $context;
  478. if (empty($modSettings['spider_mode']) || empty($_SESSION['id_robot']))
  479. return;
  480. // Attempt to update today's entry.
  481. if ($modSettings['spider_mode'] == 1)
  482. {
  483. $date = strftime('%Y-%m-%d', forum_time(false));
  484. $smcFunc['db_query']('', '
  485. UPDATE {db_prefix}log_spider_stats
  486. SET last_seen = {int:current_time}, page_hits = page_hits + 1
  487. WHERE id_spider = {int:current_spider}
  488. AND stat_date = {date:current_date}',
  489. array(
  490. 'current_date' => $date,
  491. 'current_time' => time(),
  492. 'current_spider' => $_SESSION['id_robot'],
  493. )
  494. );
  495. // Nothing updated?
  496. if ($smcFunc['db_affected_rows']() == 0)
  497. {
  498. $smcFunc['db_insert']('ignore',
  499. '{db_prefix}log_spider_stats',
  500. array(
  501. 'id_spider' => 'int', 'last_seen' => 'int', 'stat_date' => 'date', 'page_hits' => 'int',
  502. ),
  503. array(
  504. $_SESSION['id_robot'], time(), $date, 1,
  505. ),
  506. array('id_spider', 'stat_date')
  507. );
  508. }
  509. }
  510. // If we're tracking better stats than track, better stats - we sort out the today thing later.
  511. else
  512. {
  513. if ($modSettings['spider_mode'] > 2)
  514. {
  515. $url = $_GET + array('USER_AGENT' => $_SERVER['HTTP_USER_AGENT']);
  516. unset($url['sesc'], $url[$context['session_var']]);
  517. $url = serialize($url);
  518. }
  519. else
  520. $url = '';
  521. $smcFunc['db_insert']('insert',
  522. '{db_prefix}log_spider_hits',
  523. array('id_spider' => 'int', 'log_time' => 'int', 'url' => 'string'),
  524. array($_SESSION['id_robot'], time(), $url),
  525. array()
  526. );
  527. }
  528. }
  529. /**
  530. * This function takes any unprocessed hits and turns them into stats.
  531. */
  532. function consolidateSpiderStats()
  533. {
  534. global $smcFunc;
  535. $request = $smcFunc['db_query']('consolidate_spider_stats', '
  536. SELECT id_spider, MAX(log_time) AS last_seen, COUNT(*) AS num_hits
  537. FROM {db_prefix}log_spider_hits
  538. WHERE processed = {int:not_processed}
  539. GROUP BY id_spider, MONTH(log_time), DAYOFMONTH(log_time)',
  540. array(
  541. 'not_processed' => 0,
  542. )
  543. );
  544. $spider_hits = array();
  545. while ($row = $smcFunc['db_fetch_assoc']($request))
  546. $spider_hits[] = $row;
  547. $smcFunc['db_free_result']($request);
  548. if (empty($spider_hits))
  549. return;
  550. // Attempt to update the master data.
  551. $stat_inserts = array();
  552. foreach ($spider_hits as $stat)
  553. {
  554. // We assume the max date is within the right day.
  555. $date = strftime('%Y-%m-%d', $stat['last_seen']);
  556. $smcFunc['db_query']('', '
  557. UPDATE {db_prefix}log_spider_stats
  558. SET page_hits = page_hits + ' . $stat['num_hits'] . ',
  559. last_seen = CASE WHEN last_seen > {int:last_seen} THEN last_seen ELSE {int:last_seen} END
  560. WHERE id_spider = {int:current_spider}
  561. AND stat_date = {date:last_seen_date}',
  562. array(
  563. 'last_seen_date' => $date,
  564. 'last_seen' => $stat['last_seen'],
  565. 'current_spider' => $stat['id_spider'],
  566. )
  567. );
  568. if ($smcFunc['db_affected_rows']() == 0)
  569. $stat_inserts[] = array($date, $stat['id_spider'], $stat['num_hits'], $stat['last_seen']);
  570. }
  571. // New stats?
  572. if (!empty($stat_inserts))
  573. $smcFunc['db_insert']('ignore',
  574. '{db_prefix}log_spider_stats',
  575. array('stat_date' => 'date', 'id_spider' => 'int', 'page_hits' => 'int', 'last_seen' => 'int'),
  576. $stat_inserts,
  577. array('stat_date', 'id_spider')
  578. );
  579. // All processed.
  580. $smcFunc['db_query']('', '
  581. UPDATE {db_prefix}log_spider_hits
  582. SET processed = {int:is_processed}
  583. WHERE processed = {int:not_processed}',
  584. array(
  585. 'is_processed' => 1,
  586. 'not_processed' => 0,
  587. )
  588. );
  589. }
  590. /**
  591. * See what spiders have been up to.
  592. */
  593. function SpiderLogs()
  594. {
  595. global $context, $txt, $sourcedir, $scripturl, $smcFunc, $modSettings;
  596. // Load the template and language just incase.
  597. loadLanguage('Search');
  598. loadTemplate('ManageSearch');
  599. // Did they want to delete some entries?
  600. if (!empty($_POST['delete_entries']) && isset($_POST['older']))
  601. {
  602. checkSession();
  603. validateToken('admin-sl');
  604. $deleteTime = time() - (((int) $_POST['older']) * 24 * 60 * 60);
  605. // Delete the entires.
  606. $smcFunc['db_query']('', '
  607. DELETE FROM {db_prefix}log_spider_hits
  608. WHERE log_time < {int:delete_period}',
  609. array(
  610. 'delete_period' => $deleteTime,
  611. )
  612. );
  613. }
  614. $listOptions = array(
  615. 'id' => 'spider_logs',
  616. 'items_per_page' => 20,
  617. 'title' => $txt['spider_logs'],
  618. 'no_items_label' => $txt['spider_logs_empty'],
  619. 'base_href' => $context['admin_area'] == 'sengines' ? $scripturl . '?action=admin;area=sengines;sa=logs' : $scripturl . '?action=admin;area=logs;sa=spiderlog',
  620. 'default_sort_col' => 'log_time',
  621. 'get_items' => array(
  622. 'function' => 'list_getSpiderLogs',
  623. ),
  624. 'get_count' => array(
  625. 'function' => 'list_getNumSpiderLogs',
  626. ),
  627. 'columns' => array(
  628. 'name' => array(
  629. 'header' => array(
  630. 'value' => $txt['spider'],
  631. ),
  632. 'data' => array(
  633. 'db' => 'spider_name',
  634. ),
  635. 'sort' => array(
  636. 'default' => 's.spider_name',
  637. 'reverse' => 's.spider_name DESC',
  638. ),
  639. ),
  640. 'log_time' => array(
  641. 'header' => array(
  642. 'value' => $txt['spider_time'],
  643. ),
  644. 'data' => array(
  645. 'function' => create_function('$rowData', '
  646. return timeformat($rowData[\'log_time\']);
  647. '),
  648. ),
  649. 'sort' => array(
  650. 'default' => 'sl.id_hit DESC',
  651. 'reverse' => 'sl.id_hit',
  652. ),
  653. ),
  654. 'viewing' => array(
  655. 'header' => array(
  656. 'value' => $txt['spider_viewing'],
  657. ),
  658. 'data' => array(
  659. 'db' => 'url',
  660. ),
  661. ),
  662. ),
  663. 'form' => array(
  664. 'token' => 'admin-sl',
  665. 'href' => $scripturl . '?action=admin;area=sengines;sa=logs',
  666. ),
  667. 'additional_rows' => array(
  668. array(
  669. 'position' => 'after_title',
  670. 'value' => $txt['spider_logs_info'],
  671. 'class' => 'windowbg2',
  672. ),
  673. array(
  674. 'position' => 'below_table_data',
  675. 'value' => '<input type="submit" name="removeAll" value="' . $txt['spider_log_empty_log'] . '" onclick="return confirm(\'' . $txt['spider_log_empty_log_confirm'] . '\');" class="button_submit" />',
  676. 'style' => 'text-align: right;',
  677. ),
  678. ),
  679. );
  680. createToken('admin-sl');
  681. require_once($sourcedir . '/Subs-List.php');
  682. createList($listOptions);
  683. // Now determine the actions of the URLs.
  684. if (!empty($context['spider_logs']['rows']))
  685. {
  686. $urls = array();
  687. // Grab the current /url.
  688. foreach ($context['spider_logs']['rows'] as $k => $row)
  689. {
  690. // Feature disabled?
  691. if (empty($row['viewing']['value']) && isset($modSettings['spider_mode']) && $modSettings['spider_mode'] < 3)
  692. $context['spider_logs']['rows'][$k]['viewing']['value'] = '<em>' . $txt['spider_disabled'] . '</em>';
  693. else
  694. $urls[$k] = array($row['viewing']['value'], -1);
  695. }
  696. // Now stick in the new URLs.
  697. require_once($sourcedir . '/Who.php');
  698. $urls = determineActions($urls, 'whospider_');
  699. foreach ($urls as $k => $new_url)
  700. {
  701. $context['spider_logs']['rows'][$k]['viewing']['value'] = $new_url;
  702. }
  703. }
  704. $context['page_title'] = $txt['spider_logs'];
  705. $context['sub_template'] = 'show_spider_logs';
  706. $context['default_list'] = 'spider_logs';
  707. }
  708. /**
  709. * Callback function for createList()
  710. *
  711. * @param int $start
  712. * @param int $items_per_page
  713. * @param string $sort
  714. * @return array
  715. */
  716. function list_getSpiderLogs($start, $items_per_page, $sort)
  717. {
  718. global $smcFunc;
  719. $request = $smcFunc['db_query']('', '
  720. SELECT sl.id_spider, sl.url, sl.log_time, s.spider_name
  721. FROM {db_prefix}log_spider_hits AS sl
  722. INNER JOIN {db_prefix}spiders AS s ON (s.id_spider = sl.id_spider)
  723. ORDER BY ' . $sort . '
  724. LIMIT ' . $start . ', ' . $items_per_page,
  725. array(
  726. )
  727. );
  728. $spider_logs = array();
  729. while ($row = $smcFunc['db_fetch_assoc']($request))
  730. $spider_logs[] = $row;
  731. $smcFunc['db_free_result']($request);
  732. return $spider_logs;
  733. }
  734. /**
  735. * Callback function for createList()
  736. * @return int
  737. */
  738. function list_getNumSpiderLogs()
  739. {
  740. global $smcFunc;
  741. $request = $smcFunc['db_query']('', '
  742. SELECT COUNT(*) AS num_logs
  743. FROM {db_prefix}log_spider_hits',
  744. array(
  745. )
  746. );
  747. list ($numLogs) = $smcFunc['db_fetch_row']($request);
  748. $smcFunc['db_free_result']($request);
  749. return $numLogs;
  750. }
  751. /**
  752. * Show the spider statistics.
  753. */
  754. function SpiderStats()
  755. {
  756. global $context, $txt, $sourcedir, $scripturl, $smcFunc;
  757. // Force an update of the stats every 60 seconds.
  758. if (!isset($_SESSION['spider_stat']) || $_SESSION['spider_stat'] < time() - 60)
  759. {
  760. consolidateSpiderStats();
  761. $_SESSION['spider_stat'] = time();
  762. }
  763. // Are we cleaning up some old stats?
  764. if (!empty($_POST['delete_entries']) && isset($_POST['older']))
  765. {
  766. checkSession();
  767. validateToken('admin-ss');
  768. $deleteTime = time() - (((int) $_POST['older']) * 24 * 60 * 60);
  769. // Delete the entires.
  770. $smcFunc['db_query']('', '
  771. DELETE FROM {db_prefix}log_spider_stats
  772. WHERE last_seen < {int:delete_period}',
  773. array(
  774. 'delete_period' => $deleteTime,
  775. )
  776. );
  777. }
  778. // Get the earliest and latest dates.
  779. $request = $smcFunc['db_query']('', '
  780. SELECT MIN(stat_date) AS first_date, MAX(stat_date) AS last_date
  781. FROM {db_prefix}log_spider_stats',
  782. array(
  783. )
  784. );
  785. list ($min_date, $max_date) = $smcFunc['db_fetch_row']($request);
  786. $smcFunc['db_free_result']($request);
  787. $min_year = (int) substr($min_date, 0, 4);
  788. $max_year = (int) substr($max_date, 0, 4);
  789. $min_month = (int) substr($min_date, 5, 2);
  790. $max_month = (int) substr($max_date, 5, 2);
  791. // Prepare the dates for the drop down.
  792. $date_choices = array();
  793. for ($y = $min_year; $y <= $max_year; $y++)
  794. for ($m = 1; $m <= 12; $m++)
  795. {
  796. // This doesn't count?
  797. if ($y == $min_year && $m < $min_month)
  798. continue;
  799. if ($y == $max_year && $m > $max_month)
  800. break;
  801. $date_choices[$y . $m] = $txt['months_short'][$m] . ' ' . $y;
  802. }
  803. // What are we currently viewing?
  804. $current_date = isset($_REQUEST['new_date']) && isset($date_choices[$_REQUEST['new_date']]) ? $_REQUEST['new_date'] : $max_date;
  805. // Prepare the HTML.
  806. $date_select = '
  807. ' . $txt['spider_stats_select_month'] . ':
  808. <select name="new_date" onchange="document.spider_stat_list.submit();">';
  809. if (empty($date_choices))
  810. $date_select .= '
  811. <option></option>';
  812. else
  813. foreach ($date_choices as $id => $text)
  814. $date_select .= '
  815. <option value="' . $id . '"' . ($current_date == $id ? ' selected="selected"' : '') . '>' . $text . '</option>';
  816. $date_select .= '
  817. </select>
  818. <noscript>
  819. <input type="submit" name="go" value="' . $txt['go'] . '" class="button_submit" />
  820. </noscript>';
  821. // If we manually jumped to a date work out the offset.
  822. if (isset($_REQUEST['new_date']))
  823. {
  824. $date_query = sprintf('%04d-%02d-01', substr($current_date, 0, 4), substr($current_date, 4));
  825. $request = $smcFunc['db_query']('', '
  826. SELECT COUNT(*) AS offset
  827. FROM {db_prefix}log_spider_stats
  828. WHERE stat_date < {date:date_being_viewed}',
  829. array(
  830. 'date_being_viewed' => $date_query,
  831. )
  832. );
  833. list ($_REQUEST['start']) = $smcFunc['db_fetch_row']($request);
  834. $smcFunc['db_free_result']($request);
  835. }
  836. $listOptions = array(
  837. 'id' => 'spider_stat_list',
  838. 'title' => $txt['spider'] . ' ' . $txt['spider_stats'],
  839. 'items_per_page' => 20,
  840. 'base_href' => $scripturl . '?action=admin;area=sengines;sa=stats',
  841. 'default_sort_col' => 'stat_date',
  842. 'get_items' => array(
  843. 'function' => 'list_getSpiderStats',
  844. ),
  845. 'get_count' => array(
  846. 'function' => 'list_getNumSpiderStats',
  847. ),
  848. 'no_items_label' => $txt['spider_stats_no_entries'],
  849. 'columns' => array(
  850. 'stat_date' => array(
  851. 'header' => array(
  852. 'value' => $txt['date'],
  853. ),
  854. 'data' => array(
  855. 'db' => 'stat_date',
  856. ),
  857. 'sort' => array(
  858. 'default' => 'stat_date',
  859. 'reverse' => 'stat_date DESC',
  860. ),
  861. ),
  862. 'name' => array(
  863. 'header' => array(
  864. 'value' => $txt['spider_name'],
  865. ),
  866. 'data' => array(
  867. 'db' => 'spider_name',
  868. ),
  869. 'sort' => array(
  870. 'default' => 's.spider_name',
  871. 'reverse' => 's.spider_name DESC',
  872. ),
  873. ),
  874. 'page_hits' => array(
  875. 'header' => array(
  876. 'value' => $txt['spider_stats_page_hits'],
  877. ),
  878. 'data' => array(
  879. 'db' => 'page_hits',
  880. ),
  881. 'sort' => array(
  882. 'default' => 'ss.page_hits',
  883. 'reverse' => 'ss.page_hits DESC',
  884. ),
  885. ),
  886. ),
  887. 'form' => array(
  888. 'href' => $scripturl . '?action=admin;area=sengines;sa=stats',
  889. 'name' => 'spider_stat_list',
  890. ),
  891. 'additional_rows' => array(
  892. array(
  893. 'position' => 'below_table_data',
  894. 'value' => $date_select,
  895. 'style' => 'text-align: right;',
  896. ),
  897. ),
  898. );
  899. createToken('admin-ss');
  900. require_once($sourcedir . '/Subs-List.php');
  901. createList($listOptions);
  902. $context['sub_template'] = 'show_spider_stats';
  903. $context['default_list'] = 'spider_stat_list';
  904. }
  905. /**
  906. * Callback function for createList()
  907. * @return array
  908. */
  909. function list_getSpiderStats($start, $items_per_page, $sort)
  910. {
  911. global $smcFunc;
  912. $request = $smcFunc['db_query']('', '
  913. SELECT ss.id_spider, ss.stat_date, ss.page_hits, s.spider_name
  914. FROM {db_prefix}log_spider_stats AS ss
  915. INNER JOIN {db_prefix}spiders AS s ON (s.id_spider = ss.id_spider)
  916. ORDER BY ' . $sort . '
  917. LIMIT ' . $start . ', ' . $items_per_page,
  918. array(
  919. )
  920. );
  921. $spider_stats = array();
  922. while ($row = $smcFunc['db_fetch_assoc']($request))
  923. $spider_stats[] = $row;
  924. $smcFunc['db_free_result']($request);
  925. return $spider_stats;
  926. }
  927. /**
  928. * Callback function for createList()
  929. * @return int
  930. */
  931. function list_getNumSpiderStats()
  932. {
  933. global $smcFunc;
  934. $request = $smcFunc['db_query']('', '
  935. SELECT COUNT(*) AS num_stats
  936. FROM {db_prefix}log_spider_stats',
  937. array(
  938. )
  939. );
  940. list ($numStats) = $smcFunc['db_fetch_row']($request);
  941. $smcFunc['db_free_result']($request);
  942. return $numStats;
  943. }
  944. /**
  945. * Recache spider names?
  946. */
  947. function recacheSpiderNames()
  948. {
  949. global $smcFunc;
  950. $request = $smcFunc['db_query']('', '
  951. SELECT id_spider, spider_name
  952. FROM {db_prefix}spiders',
  953. array(
  954. )
  955. );
  956. $spiders = array();
  957. while ($row = $smcFunc['db_fetch_assoc']($request))
  958. $spiders[$row['id_spider']] = $row['spider_name'];
  959. $smcFunc['db_free_result']($request);
  960. updateSettings(array('spider_name_cache' => serialize($spiders)));
  961. }
  962. /**
  963. * Sort the search engine table by user agent name to avoid misidentification of engine.
  964. */
  965. function sortSpiderTable()
  966. {
  967. global $smcFunc;
  968. db_extend('packages');
  969. // Add a sorting column.
  970. $smcFunc['db_add_column']('{db_prefix}spiders', array('name' => 'temp_order', 'size' => 8, 'type' => 'mediumint', 'null' => false));
  971. // Set the contents of this column.
  972. $smcFunc['db_query']('set_spider_order', '
  973. UPDATE {db_prefix}spiders
  974. SET temp_order = LENGTH(user_agent)',
  975. array(
  976. )
  977. );
  978. // Order the table by this column.
  979. $smcFunc['db_query']('alter_table_spiders', '
  980. ALTER TABLE {db_prefix}spiders
  981. ORDER BY temp_order DESC',
  982. array(
  983. 'db_error_skip' => true,
  984. )
  985. );
  986. // Remove the sorting column.
  987. $smcFunc['db_remove_column']('{db_prefix}spiders', 'temp_order');
  988. }