ManageSearch.php 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781
  1. <?php
  2. /**
  3. * The admin screen to change the search settings.
  4. *
  5. * Simple Machines Forum (SMF)
  6. *
  7. * @package SMF
  8. * @author Simple Machines http://www.simplemachines.org
  9. * @copyright 2011 Simple Machines
  10. * @license http://www.simplemachines.org/about/smf/license.php BSD
  11. *
  12. * @version 2.0
  13. */
  14. if (!defined('SMF'))
  15. die('Hacking attempt...');
  16. /**
  17. * Main entry point for the admin search settings screen.
  18. * It checks permissions, and it forwards to the appropriate function based on
  19. * the given sub-action.
  20. * Defaults to sub-action 'settings'.
  21. * Called by ?action=admin;area=managesearch.
  22. * Requires the admin_forum permission.
  23. *
  24. * @uses ManageSearch template.
  25. * @uses Search language file.
  26. */
  27. function ManageSearch()
  28. {
  29. global $context, $txt, $scripturl;
  30. isAllowedTo('admin_forum');
  31. loadLanguage('Search');
  32. loadTemplate('ManageSearch');
  33. db_extend('search');
  34. $subActions = array(
  35. 'settings' => 'EditSearchSettings',
  36. 'weights' => 'EditWeights',
  37. 'method' => 'EditSearchMethod',
  38. 'createfulltext' => 'EditSearchMethod',
  39. 'removecustom' => 'EditSearchMethod',
  40. 'removefulltext' => 'EditSearchMethod',
  41. 'createmsgindex' => 'CreateMessageIndex',
  42. );
  43. // Default the sub-action to 'edit search settings'.
  44. $_REQUEST['sa'] = isset($_REQUEST['sa']) && isset($subActions[$_REQUEST['sa']]) ? $_REQUEST['sa'] : 'weights';
  45. $context['sub_action'] = $_REQUEST['sa'];
  46. // Create the tabs for the template.
  47. $context[$context['admin_menu_name']]['tab_data'] = array(
  48. 'title' => $txt['manage_search'],
  49. 'help' => 'search',
  50. 'description' => $txt['search_settings_desc'],
  51. 'tabs' => array(
  52. 'weights' => array(
  53. 'description' => $txt['search_weights_desc'],
  54. ),
  55. 'method' => array(
  56. 'description' => $txt['search_method_desc'],
  57. ),
  58. 'settings' => array(
  59. 'description' => $txt['search_settings_desc'],
  60. ),
  61. ),
  62. );
  63. // Call the right function for this sub-acton.
  64. $subActions[$_REQUEST['sa']]();
  65. }
  66. /**
  67. * Edit some general settings related to the search function.
  68. * Called by ?action=admin;area=managesearch;sa=settings.
  69. * Requires the admin_forum permission.
  70. *
  71. * @param $return_config
  72. * @uses ManageSearch template, 'modify_settings' sub-template.
  73. */
  74. function EditSearchSettings($return_config = false)
  75. {
  76. global $txt, $context, $scripturl, $sourcedir, $modSettings;
  77. // What are we editing anyway?
  78. $config_vars = array(
  79. // Permission...
  80. array('permissions', 'search_posts'),
  81. // Some simple settings.
  82. array('check', 'simpleSearch'),
  83. array('int', 'search_results_per_page'),
  84. array('int', 'search_max_results', 'subtext' => $txt['search_max_results_disable']),
  85. '',
  86. // Some limitations.
  87. array('int', 'search_floodcontrol_time', 'subtext' => $txt['search_floodcontrol_time_desc']),
  88. );
  89. // Perhaps the search method wants to add some settings?
  90. $modSettings['search_index'] = empty($modSettings['search_index']) ? 'standard' : $modSettings['search_index'];
  91. if (file_exists($sourcedir . '/SearchAPI-' . ucwords($modSettings['search_index']) . '.php'))
  92. {
  93. loadClassFile('SearchAPI-' . ucwords($modSettings['search_index']) . '.php');
  94. $method_call = array($modSettings['search_index'] . '_search', 'searchSettings');
  95. if (is_callable($method_call))
  96. call_user_func_array($method_call, array(&$config_vars));
  97. }
  98. if ($return_config)
  99. return $config_vars;
  100. $context['page_title'] = $txt['search_settings_title'];
  101. $context['sub_template'] = 'show_settings';
  102. // We'll need this for the settings.
  103. require_once($sourcedir . '/ManageServer.php');
  104. // A form was submitted.
  105. if (isset($_REQUEST['save']))
  106. {
  107. checkSession();
  108. saveDBSettings($config_vars);
  109. redirectexit('action=admin;area=managesearch;sa=settings;' . $context['session_var'] . '=' . $context['session_id']);
  110. }
  111. // Prep the template!
  112. $context['post_url'] = $scripturl . '?action=admin;area=managesearch;save;sa=settings';
  113. $context['settings_title'] = $txt['search_settings_title'];
  114. prepareDBSettingContext($config_vars);
  115. }
  116. /**
  117. * Edit the relative weight of the search factors.
  118. * Called by ?action=admin;area=managesearch;sa=weights.
  119. * Requires the admin_forum permission.
  120. *
  121. * @uses ManageSearch template, 'modify_weights' sub-template.
  122. */
  123. function EditWeights()
  124. {
  125. global $txt, $context, $modSettings;
  126. $context['page_title'] = $txt['search_weights_title'];
  127. $context['sub_template'] = 'modify_weights';
  128. $factors = array(
  129. 'search_weight_frequency',
  130. 'search_weight_age',
  131. 'search_weight_length',
  132. 'search_weight_subject',
  133. 'search_weight_first_message',
  134. 'search_weight_sticky',
  135. );
  136. // A form was submitted.
  137. if (isset($_POST['save']))
  138. {
  139. checkSession();
  140. $changes = array();
  141. foreach ($factors as $factor)
  142. $changes[$factor] = (int) $_POST[$factor];
  143. updateSettings($changes);
  144. }
  145. $context['relative_weights'] = array('total' => 0);
  146. foreach ($factors as $factor)
  147. $context['relative_weights']['total'] += isset($modSettings[$factor]) ? $modSettings[$factor] : 0;
  148. foreach ($factors as $factor)
  149. $context['relative_weights'][$factor] = round(100 * (isset($modSettings[$factor]) ? $modSettings[$factor] : 0) / $context['relative_weights']['total'], 1);
  150. }
  151. /**
  152. * Edit the search method and search index used.
  153. * Calculates the size of the current search indexes in use.
  154. * Allows to create and delete a fulltext index on the messages table.
  155. * Allows to delete a custom index (that CreateMessageIndex() created).
  156. * Called by ?action=admin;area=managesearch;sa=method.
  157. * Requires the admin_forum permission.
  158. *
  159. * @uses ManageSearch template, 'select_search_method' sub-template.
  160. */
  161. function EditSearchMethod()
  162. {
  163. global $txt, $context, $modSettings, $smcFunc, $db_type, $db_prefix;
  164. $context[$context['admin_menu_name']]['current_subsection'] = 'method';
  165. $context['page_title'] = $txt['search_method_title'];
  166. $context['sub_template'] = 'select_search_method';
  167. $context['supports_fulltext'] = $smcFunc['db_search_support']('fulltext');
  168. // Load any apis.
  169. $context['search_apis'] = loadSearchAPIs();
  170. // Detect whether a fulltext index is set.
  171. if ($context['supports_fulltext'])
  172. {
  173. $request = $smcFunc['db_query']('', '
  174. SHOW INDEX
  175. FROM {db_prefix}messages',
  176. array(
  177. )
  178. );
  179. $context['fulltext_index'] = '';
  180. if ($request !== false || $smcFunc['db_num_rows']($request) != 0)
  181. {
  182. while ($row = $smcFunc['db_fetch_assoc']($request))
  183. if ($row['Column_name'] == 'body' && (isset($row['Index_type']) && $row['Index_type'] == 'FULLTEXT' || isset($row['Comment']) && $row['Comment'] == 'FULLTEXT'))
  184. $context['fulltext_index'][] = $row['Key_name'];
  185. $smcFunc['db_free_result']($request);
  186. if (is_array($context['fulltext_index']))
  187. $context['fulltext_index'] = array_unique($context['fulltext_index']);
  188. }
  189. $request = $smcFunc['db_query']('', '
  190. SHOW COLUMNS
  191. FROM {db_prefix}messages',
  192. array(
  193. )
  194. );
  195. if ($request !== false)
  196. {
  197. while ($row = $smcFunc['db_fetch_assoc']($request))
  198. if ($row['Field'] == 'body' && $row['Type'] == 'mediumtext')
  199. $context['cannot_create_fulltext'] = true;
  200. $smcFunc['db_free_result']($request);
  201. }
  202. if (preg_match('~^`(.+?)`\.(.+?)$~', $db_prefix, $match) !== 0)
  203. $request = $smcFunc['db_query']('', '
  204. SHOW TABLE STATUS
  205. FROM {string:database_name}
  206. LIKE {string:table_name}',
  207. array(
  208. 'database_name' => '`' . strtr($match[1], array('`' => '')) . '`',
  209. 'table_name' => str_replace('_', '\_', $match[2]) . 'messages',
  210. )
  211. );
  212. else
  213. $request = $smcFunc['db_query']('', '
  214. SHOW TABLE STATUS
  215. LIKE {string:table_name}',
  216. array(
  217. 'table_name' => str_replace('_', '\_', $db_prefix) . 'messages',
  218. )
  219. );
  220. if ($request !== false)
  221. {
  222. while ($row = $smcFunc['db_fetch_assoc']($request))
  223. if ((isset($row['Type']) && strtolower($row['Type']) != 'myisam') || (isset($row['Engine']) && strtolower($row['Engine']) != 'myisam'))
  224. $context['cannot_create_fulltext'] = true;
  225. $smcFunc['db_free_result']($request);
  226. }
  227. }
  228. if (!empty($_REQUEST['sa']) && $_REQUEST['sa'] == 'createfulltext')
  229. {
  230. checkSession('get');
  231. // Make sure it's gone before creating it.
  232. $smcFunc['db_query']('', '
  233. ALTER TABLE {db_prefix}messages
  234. DROP INDEX body',
  235. array(
  236. 'db_error_skip' => true,
  237. )
  238. );
  239. $smcFunc['db_query']('', '
  240. ALTER TABLE {db_prefix}messages
  241. ADD FULLTEXT body (body)',
  242. array(
  243. )
  244. );
  245. $context['fulltext_index'] = 'body';
  246. }
  247. elseif (!empty($_REQUEST['sa']) && $_REQUEST['sa'] == 'removefulltext' && !empty($context['fulltext_index']))
  248. {
  249. checkSession('get');
  250. $smcFunc['db_query']('', '
  251. ALTER TABLE {db_prefix}messages
  252. DROP INDEX ' . implode(',
  253. DROP INDEX ', $context['fulltext_index']),
  254. array(
  255. 'db_error_skip' => true,
  256. )
  257. );
  258. $context['fulltext_index'] = '';
  259. // Go back to the default search method.
  260. if (!empty($modSettings['search_index']) && $modSettings['search_index'] == 'fulltext')
  261. updateSettings(array(
  262. 'search_index' => '',
  263. ));
  264. }
  265. elseif (!empty($_REQUEST['sa']) && $_REQUEST['sa'] == 'removecustom')
  266. {
  267. checkSession('get');
  268. db_extend();
  269. $tables = $smcFunc['db_list_tables'](false, $db_prefix . 'log_search_words');
  270. if (!empty($tables))
  271. {
  272. $smcFunc['db_search_query']('drop_words_table', '
  273. DROP TABLE {db_prefix}log_search_words',
  274. array(
  275. )
  276. );
  277. }
  278. updateSettings(array(
  279. 'search_custom_index_config' => '',
  280. 'search_custom_index_resume' => '',
  281. ));
  282. // Go back to the default search method.
  283. if (!empty($modSettings['search_index']) && $modSettings['search_index'] == 'custom')
  284. updateSettings(array(
  285. 'search_index' => '',
  286. ));
  287. }
  288. elseif (isset($_POST['save']))
  289. {
  290. checkSession();
  291. updateSettings(array(
  292. 'search_index' => empty($_POST['search_index']) || (!in_array($_POST['search_index'], array('fulltext', 'custom')) && !isset($context['search_apis'][$_POST['search_index']])) ? '' : $_POST['search_index'],
  293. 'search_force_index' => isset($_POST['search_force_index']) ? '1' : '0',
  294. 'search_match_words' => isset($_POST['search_match_words']) ? '1' : '0',
  295. ));
  296. }
  297. $context['table_info'] = array(
  298. 'data_length' => 0,
  299. 'index_length' => 0,
  300. 'fulltext_length' => 0,
  301. 'custom_index_length' => 0,
  302. );
  303. // Get some info about the messages table, to show its size and index size.
  304. if ($db_type == 'mysql')
  305. {
  306. if (preg_match('~^`(.+?)`\.(.+?)$~', $db_prefix, $match) !== 0)
  307. $request = $smcFunc['db_query']('', '
  308. SHOW TABLE STATUS
  309. FROM {string:database_name}
  310. LIKE {string:table_name}',
  311. array(
  312. 'database_name' => '`' . strtr($match[1], array('`' => '')) . '`',
  313. 'table_name' => str_replace('_', '\_', $match[2]) . 'messages',
  314. )
  315. );
  316. else
  317. $request = $smcFunc['db_query']('', '
  318. SHOW TABLE STATUS
  319. LIKE {string:table_name}',
  320. array(
  321. 'table_name' => str_replace('_', '\_', $db_prefix) . 'messages',
  322. )
  323. );
  324. if ($request !== false && $smcFunc['db_num_rows']($request) == 1)
  325. {
  326. // Only do this if the user has permission to execute this query.
  327. $row = $smcFunc['db_fetch_assoc']($request);
  328. $context['table_info']['data_length'] = $row['Data_length'];
  329. $context['table_info']['index_length'] = $row['Index_length'];
  330. $context['table_info']['fulltext_length'] = $row['Index_length'];
  331. $smcFunc['db_free_result']($request);
  332. }
  333. // Now check the custom index table, if it exists at all.
  334. if (preg_match('~^`(.+?)`\.(.+?)$~', $db_prefix, $match) !== 0)
  335. $request = $smcFunc['db_query']('', '
  336. SHOW TABLE STATUS
  337. FROM {string:database_name}
  338. LIKE {string:table_name}',
  339. array(
  340. 'database_name' => '`' . strtr($match[1], array('`' => '')) . '`',
  341. 'table_name' => str_replace('_', '\_', $match[2]) . 'log_search_words',
  342. )
  343. );
  344. else
  345. $request = $smcFunc['db_query']('', '
  346. SHOW TABLE STATUS
  347. LIKE {string:table_name}',
  348. array(
  349. 'table_name' => str_replace('_', '\_', $db_prefix) . 'log_search_words',
  350. )
  351. );
  352. if ($request !== false && $smcFunc['db_num_rows']($request) == 1)
  353. {
  354. // Only do this if the user has permission to execute this query.
  355. $row = $smcFunc['db_fetch_assoc']($request);
  356. $context['table_info']['index_length'] += $row['Data_length'] + $row['Index_length'];
  357. $context['table_info']['custom_index_length'] = $row['Data_length'] + $row['Index_length'];
  358. $smcFunc['db_free_result']($request);
  359. }
  360. }
  361. elseif ($db_type == 'postgresql')
  362. {
  363. // In order to report the sizes correctly we need to perform vacuum (optimize) on the tables we will be using.
  364. db_extend();
  365. $temp_tables = $smcFunc['db_list_tables']();
  366. foreach ($temp_tables as $table)
  367. if ($table == $db_prefix. 'messages' || $table == $db_prefix. 'log_search_words')
  368. $smcFunc['db_optimize_table']($table);
  369. // PostGreSql has some hidden sizes.
  370. $request = $smcFunc['db_query']('', '
  371. SELECT relname, relpages * 8 *1024 AS "KB" FROM pg_class
  372. WHERE relname = {string:messages} OR relname = {string:log_search_words}
  373. ORDER BY relpages DESC',
  374. array(
  375. 'messages' => $db_prefix. 'messages',
  376. 'log_search_words' => $db_prefix. 'log_search_words',
  377. )
  378. );
  379. if ($request !== false && $smcFunc['db_num_rows']($request) > 0)
  380. {
  381. while ($row = $smcFunc['db_fetch_assoc']($request))
  382. {
  383. if ($row['relname'] == $db_prefix . 'messages')
  384. {
  385. $context['table_info']['data_length'] = (int) $row['KB'];
  386. $context['table_info']['index_length'] = (int) $row['KB'];
  387. // Doesn't support fulltext
  388. $context['table_info']['fulltext_length'] = $txt['not_applicable'];
  389. }
  390. elseif ($row['relname'] == $db_prefix. 'log_search_words')
  391. {
  392. $context['table_info']['index_length'] = (int) $row['KB'];
  393. $context['table_info']['custom_index_length'] = (int) $row['KB'];
  394. }
  395. }
  396. $smcFunc['db_free_result']($request);
  397. }
  398. else
  399. // Didn't work for some reason...
  400. $context['table_info'] = array(
  401. 'data_length' => $txt['not_applicable'],
  402. 'index_length' => $txt['not_applicable'],
  403. 'fulltext_length' => $txt['not_applicable'],
  404. 'custom_index_length' => $txt['not_applicable'],
  405. );
  406. }
  407. else
  408. $context['table_info'] = array(
  409. 'data_length' => $txt['not_applicable'],
  410. 'index_length' => $txt['not_applicable'],
  411. 'fulltext_length' => $txt['not_applicable'],
  412. 'custom_index_length' => $txt['not_applicable'],
  413. );
  414. // Format the data and index length in kilobytes.
  415. foreach ($context['table_info'] as $type => $size)
  416. {
  417. // If it's not numeric then just break. This database engine doesn't support size.
  418. if (!is_numeric($size))
  419. break;
  420. $context['table_info'][$type] = comma_format($context['table_info'][$type] / 1024) . ' ' . $txt['search_method_kilobytes'];
  421. }
  422. $context['custom_index'] = !empty($modSettings['search_custom_index_config']);
  423. $context['partial_custom_index'] = !empty($modSettings['search_custom_index_resume']) && empty($modSettings['search_custom_index_config']);
  424. $context['double_index'] = !empty($context['fulltext_index']) && $context['custom_index'];
  425. }
  426. /**
  427. * Create a custom search index for the messages table.
  428. * Called by ?action=admin;area=managesearch;sa=createmsgindex.
  429. * Linked from the EditSearchMethod screen.
  430. * Requires the admin_forum permission.
  431. * Depending on the size of the message table, the process is divided in steps.
  432. *
  433. * @uses ManageSearch template, 'create_index', 'create_index_progress', and 'create_index_done'
  434. * sub-templates.
  435. */
  436. function CreateMessageIndex()
  437. {
  438. global $modSettings, $context, $smcFunc, $db_prefix, $txt;
  439. // Scotty, we need more time...
  440. @set_time_limit(600);
  441. if (function_exists('apache_reset_timeout'))
  442. @apache_reset_timeout();
  443. $context[$context['admin_menu_name']]['current_subsection'] = 'method';
  444. $context['page_title'] = $txt['search_index_custom'];
  445. $messages_per_batch = 50;
  446. $index_properties = array(
  447. 2 => array(
  448. 'column_definition' => 'small',
  449. 'step_size' => 1000000,
  450. ),
  451. 4 => array(
  452. 'column_definition' => 'medium',
  453. 'step_size' => 1000000,
  454. 'max_size' => 16777215,
  455. ),
  456. 5 => array(
  457. 'column_definition' => 'large',
  458. 'step_size' => 100000000,
  459. 'max_size' => 2000000000,
  460. ),
  461. );
  462. if (isset($_REQUEST['resume']) && !empty($modSettings['search_custom_index_resume']))
  463. {
  464. $context['index_settings'] = unserialize($modSettings['search_custom_index_resume']);
  465. $context['start'] = (int) $context['index_settings']['resume_at'];
  466. unset($context['index_settings']['resume_at']);
  467. $context['step'] = 1;
  468. }
  469. else
  470. {
  471. $context['index_settings'] = array(
  472. 'bytes_per_word' => isset($_REQUEST['bytes_per_word']) && isset($index_properties[$_REQUEST['bytes_per_word']]) ? (int) $_REQUEST['bytes_per_word'] : 2,
  473. );
  474. $context['start'] = isset($_REQUEST['start']) ? (int) $_REQUEST['start'] : 0;
  475. $context['step'] = isset($_REQUEST['step']) ? (int) $_REQUEST['step'] : 0;
  476. }
  477. if ($context['step'] !== 0)
  478. checkSession('request');
  479. // Step 0: let the user determine how they like their index.
  480. if ($context['step'] === 0)
  481. {
  482. $context['sub_template'] = 'create_index';
  483. }
  484. // Step 1: insert all the words.
  485. if ($context['step'] === 1)
  486. {
  487. $context['sub_template'] = 'create_index_progress';
  488. if ($context['start'] === 0)
  489. {
  490. db_extend();
  491. $tables = $smcFunc['db_list_tables'](false, $db_prefix . 'log_search_words');
  492. if (!empty($tables))
  493. {
  494. $smcFunc['db_search_query']('drop_words_table', '
  495. DROP TABLE {db_prefix}log_search_words',
  496. array(
  497. )
  498. );
  499. }
  500. $smcFunc['db_create_word_search']($index_properties[$context['index_settings']['bytes_per_word']]['column_definition']);
  501. // Temporarily switch back to not using a search index.
  502. if (!empty($modSettings['search_index']) && $modSettings['search_index'] == 'custom')
  503. updateSettings(array('search_index' => ''));
  504. // Don't let simultanious processes be updating the search index.
  505. if (!empty($modSettings['search_custom_index_config']))
  506. updateSettings(array('search_custom_index_config' => ''));
  507. }
  508. $num_messages = array(
  509. 'done' => 0,
  510. 'todo' => 0,
  511. );
  512. $request = $smcFunc['db_query']('', '
  513. SELECT id_msg >= {int:starting_id} AS todo, COUNT(*) AS num_messages
  514. FROM {db_prefix}messages
  515. GROUP BY todo',
  516. array(
  517. 'starting_id' => $context['start'],
  518. )
  519. );
  520. while ($row = $smcFunc['db_fetch_assoc']($request))
  521. $num_messages[empty($row['todo']) ? 'done' : 'todo'] = $row['num_messages'];
  522. if (empty($num_messages['todo']))
  523. {
  524. $context['step'] = 2;
  525. $context['percentage'] = 80;
  526. $context['start'] = 0;
  527. }
  528. else
  529. {
  530. // Number of seconds before the next step.
  531. $stop = time() + 3;
  532. while (time() < $stop)
  533. {
  534. $inserts = array();
  535. $request = $smcFunc['db_query']('', '
  536. SELECT id_msg, body
  537. FROM {db_prefix}messages
  538. WHERE id_msg BETWEEN {int:starting_id} AND {int:ending_id}
  539. LIMIT {int:limit}',
  540. array(
  541. 'starting_id' => $context['start'],
  542. 'ending_id' => $context['start'] + $messages_per_batch - 1,
  543. 'limit' => $messages_per_batch,
  544. )
  545. );
  546. $forced_break = false;
  547. $number_processed = 0;
  548. while ($row = $smcFunc['db_fetch_assoc']($request))
  549. {
  550. // In theory it's possible for one of these to take friggin ages so add more timeout protection.
  551. if ($stop < time())
  552. {
  553. $forced_break = true;
  554. break;
  555. }
  556. $number_processed++;
  557. foreach (text2words($row['body'], $context['index_settings']['bytes_per_word'], true) as $id_word)
  558. {
  559. $inserts[] = array($id_word, $row['id_msg']);
  560. }
  561. }
  562. $num_messages['done'] += $number_processed;
  563. $num_messages['todo'] -= $number_processed;
  564. $smcFunc['db_free_result']($request);
  565. $context['start'] += $forced_break ? $number_processed : $messages_per_batch;
  566. if (!empty($inserts))
  567. $smcFunc['db_insert']('ignore',
  568. '{db_prefix}log_search_words',
  569. array('id_word' => 'int', 'id_msg' => 'int'),
  570. $inserts,
  571. array('id_word', 'id_msg')
  572. );
  573. if ($num_messages['todo'] === 0)
  574. {
  575. $context['step'] = 2;
  576. $context['start'] = 0;
  577. break;
  578. }
  579. else
  580. updateSettings(array('search_custom_index_resume' => serialize(array_merge($context['index_settings'], array('resume_at' => $context['start'])))));
  581. }
  582. // Since there are still two steps to go, 90% is the maximum here.
  583. $context['percentage'] = round($num_messages['done'] / ($num_messages['done'] + $num_messages['todo']), 3) * 80;
  584. }
  585. }
  586. // Step 2: removing the words that occur too often and are of no use.
  587. elseif ($context['step'] === 2)
  588. {
  589. if ($context['index_settings']['bytes_per_word'] < 4)
  590. $context['step'] = 3;
  591. else
  592. {
  593. $stop_words = $context['start'] === 0 || empty($modSettings['search_stopwords']) ? array() : explode(',', $modSettings['search_stopwords']);
  594. $stop = time() + 3;
  595. $context['sub_template'] = 'create_index_progress';
  596. $max_messages = ceil(60 * $modSettings['totalMessages'] / 100);
  597. while (time() < $stop)
  598. {
  599. $request = $smcFunc['db_query']('', '
  600. SELECT id_word, COUNT(id_word) AS num_words
  601. FROM {db_prefix}log_search_words
  602. WHERE id_word BETWEEN {int:starting_id} AND {int:ending_id}
  603. GROUP BY id_word
  604. HAVING COUNT(id_word) > {int:minimum_messages}',
  605. array(
  606. 'starting_id' => $context['start'],
  607. 'ending_id' => $context['start'] + $index_properties[$context['index_settings']['bytes_per_word']]['step_size'] - 1,
  608. 'minimum_messages' => $max_messages,
  609. )
  610. );
  611. while ($row = $smcFunc['db_fetch_assoc']($request))
  612. $stop_words[] = $row['id_word'];
  613. $smcFunc['db_free_result']($request);
  614. updateSettings(array('search_stopwords' => implode(',', $stop_words)));
  615. if (!empty($stop_words))
  616. $smcFunc['db_query']('', '
  617. DELETE FROM {db_prefix}log_search_words
  618. WHERE id_word in ({array_int:stop_words})',
  619. array(
  620. 'stop_words' => $stop_words,
  621. )
  622. );
  623. $context['start'] += $index_properties[$context['index_settings']['bytes_per_word']]['step_size'];
  624. if ($context['start'] > $index_properties[$context['index_settings']['bytes_per_word']]['max_size'])
  625. {
  626. $context['step'] = 3;
  627. break;
  628. }
  629. }
  630. $context['percentage'] = 80 + round($context['start'] / $index_properties[$context['index_settings']['bytes_per_word']]['max_size'], 3) * 20;
  631. }
  632. }
  633. // Step 3: remove words not distinctive enough.
  634. if ($context['step'] === 3)
  635. {
  636. $context['sub_template'] = 'create_index_done';
  637. updateSettings(array('search_index' => 'custom', 'search_custom_index_config' => serialize($context['index_settings'])));
  638. $smcFunc['db_query']('', '
  639. DELETE FROM {db_prefix}settings
  640. WHERE variable = {string:search_custom_index_resume}',
  641. array(
  642. 'search_custom_index_resume' => 'search_custom_index_resume',
  643. )
  644. );
  645. }
  646. }
  647. /**
  648. * Get the installed Search API implementations.
  649. * This function checks for patterns in comments on top of the Search-API files!
  650. * In addition to filenames pattern.
  651. * It loads the search API classes if identified.
  652. * This function is used by EditSearchMethod to list all installed API implementations.
  653. */
  654. function loadSearchAPIs()
  655. {
  656. global $sourcedir, $txt;
  657. $apis = array();
  658. if ($dh = opendir($sourcedir))
  659. {
  660. while (($file = readdir($dh)) !== false)
  661. {
  662. if (is_file($sourcedir . '/' . $file) && preg_match('~SearchAPI-([A-Za-z\d_]+)\.php~', $file, $matches))
  663. {
  664. // Check this is definitely a valid API!
  665. $fp = fopen($sourcedir . '/' . $file, 'rb');
  666. $header = fread($fp, 4096);
  667. fclose($fp);
  668. if (strpos($header, '* SearchAPI-' . $matches[1] . '.php') !== false)
  669. {
  670. loadClassFile($file);
  671. $index_name = strtolower($matches[1]);
  672. $search_class_name = $index_name . '_search';
  673. $searchAPI = new $search_class_name();
  674. // No Support? NEXT!
  675. if (!$searchAPI->is_supported)
  676. continue;
  677. $apis[$index_name] = array(
  678. 'filename' => $file,
  679. 'setting_index' => $index_name,
  680. 'has_template' => in_array($index_name, array('custom', 'fulltext', 'standard')),
  681. 'label' => $index_name && isset($txt['search_index_' . $index_name]) ? $txt['search_index_' . $index_name] : '',
  682. 'desc' => $index_name && isset($txt['search_index_' . $index_name . '_desc']) ? $txt['search_index_' . $index_name . '_desc'] : '',
  683. );
  684. }
  685. }
  686. }
  687. }
  688. closedir($dh);
  689. return $apis;
  690. }
  691. ?>