ManageSearch.php 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775
  1. <?php
  2. /**
  3. * Simple Machines Forum (SMF)
  4. *
  5. * @package SMF
  6. * @author Simple Machines http://www.simplemachines.org
  7. * @copyright 2011 Simple Machines
  8. * @license http://www.simplemachines.org/about/smf/license.php BSD
  9. *
  10. * @version 2.0
  11. */
  12. if (!defined('SMF'))
  13. die('Hacking attempt...');
  14. /* The admin screen to change the search settings.
  15. void ManageSearch()
  16. - main entry point for the admin search settings screen.
  17. - called by ?action=admin;area=managesearch.
  18. - requires the admin_forum permission.
  19. - loads the ManageSearch template.
  20. - loads the Search language file.
  21. - calls a function based on the given sub-action.
  22. - defaults to sub-action 'settings'.
  23. void EditSearchSettings()
  24. - edit some general settings related to the search function.
  25. - called by ?action=admin;area=managesearch;sa=settings.
  26. - requires the admin_forum permission.
  27. - uses the 'modify_settings' sub template of the ManageSearch template.
  28. void EditWeights()
  29. - edit the relative weight of the search factors.
  30. - called by ?action=admin;area=managesearch;sa=weights.
  31. - requires the admin_forum permission.
  32. - uses the 'modify_weights' sub template of the ManageSearch template.
  33. void EditSearchMethod()
  34. - edit the search method and search index used.
  35. - called by ?action=admin;area=managesearch;sa=method.
  36. - requires the admin_forum permission.
  37. - uses the 'select_search_method' sub template of the ManageSearch
  38. template.
  39. - allows to create and delete a fulltext index on the messages table.
  40. - allows to delete a custom index (that CreateMessageIndex() created).
  41. - calculates the size of the current search indexes in use.
  42. void CreateMessageIndex()
  43. - create a custom search index for the messages table.
  44. - called by ?action=admin;area=managesearch;sa=createmsgindex.
  45. - linked from the EditSearchMethod screen.
  46. - requires the admin_forum permission.
  47. - uses the 'create_index', 'create_index_progress', and
  48. 'create_index_done' sub templates of the ManageSearch template.
  49. - depending on the size of the message table, the process is divided
  50. in steps.
  51. array loadSearchAPIs()
  52. - get the installed APIs.
  53. */
  54. function ManageSearch()
  55. {
  56. global $context, $txt, $scripturl;
  57. isAllowedTo('admin_forum');
  58. loadLanguage('Search');
  59. loadTemplate('ManageSearch');
  60. db_extend('search');
  61. $subActions = array(
  62. 'settings' => 'EditSearchSettings',
  63. 'weights' => 'EditWeights',
  64. 'method' => 'EditSearchMethod',
  65. 'createfulltext' => 'EditSearchMethod',
  66. 'removecustom' => 'EditSearchMethod',
  67. 'removefulltext' => 'EditSearchMethod',
  68. 'createmsgindex' => 'CreateMessageIndex',
  69. );
  70. // Default the sub-action to 'edit search settings'.
  71. $_REQUEST['sa'] = isset($_REQUEST['sa']) && isset($subActions[$_REQUEST['sa']]) ? $_REQUEST['sa'] : 'weights';
  72. $context['sub_action'] = $_REQUEST['sa'];
  73. // Create the tabs for the template.
  74. $context[$context['admin_menu_name']]['tab_data'] = array(
  75. 'title' => $txt['manage_search'],
  76. 'help' => 'search',
  77. 'description' => $txt['search_settings_desc'],
  78. 'tabs' => array(
  79. 'weights' => array(
  80. 'description' => $txt['search_weights_desc'],
  81. ),
  82. 'method' => array(
  83. 'description' => $txt['search_method_desc'],
  84. ),
  85. 'settings' => array(
  86. 'description' => $txt['search_settings_desc'],
  87. ),
  88. ),
  89. );
  90. // Call the right function for this sub-acton.
  91. $subActions[$_REQUEST['sa']]();
  92. }
  93. function EditSearchSettings($return_config = false)
  94. {
  95. global $txt, $context, $scripturl, $sourcedir, $modSettings;
  96. // What are we editing anyway?
  97. $config_vars = array(
  98. // Permission...
  99. array('permissions', 'search_posts'),
  100. // Some simple settings.
  101. array('check', 'simpleSearch'),
  102. array('int', 'search_results_per_page'),
  103. array('int', 'search_max_results', 'subtext' => $txt['search_max_results_disable']),
  104. '',
  105. // Some limitations.
  106. array('int', 'search_floodcontrol_time', 'subtext' => $txt['search_floodcontrol_time_desc']),
  107. );
  108. // Perhaps the search method wants to add some settings?
  109. $modSettings['search_index'] = empty($modSettings['search_index']) ? 'standard' : $modSettings['search_index'];
  110. if (file_exists($sourcedir . '/SearchAPI-' . ucwords($modSettings['search_index']) . '.php'))
  111. {
  112. loadClassFile('SearchAPI-' . ucwords($modSettings['search_index']) . '.php');
  113. $method_call = array($modSettings['search_index'] . '_search', 'searchSettings');
  114. if (is_callable($method_call))
  115. call_user_func_array($method_call, array(&$config_vars));
  116. }
  117. if ($return_config)
  118. return $config_vars;
  119. $context['page_title'] = $txt['search_settings_title'];
  120. $context['sub_template'] = 'show_settings';
  121. // We'll need this for the settings.
  122. require_once($sourcedir . '/ManageServer.php');
  123. // A form was submitted.
  124. if (isset($_REQUEST['save']))
  125. {
  126. checkSession();
  127. saveDBSettings($config_vars);
  128. redirectexit('action=admin;area=managesearch;sa=settings;' . $context['session_var'] . '=' . $context['session_id']);
  129. }
  130. // Prep the template!
  131. $context['post_url'] = $scripturl . '?action=admin;area=managesearch;save;sa=settings';
  132. $context['settings_title'] = $txt['search_settings_title'];
  133. prepareDBSettingContext($config_vars);
  134. }
  135. function EditWeights()
  136. {
  137. global $txt, $context, $modSettings;
  138. $context['page_title'] = $txt['search_weights_title'];
  139. $context['sub_template'] = 'modify_weights';
  140. $factors = array(
  141. 'search_weight_frequency',
  142. 'search_weight_age',
  143. 'search_weight_length',
  144. 'search_weight_subject',
  145. 'search_weight_first_message',
  146. 'search_weight_sticky',
  147. );
  148. // A form was submitted.
  149. if (isset($_POST['save']))
  150. {
  151. checkSession();
  152. $changes = array();
  153. foreach ($factors as $factor)
  154. $changes[$factor] = (int) $_POST[$factor];
  155. updateSettings($changes);
  156. }
  157. $context['relative_weights'] = array('total' => 0);
  158. foreach ($factors as $factor)
  159. $context['relative_weights']['total'] += isset($modSettings[$factor]) ? $modSettings[$factor] : 0;
  160. foreach ($factors as $factor)
  161. $context['relative_weights'][$factor] = round(100 * (isset($modSettings[$factor]) ? $modSettings[$factor] : 0) / $context['relative_weights']['total'], 1);
  162. }
  163. function EditSearchMethod()
  164. {
  165. global $txt, $context, $modSettings, $smcFunc, $db_type, $db_prefix;
  166. $context[$context['admin_menu_name']]['current_subsection'] = 'method';
  167. $context['page_title'] = $txt['search_method_title'];
  168. $context['sub_template'] = 'select_search_method';
  169. $context['supports_fulltext'] = $smcFunc['db_search_support']('fulltext');
  170. // Load any apis.
  171. $context['search_apis'] = loadSearchAPIs();
  172. // Detect whether a fulltext index is set.
  173. if ($context['supports_fulltext'])
  174. {
  175. $request = $smcFunc['db_query']('', '
  176. SHOW INDEX
  177. FROM {db_prefix}messages',
  178. array(
  179. )
  180. );
  181. $context['fulltext_index'] = '';
  182. if ($request !== false || $smcFunc['db_num_rows']($request) != 0)
  183. {
  184. while ($row = $smcFunc['db_fetch_assoc']($request))
  185. if ($row['Column_name'] == 'body' && (isset($row['Index_type']) && $row['Index_type'] == 'FULLTEXT' || isset($row['Comment']) && $row['Comment'] == 'FULLTEXT'))
  186. $context['fulltext_index'][] = $row['Key_name'];
  187. $smcFunc['db_free_result']($request);
  188. if (is_array($context['fulltext_index']))
  189. $context['fulltext_index'] = array_unique($context['fulltext_index']);
  190. }
  191. $request = $smcFunc['db_query']('', '
  192. SHOW COLUMNS
  193. FROM {db_prefix}messages',
  194. array(
  195. )
  196. );
  197. if ($request !== false)
  198. {
  199. while ($row = $smcFunc['db_fetch_assoc']($request))
  200. if ($row['Field'] == 'body' && $row['Type'] == 'mediumtext')
  201. $context['cannot_create_fulltext'] = true;
  202. $smcFunc['db_free_result']($request);
  203. }
  204. if (preg_match('~^`(.+?)`\.(.+?)$~', $db_prefix, $match) !== 0)
  205. $request = $smcFunc['db_query']('', '
  206. SHOW TABLE STATUS
  207. FROM {string:database_name}
  208. LIKE {string:table_name}',
  209. array(
  210. 'database_name' => '`' . strtr($match[1], array('`' => '')) . '`',
  211. 'table_name' => str_replace('_', '\_', $match[2]) . 'messages',
  212. )
  213. );
  214. else
  215. $request = $smcFunc['db_query']('', '
  216. SHOW TABLE STATUS
  217. LIKE {string:table_name}',
  218. array(
  219. 'table_name' => str_replace('_', '\_', $db_prefix) . 'messages',
  220. )
  221. );
  222. if ($request !== false)
  223. {
  224. while ($row = $smcFunc['db_fetch_assoc']($request))
  225. if ((isset($row['Type']) && strtolower($row['Type']) != 'myisam') || (isset($row['Engine']) && strtolower($row['Engine']) != 'myisam'))
  226. $context['cannot_create_fulltext'] = true;
  227. $smcFunc['db_free_result']($request);
  228. }
  229. }
  230. if (!empty($_REQUEST['sa']) && $_REQUEST['sa'] == 'createfulltext')
  231. {
  232. checkSession('get');
  233. // Make sure it's gone before creating it.
  234. $smcFunc['db_query']('', '
  235. ALTER TABLE {db_prefix}messages
  236. DROP INDEX body',
  237. array(
  238. 'db_error_skip' => true,
  239. )
  240. );
  241. $smcFunc['db_query']('', '
  242. ALTER TABLE {db_prefix}messages
  243. ADD FULLTEXT body (body)',
  244. array(
  245. )
  246. );
  247. $context['fulltext_index'] = 'body';
  248. }
  249. elseif (!empty($_REQUEST['sa']) && $_REQUEST['sa'] == 'removefulltext' && !empty($context['fulltext_index']))
  250. {
  251. checkSession('get');
  252. $smcFunc['db_query']('', '
  253. ALTER TABLE {db_prefix}messages
  254. DROP INDEX ' . implode(',
  255. DROP INDEX ', $context['fulltext_index']),
  256. array(
  257. 'db_error_skip' => true,
  258. )
  259. );
  260. $context['fulltext_index'] = '';
  261. // Go back to the default search method.
  262. if (!empty($modSettings['search_index']) && $modSettings['search_index'] == 'fulltext')
  263. updateSettings(array(
  264. 'search_index' => '',
  265. ));
  266. }
  267. elseif (!empty($_REQUEST['sa']) && $_REQUEST['sa'] == 'removecustom')
  268. {
  269. checkSession('get');
  270. db_extend();
  271. $tables = $smcFunc['db_list_tables'](false, $db_prefix . 'log_search_words');
  272. if (!empty($tables))
  273. {
  274. $smcFunc['db_search_query']('drop_words_table', '
  275. DROP TABLE {db_prefix}log_search_words',
  276. array(
  277. )
  278. );
  279. }
  280. updateSettings(array(
  281. 'search_custom_index_config' => '',
  282. 'search_custom_index_resume' => '',
  283. ));
  284. // Go back to the default search method.
  285. if (!empty($modSettings['search_index']) && $modSettings['search_index'] == 'custom')
  286. updateSettings(array(
  287. 'search_index' => '',
  288. ));
  289. }
  290. elseif (isset($_POST['save']))
  291. {
  292. checkSession();
  293. updateSettings(array(
  294. 'search_index' => empty($_POST['search_index']) || (!in_array($_POST['search_index'], array('fulltext', 'custom')) && !isset($context['search_apis'][$_POST['search_index']])) ? '' : $_POST['search_index'],
  295. 'search_force_index' => isset($_POST['search_force_index']) ? '1' : '0',
  296. 'search_match_words' => isset($_POST['search_match_words']) ? '1' : '0',
  297. ));
  298. }
  299. $context['table_info'] = array(
  300. 'data_length' => 0,
  301. 'index_length' => 0,
  302. 'fulltext_length' => 0,
  303. 'custom_index_length' => 0,
  304. );
  305. // Get some info about the messages table, to show its size and index size.
  306. if ($db_type == 'mysql')
  307. {
  308. if (preg_match('~^`(.+?)`\.(.+?)$~', $db_prefix, $match) !== 0)
  309. $request = $smcFunc['db_query']('', '
  310. SHOW TABLE STATUS
  311. FROM {string:database_name}
  312. LIKE {string:table_name}',
  313. array(
  314. 'database_name' => '`' . strtr($match[1], array('`' => '')) . '`',
  315. 'table_name' => str_replace('_', '\_', $match[2]) . 'messages',
  316. )
  317. );
  318. else
  319. $request = $smcFunc['db_query']('', '
  320. SHOW TABLE STATUS
  321. LIKE {string:table_name}',
  322. array(
  323. 'table_name' => str_replace('_', '\_', $db_prefix) . 'messages',
  324. )
  325. );
  326. if ($request !== false && $smcFunc['db_num_rows']($request) == 1)
  327. {
  328. // Only do this if the user has permission to execute this query.
  329. $row = $smcFunc['db_fetch_assoc']($request);
  330. $context['table_info']['data_length'] = $row['Data_length'];
  331. $context['table_info']['index_length'] = $row['Index_length'];
  332. $context['table_info']['fulltext_length'] = $row['Index_length'];
  333. $smcFunc['db_free_result']($request);
  334. }
  335. // Now check the custom index table, if it exists at all.
  336. if (preg_match('~^`(.+?)`\.(.+?)$~', $db_prefix, $match) !== 0)
  337. $request = $smcFunc['db_query']('', '
  338. SHOW TABLE STATUS
  339. FROM {string:database_name}
  340. LIKE {string:table_name}',
  341. array(
  342. 'database_name' => '`' . strtr($match[1], array('`' => '')) . '`',
  343. 'table_name' => str_replace('_', '\_', $match[2]) . 'log_search_words',
  344. )
  345. );
  346. else
  347. $request = $smcFunc['db_query']('', '
  348. SHOW TABLE STATUS
  349. LIKE {string:table_name}',
  350. array(
  351. 'table_name' => str_replace('_', '\_', $db_prefix) . 'log_search_words',
  352. )
  353. );
  354. if ($request !== false && $smcFunc['db_num_rows']($request) == 1)
  355. {
  356. // Only do this if the user has permission to execute this query.
  357. $row = $smcFunc['db_fetch_assoc']($request);
  358. $context['table_info']['index_length'] += $row['Data_length'] + $row['Index_length'];
  359. $context['table_info']['custom_index_length'] = $row['Data_length'] + $row['Index_length'];
  360. $smcFunc['db_free_result']($request);
  361. }
  362. }
  363. elseif ($db_type == 'postgresql')
  364. {
  365. // In order to report the sizes correctly we need to perform vacuum (optimize) on the tables we will be using.
  366. db_extend();
  367. $temp_tables = $smcFunc['db_list_tables']();
  368. foreach ($temp_tables as $table)
  369. if ($table == $db_prefix. 'messages' || $table == $db_prefix. 'log_search_words')
  370. $smcFunc['db_optimize_table']($table);
  371. // PostGreSql has some hidden sizes.
  372. $request = $smcFunc['db_query']('', '
  373. SELECT relname, relpages * 8 *1024 AS "KB" FROM pg_class
  374. WHERE relname = {string:messages} OR relname = {string:log_search_words}
  375. ORDER BY relpages DESC',
  376. array(
  377. 'messages' => $db_prefix. 'messages',
  378. 'log_search_words' => $db_prefix. 'log_search_words',
  379. )
  380. );
  381. if ($request !== false && $smcFunc['db_num_rows']($request) > 0)
  382. {
  383. while ($row = $smcFunc['db_fetch_assoc']($request))
  384. {
  385. if ($row['relname'] == $db_prefix . 'messages')
  386. {
  387. $context['table_info']['data_length'] = (int) $row['KB'];
  388. $context['table_info']['index_length'] = (int) $row['KB'];
  389. // Doesn't support fulltext
  390. $context['table_info']['fulltext_length'] = $txt['not_applicable'];
  391. }
  392. elseif ($row['relname'] == $db_prefix. 'log_search_words')
  393. {
  394. $context['table_info']['index_length'] = (int) $row['KB'];
  395. $context['table_info']['custom_index_length'] = (int) $row['KB'];
  396. }
  397. }
  398. $smcFunc['db_free_result']($request);
  399. }
  400. else
  401. // Didn't work for some reason...
  402. $context['table_info'] = array(
  403. 'data_length' => $txt['not_applicable'],
  404. 'index_length' => $txt['not_applicable'],
  405. 'fulltext_length' => $txt['not_applicable'],
  406. 'custom_index_length' => $txt['not_applicable'],
  407. );
  408. }
  409. else
  410. $context['table_info'] = array(
  411. 'data_length' => $txt['not_applicable'],
  412. 'index_length' => $txt['not_applicable'],
  413. 'fulltext_length' => $txt['not_applicable'],
  414. 'custom_index_length' => $txt['not_applicable'],
  415. );
  416. // Format the data and index length in kilobytes.
  417. foreach ($context['table_info'] as $type => $size)
  418. {
  419. // If it's not numeric then just break. This database engine doesn't support size.
  420. if (!is_numeric($size))
  421. break;
  422. $context['table_info'][$type] = comma_format($context['table_info'][$type] / 1024) . ' ' . $txt['search_method_kilobytes'];
  423. }
  424. $context['custom_index'] = !empty($modSettings['search_custom_index_config']);
  425. $context['partial_custom_index'] = !empty($modSettings['search_custom_index_resume']) && empty($modSettings['search_custom_index_config']);
  426. $context['double_index'] = !empty($context['fulltext_index']) && $context['custom_index'];
  427. }
  428. function CreateMessageIndex()
  429. {
  430. global $modSettings, $context, $smcFunc, $db_prefix, $txt;
  431. // Scotty, we need more time...
  432. @set_time_limit(600);
  433. if (function_exists('apache_reset_timeout'))
  434. @apache_reset_timeout();
  435. $context[$context['admin_menu_name']]['current_subsection'] = 'method';
  436. $context['page_title'] = $txt['search_index_custom'];
  437. $messages_per_batch = 50;
  438. $index_properties = array(
  439. 2 => array(
  440. 'column_definition' => 'small',
  441. 'step_size' => 1000000,
  442. ),
  443. 4 => array(
  444. 'column_definition' => 'medium',
  445. 'step_size' => 1000000,
  446. 'max_size' => 16777215,
  447. ),
  448. 5 => array(
  449. 'column_definition' => 'large',
  450. 'step_size' => 100000000,
  451. 'max_size' => 2000000000,
  452. ),
  453. );
  454. if (isset($_REQUEST['resume']) && !empty($modSettings['search_custom_index_resume']))
  455. {
  456. $context['index_settings'] = unserialize($modSettings['search_custom_index_resume']);
  457. $context['start'] = (int) $context['index_settings']['resume_at'];
  458. unset($context['index_settings']['resume_at']);
  459. $context['step'] = 1;
  460. }
  461. else
  462. {
  463. $context['index_settings'] = array(
  464. 'bytes_per_word' => isset($_REQUEST['bytes_per_word']) && isset($index_properties[$_REQUEST['bytes_per_word']]) ? (int) $_REQUEST['bytes_per_word'] : 2,
  465. );
  466. $context['start'] = isset($_REQUEST['start']) ? (int) $_REQUEST['start'] : 0;
  467. $context['step'] = isset($_REQUEST['step']) ? (int) $_REQUEST['step'] : 0;
  468. }
  469. if ($context['step'] !== 0)
  470. checkSession('request');
  471. // Step 0: let the user determine how they like their index.
  472. if ($context['step'] === 0)
  473. {
  474. $context['sub_template'] = 'create_index';
  475. }
  476. // Step 1: insert all the words.
  477. if ($context['step'] === 1)
  478. {
  479. $context['sub_template'] = 'create_index_progress';
  480. if ($context['start'] === 0)
  481. {
  482. db_extend();
  483. $tables = $smcFunc['db_list_tables'](false, $db_prefix . 'log_search_words');
  484. if (!empty($tables))
  485. {
  486. $smcFunc['db_search_query']('drop_words_table', '
  487. DROP TABLE {db_prefix}log_search_words',
  488. array(
  489. )
  490. );
  491. }
  492. $smcFunc['db_create_word_search']($index_properties[$context['index_settings']['bytes_per_word']]['column_definition']);
  493. // Temporarily switch back to not using a search index.
  494. if (!empty($modSettings['search_index']) && $modSettings['search_index'] == 'custom')
  495. updateSettings(array('search_index' => ''));
  496. // Don't let simultanious processes be updating the search index.
  497. if (!empty($modSettings['search_custom_index_config']))
  498. updateSettings(array('search_custom_index_config' => ''));
  499. }
  500. $num_messages = array(
  501. 'done' => 0,
  502. 'todo' => 0,
  503. );
  504. $request = $smcFunc['db_query']('', '
  505. SELECT id_msg >= {int:starting_id} AS todo, COUNT(*) AS num_messages
  506. FROM {db_prefix}messages
  507. GROUP BY todo',
  508. array(
  509. 'starting_id' => $context['start'],
  510. )
  511. );
  512. while ($row = $smcFunc['db_fetch_assoc']($request))
  513. $num_messages[empty($row['todo']) ? 'done' : 'todo'] = $row['num_messages'];
  514. if (empty($num_messages['todo']))
  515. {
  516. $context['step'] = 2;
  517. $context['percentage'] = 80;
  518. $context['start'] = 0;
  519. }
  520. else
  521. {
  522. // Number of seconds before the next step.
  523. $stop = time() + 3;
  524. while (time() < $stop)
  525. {
  526. $inserts = array();
  527. $request = $smcFunc['db_query']('', '
  528. SELECT id_msg, body
  529. FROM {db_prefix}messages
  530. WHERE id_msg BETWEEN {int:starting_id} AND {int:ending_id}
  531. LIMIT {int:limit}',
  532. array(
  533. 'starting_id' => $context['start'],
  534. 'ending_id' => $context['start'] + $messages_per_batch - 1,
  535. 'limit' => $messages_per_batch,
  536. )
  537. );
  538. $forced_break = false;
  539. $number_processed = 0;
  540. while ($row = $smcFunc['db_fetch_assoc']($request))
  541. {
  542. // In theory it's possible for one of these to take friggin ages so add more timeout protection.
  543. if ($stop < time())
  544. {
  545. $forced_break = true;
  546. break;
  547. }
  548. $number_processed++;
  549. foreach (text2words($row['body'], $context['index_settings']['bytes_per_word'], true) as $id_word)
  550. {
  551. $inserts[] = array($id_word, $row['id_msg']);
  552. }
  553. }
  554. $num_messages['done'] += $number_processed;
  555. $num_messages['todo'] -= $number_processed;
  556. $smcFunc['db_free_result']($request);
  557. $context['start'] += $forced_break ? $number_processed : $messages_per_batch;
  558. if (!empty($inserts))
  559. $smcFunc['db_insert']('ignore',
  560. '{db_prefix}log_search_words',
  561. array('id_word' => 'int', 'id_msg' => 'int'),
  562. $inserts,
  563. array('id_word', 'id_msg')
  564. );
  565. if ($num_messages['todo'] === 0)
  566. {
  567. $context['step'] = 2;
  568. $context['start'] = 0;
  569. break;
  570. }
  571. else
  572. updateSettings(array('search_custom_index_resume' => serialize(array_merge($context['index_settings'], array('resume_at' => $context['start'])))));
  573. }
  574. // Since there are still two steps to go, 90% is the maximum here.
  575. $context['percentage'] = round($num_messages['done'] / ($num_messages['done'] + $num_messages['todo']), 3) * 80;
  576. }
  577. }
  578. // Step 2: removing the words that occur too often and are of no use.
  579. elseif ($context['step'] === 2)
  580. {
  581. if ($context['index_settings']['bytes_per_word'] < 4)
  582. $context['step'] = 3;
  583. else
  584. {
  585. $stop_words = $context['start'] === 0 || empty($modSettings['search_stopwords']) ? array() : explode(',', $modSettings['search_stopwords']);
  586. $stop = time() + 3;
  587. $context['sub_template'] = 'create_index_progress';
  588. $max_messages = ceil(60 * $modSettings['totalMessages'] / 100);
  589. while (time() < $stop)
  590. {
  591. $request = $smcFunc['db_query']('', '
  592. SELECT id_word, COUNT(id_word) AS num_words
  593. FROM {db_prefix}log_search_words
  594. WHERE id_word BETWEEN {int:starting_id} AND {int:ending_id}
  595. GROUP BY id_word
  596. HAVING COUNT(id_word) > {int:minimum_messages}',
  597. array(
  598. 'starting_id' => $context['start'],
  599. 'ending_id' => $context['start'] + $index_properties[$context['index_settings']['bytes_per_word']]['step_size'] - 1,
  600. 'minimum_messages' => $max_messages,
  601. )
  602. );
  603. while ($row = $smcFunc['db_fetch_assoc']($request))
  604. $stop_words[] = $row['id_word'];
  605. $smcFunc['db_free_result']($request);
  606. updateSettings(array('search_stopwords' => implode(',', $stop_words)));
  607. if (!empty($stop_words))
  608. $smcFunc['db_query']('', '
  609. DELETE FROM {db_prefix}log_search_words
  610. WHERE id_word in ({array_int:stop_words})',
  611. array(
  612. 'stop_words' => $stop_words,
  613. )
  614. );
  615. $context['start'] += $index_properties[$context['index_settings']['bytes_per_word']]['step_size'];
  616. if ($context['start'] > $index_properties[$context['index_settings']['bytes_per_word']]['max_size'])
  617. {
  618. $context['step'] = 3;
  619. break;
  620. }
  621. }
  622. $context['percentage'] = 80 + round($context['start'] / $index_properties[$context['index_settings']['bytes_per_word']]['max_size'], 3) * 20;
  623. }
  624. }
  625. // Step 3: remove words not distinctive enough.
  626. if ($context['step'] === 3)
  627. {
  628. $context['sub_template'] = 'create_index_done';
  629. updateSettings(array('search_index' => 'custom', 'search_custom_index_config' => serialize($context['index_settings'])));
  630. $smcFunc['db_query']('', '
  631. DELETE FROM {db_prefix}settings
  632. WHERE variable = {string:search_custom_index_resume}',
  633. array(
  634. 'search_custom_index_resume' => 'search_custom_index_resume',
  635. )
  636. );
  637. }
  638. }
  639. // Get the installed APIs.
  640. function loadSearchAPIs()
  641. {
  642. global $sourcedir, $txt;
  643. $apis = array();
  644. if ($dh = opendir($sourcedir))
  645. {
  646. while (($file = readdir($dh)) !== false)
  647. {
  648. if (is_file($sourcedir . '/' . $file) && preg_match('~SearchAPI-([A-Za-z\d_]+)\.php~', $file, $matches))
  649. {
  650. // Check this is definitely a valid API!
  651. $fp = fopen($sourcedir . '/' . $file, 'rb');
  652. $header = fread($fp, 4096);
  653. fclose($fp);
  654. if (strpos($header, '* SearchAPI-' . $matches[1] . '.php') !== false)
  655. {
  656. loadClassFile($file);
  657. $index_name = strtolower($matches[1]);
  658. $search_class_name = $index_name . '_search';
  659. $searchAPI = new $search_class_name();
  660. // No Support? NEXT!
  661. if (!$searchAPI->is_supported)
  662. continue;
  663. $apis[$index_name] = array(
  664. 'filename' => $file,
  665. 'setting_index' => $index_name,
  666. 'has_template' => in_array($index_name, array('custom', 'fulltext', 'standard')),
  667. 'label' => $index_name && isset($txt['search_index_' . $index_name]) ? $txt['search_index_' . $index_name] : '',
  668. 'desc' => $index_name && isset($txt['search_index_' . $index_name . '_desc']) ? $txt['search_index_' . $index_name . '_desc'] : '',
  669. );
  670. }
  671. }
  672. }
  673. }
  674. closedir($dh);
  675. return $apis;
  676. }
  677. ?>