Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
16.93% |
149 / 880 |
|
4.76% |
1 / 21 |
CRAP | |
0.00% |
0 / 1 |
| fulltext_native | |
16.93% |
149 / 880 |
|
4.76% |
1 / 21 |
32980.60 | |
0.00% |
0 / 1 |
| __construct | |
91.67% |
11 / 12 |
|
0.00% |
0 / 1 |
2.00 | |||
| get_name | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| is_available | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| init | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| get_search_query | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| get_common_words | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| get_word_length | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| split_keywords | |
61.21% |
101 / 165 |
|
0.00% |
0 / 1 |
311.56 | |||
| keyword_search | |
0.00% |
0 / 255 |
|
0.00% |
0 / 1 |
3192 | |||
| author_search | |
0.00% |
0 / 158 |
|
0.00% |
0 / 1 |
1482 | |||
| supports_phrase_search | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| index | |
0.00% |
0 / 98 |
|
0.00% |
0 / 1 |
272 | |||
| index_remove | |
0.00% |
0 / 29 |
|
0.00% |
0 / 1 |
42 | |||
| tidy | |
0.00% |
0 / 28 |
|
0.00% |
0 / 1 |
56 | |||
| delete_index | |
0.00% |
0 / 14 |
|
0.00% |
0 / 1 |
6 | |||
| index_created | |
0.00% |
0 / 3 |
|
0.00% |
0 / 1 |
12 | |||
| index_stats | |
0.00% |
0 / 5 |
|
0.00% |
0 / 1 |
6 | |||
| get_stats | |
0.00% |
0 / 2 |
|
0.00% |
0 / 1 |
2 | |||
| split_message | |
0.00% |
0 / 20 |
|
0.00% |
0 / 1 |
132 | |||
| cleanup | |
51.43% |
36 / 70 |
|
0.00% |
0 / 1 |
65.84 | |||
| get_acp_options | |
0.00% |
0 / 14 |
|
0.00% |
0 / 1 |
12 | |||
| 1 | <?php |
| 2 | /** |
| 3 | * |
| 4 | * This file is part of the phpBB Forum Software package. |
| 5 | * |
| 6 | * @copyright (c) phpBB Limited <https://www.phpbb.com> |
| 7 | * @license GNU General Public License, version 2 (GPL-2.0) |
| 8 | * |
| 9 | * For full copyright and license information, please see |
| 10 | * the docs/CREDITS.txt file. |
| 11 | * |
| 12 | */ |
| 13 | |
| 14 | namespace phpbb\search\backend; |
| 15 | |
| 16 | use phpbb\config\config; |
| 17 | use phpbb\db\driver\driver_interface; |
| 18 | use phpbb\db\tools\tools_interface; |
| 19 | use phpbb\event\dispatcher_interface; |
| 20 | use phpbb\language\language; |
| 21 | use phpbb\user; |
| 22 | |
| 23 | /** |
| 24 | * phpBB's own db driven fulltext search, version 2 |
| 25 | */ |
| 26 | class fulltext_native extends base implements search_backend_interface |
| 27 | { |
| 28 | protected const UTF8_HANGUL_FIRST = "\xEA\xB0\x80"; |
| 29 | protected const UTF8_HANGUL_LAST = "\xED\x9E\xA3"; |
| 30 | protected const UTF8_CJK_FIRST = "\xE4\xB8\x80"; |
| 31 | protected const UTF8_CJK_LAST = "\xE9\xBE\xBB"; |
| 32 | protected const UTF8_CJK_B_FIRST = "\xF0\xA0\x80\x80"; |
| 33 | protected const UTF8_CJK_B_LAST = "\xF0\xAA\x9B\x96"; |
| 34 | |
| 35 | /** |
| 36 | * Associative array holding index stats |
| 37 | * @var array |
| 38 | */ |
| 39 | protected $stats = array(); |
| 40 | |
| 41 | /** |
| 42 | * Associative array stores the min and max word length to be searched |
| 43 | * @var array |
| 44 | */ |
| 45 | protected $word_length = array(); |
| 46 | |
| 47 | /** |
| 48 | * Contains tidied search query. |
| 49 | * Operators are prefixed in search query and common words excluded |
| 50 | * @var string |
| 51 | */ |
| 52 | protected $search_query = ''; |
| 53 | |
| 54 | /** |
| 55 | * Contains common words. |
| 56 | * Common words are words with length less/more than min/max length |
| 57 | * @var array |
| 58 | */ |
| 59 | protected $common_words = array(); |
| 60 | |
| 61 | /** |
| 62 | * Post ids of posts containing words that are to be included |
| 63 | * @var array |
| 64 | */ |
| 65 | protected $must_contain_ids = array(); |
| 66 | |
| 67 | /** |
| 68 | * Post ids of posts containing words that should not be included |
| 69 | * @var array |
| 70 | */ |
| 71 | protected $must_not_contain_ids = array(); |
| 72 | |
| 73 | /** |
| 74 | * Post ids of posts containing at least one word that needs to be excluded |
| 75 | * @var array |
| 76 | */ |
| 77 | protected $must_exclude_one_ids = array(); |
| 78 | |
| 79 | /** |
| 80 | * Relative path to board root |
| 81 | * @var string |
| 82 | */ |
| 83 | protected $phpbb_root_path; |
| 84 | |
| 85 | /** |
| 86 | * PHP Extension |
| 87 | * @var string |
| 88 | */ |
| 89 | protected $php_ext; |
| 90 | |
| 91 | /** |
| 92 | * DBAL tools |
| 93 | * @var tools_interface |
| 94 | */ |
| 95 | protected $db_tools; |
| 96 | |
| 97 | /** |
| 98 | * phpBB event dispatcher object |
| 99 | * @var dispatcher_interface |
| 100 | */ |
| 101 | protected $phpbb_dispatcher; |
| 102 | |
| 103 | /** @var language */ |
| 104 | protected $language; |
| 105 | |
| 106 | /** @var string */ |
| 107 | protected $search_wordlist_table; |
| 108 | |
| 109 | /** @var string */ |
| 110 | protected $search_wordmatch_table; |
| 111 | |
| 112 | /** |
| 113 | * Initialises the fulltext_native search backend with min/max word length |
| 114 | * |
| 115 | * @param config $config Config object |
| 116 | * @param driver_interface $db Database object |
| 117 | * @param tools_interface $db_tools Database tools |
| 118 | * @param dispatcher_interface $phpbb_dispatcher Event dispatcher object |
| 119 | * @param language $language |
| 120 | * @param user $user User object |
| 121 | * @param string $search_results_table |
| 122 | * @param string $search_wordlist_table |
| 123 | * @param string $search_wordmatch_table |
| 124 | * @param string $phpbb_root_path phpBB root path |
| 125 | * @param string $phpEx PHP file extension |
| 126 | */ |
| 127 | public function __construct(config $config, driver_interface $db, tools_interface $db_tools, dispatcher_interface $phpbb_dispatcher, |
| 128 | language $language, user $user, string $search_results_table, string $search_wordlist_table, |
| 129 | string $search_wordmatch_table, string $phpbb_root_path, string $phpEx) |
| 130 | { |
| 131 | global $cache; |
| 132 | |
| 133 | parent::__construct($cache, $config, $db, $user, $search_results_table); |
| 134 | $this->db_tools = $db_tools; |
| 135 | $this->phpbb_dispatcher = $phpbb_dispatcher; |
| 136 | $this->language = $language; |
| 137 | |
| 138 | $this->search_wordlist_table = $search_wordlist_table; |
| 139 | $this->search_wordmatch_table = $search_wordmatch_table; |
| 140 | |
| 141 | $this->phpbb_root_path = $phpbb_root_path; |
| 142 | $this->php_ext = $phpEx; |
| 143 | |
| 144 | $this->word_length = array('min' => (int) $this->config['fulltext_native_min_chars'], 'max' => (int) $this->config['fulltext_native_max_chars']); |
| 145 | |
| 146 | /** |
| 147 | * Load the UTF tools |
| 148 | */ |
| 149 | if (!function_exists('utf8_decode_ncr')) |
| 150 | { |
| 151 | include($this->phpbb_root_path . 'includes/utf/utf_tools.' . $this->php_ext); |
| 152 | } |
| 153 | } |
| 154 | |
| 155 | /** |
| 156 | * {@inheritdoc} |
| 157 | */ |
| 158 | public function get_name(): string |
| 159 | { |
| 160 | return 'phpBB Native Fulltext'; |
| 161 | } |
| 162 | |
| 163 | /** |
| 164 | * {@inheritdoc} |
| 165 | */ |
| 166 | public function is_available(): bool |
| 167 | { |
| 168 | return true; |
| 169 | } |
| 170 | |
| 171 | /** |
| 172 | * {@inheritdoc} |
| 173 | */ |
| 174 | public function init() |
| 175 | { |
| 176 | return false; |
| 177 | } |
| 178 | |
| 179 | /** |
| 180 | * {@inheritdoc} |
| 181 | */ |
| 182 | public function get_search_query(): string |
| 183 | { |
| 184 | return $this->search_query; |
| 185 | } |
| 186 | |
| 187 | /** |
| 188 | * {@inheritdoc} |
| 189 | */ |
| 190 | public function get_common_words(): array |
| 191 | { |
| 192 | return $this->common_words; |
| 193 | } |
| 194 | |
| 195 | /** |
| 196 | * {@inheritdoc} |
| 197 | */ |
| 198 | public function get_word_length() |
| 199 | { |
| 200 | return $this->word_length; |
| 201 | } |
| 202 | |
| 203 | /** |
| 204 | * {@inheritdoc} |
| 205 | */ |
| 206 | public function split_keywords(string &$keywords, string $terms): bool |
| 207 | { |
| 208 | $tokens = '+-|()* '; |
| 209 | |
| 210 | $keywords = trim($this->cleanup($keywords, $tokens)); |
| 211 | |
| 212 | // allow word|word|word without brackets |
| 213 | if ((strpos($keywords, ' ') === false) && (strpos($keywords, '|') !== false) && (strpos($keywords, '(') === false)) |
| 214 | { |
| 215 | $keywords = '(' . $keywords . ')'; |
| 216 | } |
| 217 | |
| 218 | $open_bracket = $space = false; |
| 219 | for ($i = 0, $n = strlen($keywords); $i < $n; $i++) |
| 220 | { |
| 221 | if ($open_bracket !== false) |
| 222 | { |
| 223 | switch ($keywords[$i]) |
| 224 | { |
| 225 | case ')': |
| 226 | if ($open_bracket + 1 == $i) |
| 227 | { |
| 228 | $keywords[$i - 1] = '|'; |
| 229 | $keywords[$i] = '|'; |
| 230 | } |
| 231 | $open_bracket = false; |
| 232 | break; |
| 233 | case '(': |
| 234 | $keywords[$i] = '|'; |
| 235 | break; |
| 236 | case '+': |
| 237 | case '-': |
| 238 | case ' ': |
| 239 | $keywords[$i] = '|'; |
| 240 | break; |
| 241 | case '*': |
| 242 | // $i can never be 0 here since $open_bracket is initialised to false |
| 243 | if (strpos($tokens, $keywords[$i - 1]) !== false && ($i + 1 === $n || strpos($tokens, $keywords[$i + 1]) !== false)) |
| 244 | { |
| 245 | $keywords[$i] = '|'; |
| 246 | } |
| 247 | break; |
| 248 | } |
| 249 | } |
| 250 | else |
| 251 | { |
| 252 | switch ($keywords[$i]) |
| 253 | { |
| 254 | case ')': |
| 255 | $keywords[$i] = ' '; |
| 256 | break; |
| 257 | case '(': |
| 258 | $open_bracket = $i; |
| 259 | $space = false; |
| 260 | break; |
| 261 | case '|': |
| 262 | $keywords[$i] = ' '; |
| 263 | break; |
| 264 | case '-': |
| 265 | // Ignore hyphen if followed by a space |
| 266 | if (isset($keywords[$i + 1]) && $keywords[$i + 1] == ' ') |
| 267 | { |
| 268 | $keywords[$i] = ' '; |
| 269 | } |
| 270 | else |
| 271 | { |
| 272 | $space = $keywords[$i]; |
| 273 | } |
| 274 | break; |
| 275 | case '+': |
| 276 | $space = $keywords[$i]; |
| 277 | break; |
| 278 | case ' ': |
| 279 | if ($space !== false) |
| 280 | { |
| 281 | $keywords[$i] = $space; |
| 282 | } |
| 283 | break; |
| 284 | default: |
| 285 | $space = false; |
| 286 | } |
| 287 | } |
| 288 | } |
| 289 | |
| 290 | if ($open_bracket !== false) |
| 291 | { |
| 292 | $keywords .= ')'; |
| 293 | } |
| 294 | |
| 295 | $match = array( |
| 296 | '# +#', |
| 297 | '#\|\|+#', |
| 298 | '#(\+|\-)(?:\+|\-)+#', |
| 299 | '#\(\|#', |
| 300 | '#\|\)#', |
| 301 | ); |
| 302 | $replace = array( |
| 303 | ' ', |
| 304 | '|', |
| 305 | '$1', |
| 306 | '(', |
| 307 | ')', |
| 308 | ); |
| 309 | |
| 310 | $keywords = preg_replace($match, $replace, $keywords); |
| 311 | $num_keywords = count(explode(' ', $keywords)); |
| 312 | |
| 313 | // We limit the number of allowed keywords to minimize load on the database |
| 314 | if ($this->config['max_num_search_keywords'] && $num_keywords > $this->config['max_num_search_keywords']) |
| 315 | { |
| 316 | trigger_error($this->language->lang('MAX_NUM_SEARCH_KEYWORDS_REFINE', (int) $this->config['max_num_search_keywords'], $num_keywords)); |
| 317 | } |
| 318 | |
| 319 | // $keywords input format: each word separated by a space, words in a bracket are not separated |
| 320 | |
| 321 | // the user wants to search for any word, convert the search query |
| 322 | if ($terms == 'any') |
| 323 | { |
| 324 | $words = array(); |
| 325 | |
| 326 | preg_match_all('#([^\\s+\\-|()]+)(?:$|[\\s+\\-|()])#u', $keywords, $words); |
| 327 | if (count($words[1])) |
| 328 | { |
| 329 | $keywords = '(' . implode('|', $words[1]) . ')'; |
| 330 | } |
| 331 | } |
| 332 | |
| 333 | // Remove non trailing wildcards from each word to prevent a full table scan (it's now using the database index) |
| 334 | $match = '#\*(?!$|\s)#'; |
| 335 | $replace = '$1'; |
| 336 | $keywords = preg_replace($match, $replace, $keywords); |
| 337 | |
| 338 | // Only allow one wildcard in the search query to limit the database load |
| 339 | $match = '#\*#'; |
| 340 | $replace = '$1'; |
| 341 | $count_wildcards = substr_count($keywords, '*'); |
| 342 | |
| 343 | // Reverse the string to remove all wildcards except the first one |
| 344 | $keywords = strrev(preg_replace($match, $replace, strrev($keywords), $count_wildcards - 1)); |
| 345 | unset($count_wildcards); |
| 346 | |
| 347 | // set the search_query which is shown to the user |
| 348 | $this->search_query = $keywords; |
| 349 | |
| 350 | $exact_words = array(); |
| 351 | preg_match_all('#([^\\s+\\-|()]+)(?:$|[\\s+\\-|()])#u', $keywords, $exact_words); |
| 352 | $exact_words = $exact_words[1]; |
| 353 | |
| 354 | $common_ids = $words = array(); |
| 355 | |
| 356 | if (count($exact_words)) |
| 357 | { |
| 358 | $sql = 'SELECT word_id, word_text, word_common |
| 359 | FROM ' . $this->search_wordlist_table . ' |
| 360 | WHERE ' . $this->db->sql_in_set('word_text', $exact_words) . ' |
| 361 | ORDER BY word_count ASC'; |
| 362 | $result = $this->db->sql_query($sql); |
| 363 | |
| 364 | // store an array of words and ids, remove common words |
| 365 | while ($row = $this->db->sql_fetchrow($result)) |
| 366 | { |
| 367 | if ($row['word_common']) |
| 368 | { |
| 369 | $this->common_words[] = $row['word_text']; |
| 370 | $common_ids[$row['word_text']] = (int) $row['word_id']; |
| 371 | continue; |
| 372 | } |
| 373 | |
| 374 | $words[$row['word_text']] = (int) $row['word_id']; |
| 375 | } |
| 376 | $this->db->sql_freeresult($result); |
| 377 | } |
| 378 | |
| 379 | // Handle +, - without preceding whitespace character |
| 380 | $match = array('#(\S)\+#', '#(\S)-#'); |
| 381 | $replace = array('$1 +', '$1 +'); |
| 382 | |
| 383 | $keywords = preg_replace($match, $replace, $keywords); |
| 384 | |
| 385 | // now analyse the search query, first split it using the spaces |
| 386 | $query = explode(' ', $keywords); |
| 387 | |
| 388 | $this->must_contain_ids = array(); |
| 389 | $this->must_not_contain_ids = array(); |
| 390 | $this->must_exclude_one_ids = array(); |
| 391 | |
| 392 | foreach ($query as $word) |
| 393 | { |
| 394 | if (empty($word)) |
| 395 | { |
| 396 | continue; |
| 397 | } |
| 398 | |
| 399 | // words which should not be included |
| 400 | if ($word[0] == '-') |
| 401 | { |
| 402 | $word = substr($word, 1); |
| 403 | |
| 404 | // a group of which at least one may not be in the resulting posts |
| 405 | if (isset($word[0]) && $word[0] == '(') |
| 406 | { |
| 407 | $word = array_unique(explode('|', substr($word, 1, -1))); |
| 408 | $mode = 'must_exclude_one'; |
| 409 | } |
| 410 | // one word which should not be in the resulting posts |
| 411 | else |
| 412 | { |
| 413 | $mode = 'must_not_contain'; |
| 414 | } |
| 415 | $ignore_no_id = true; |
| 416 | } |
| 417 | // words which have to be included |
| 418 | else |
| 419 | { |
| 420 | // no prefix is the same as a +prefix |
| 421 | if ($word[0] == '+') |
| 422 | { |
| 423 | $word = substr($word, 1); |
| 424 | } |
| 425 | |
| 426 | // a group of words of which at least one word should be in every resulting post |
| 427 | if (isset($word[0]) && $word[0] == '(') |
| 428 | { |
| 429 | $word = array_unique(explode('|', substr($word, 1, -1))); |
| 430 | } |
| 431 | $ignore_no_id = false; |
| 432 | $mode = 'must_contain'; |
| 433 | } |
| 434 | |
| 435 | if (empty($word)) |
| 436 | { |
| 437 | continue; |
| 438 | } |
| 439 | |
| 440 | // if this is an array of words then retrieve an id for each |
| 441 | if (is_array($word)) |
| 442 | { |
| 443 | $non_common_words = array(); |
| 444 | $id_words = array(); |
| 445 | foreach ($word as $i => $word_part) |
| 446 | { |
| 447 | if (strpos($word_part, '*') !== false) |
| 448 | { |
| 449 | $len = utf8_strlen(str_replace('*', '', $word_part)); |
| 450 | if ($len >= $this->word_length['min'] && $len <= $this->word_length['max']) |
| 451 | { |
| 452 | $id_words[] = '\'' . $this->db->sql_escape(str_replace('*', '%', $word_part)) . '\''; |
| 453 | $non_common_words[] = $word_part; |
| 454 | } |
| 455 | else |
| 456 | { |
| 457 | $this->common_words[] = $word_part; |
| 458 | } |
| 459 | } |
| 460 | else if (isset($words[$word_part])) |
| 461 | { |
| 462 | $id_words[] = $words[$word_part]; |
| 463 | $non_common_words[] = $word_part; |
| 464 | } |
| 465 | else |
| 466 | { |
| 467 | $len = utf8_strlen($word_part); |
| 468 | if ($len < $this->word_length['min'] || $len > $this->word_length['max']) |
| 469 | { |
| 470 | $this->common_words[] = $word_part; |
| 471 | } |
| 472 | } |
| 473 | } |
| 474 | if (count($id_words)) |
| 475 | { |
| 476 | sort($id_words); |
| 477 | if (count($id_words) > 1) |
| 478 | { |
| 479 | $this->{$mode . '_ids'}[] = $id_words; |
| 480 | } |
| 481 | else |
| 482 | { |
| 483 | $mode = ($mode == 'must_exclude_one') ? 'must_not_contain' : $mode; |
| 484 | $this->{$mode . '_ids'}[] = $id_words[0]; |
| 485 | } |
| 486 | } |
| 487 | // throw an error if we shall not ignore unexistant words |
| 488 | else if (!$ignore_no_id && count($non_common_words)) |
| 489 | { |
| 490 | trigger_error(sprintf($this->language->lang('WORDS_IN_NO_POST'), implode($this->language->lang('COMMA_SEPARATOR'), $non_common_words))); |
| 491 | } |
| 492 | unset($non_common_words); |
| 493 | } |
| 494 | // else we only need one id |
| 495 | else if (($wildcard = strpos($word, '*') !== false) || isset($words[$word])) |
| 496 | { |
| 497 | if ($wildcard) |
| 498 | { |
| 499 | $len = utf8_strlen(str_replace('*', '', $word)); |
| 500 | if ($len >= $this->word_length['min'] && $len <= $this->word_length['max']) |
| 501 | { |
| 502 | $this->{$mode . '_ids'}[] = '\'' . $this->db->sql_escape(str_replace('*', '%', $word)) . '\''; |
| 503 | } |
| 504 | else |
| 505 | { |
| 506 | $this->common_words[] = $word; |
| 507 | } |
| 508 | } |
| 509 | else |
| 510 | { |
| 511 | $this->{$mode . '_ids'}[] = $words[$word]; |
| 512 | } |
| 513 | } |
| 514 | else |
| 515 | { |
| 516 | if (!isset($common_ids[$word])) |
| 517 | { |
| 518 | $len = utf8_strlen($word); |
| 519 | if ($len < $this->word_length['min'] || $len > $this->word_length['max']) |
| 520 | { |
| 521 | $this->common_words[] = $word; |
| 522 | } |
| 523 | } |
| 524 | } |
| 525 | } |
| 526 | |
| 527 | // Return true if all words are not common words |
| 528 | if (count($exact_words) - count($this->common_words) > 0) |
| 529 | { |
| 530 | return true; |
| 531 | } |
| 532 | return false; |
| 533 | } |
| 534 | |
| 535 | /** |
| 536 | * {@inheritdoc} |
| 537 | */ |
| 538 | public function keyword_search(string $type, string $fields, string $terms, array $sort_by_sql, string $sort_key, string $sort_dir, string $sort_days, array $ex_fid_ary, string $post_visibility, int $topic_id, array $author_ary, string $author_name, array &$id_ary, int &$start, int $per_page) |
| 539 | { |
| 540 | // No keywords? No posts. |
| 541 | if (empty($this->search_query)) |
| 542 | { |
| 543 | return false; |
| 544 | } |
| 545 | |
| 546 | // we can't search for negatives only |
| 547 | if (empty($this->must_contain_ids)) |
| 548 | { |
| 549 | return false; |
| 550 | } |
| 551 | |
| 552 | $must_contain_ids = $this->must_contain_ids; |
| 553 | $must_not_contain_ids = $this->must_not_contain_ids; |
| 554 | $must_exclude_one_ids = $this->must_exclude_one_ids; |
| 555 | |
| 556 | sort($must_contain_ids); |
| 557 | sort($must_not_contain_ids); |
| 558 | sort($must_exclude_one_ids); |
| 559 | |
| 560 | // generate a search_key from all the options to identify the results |
| 561 | $search_key_array = array( |
| 562 | serialize($must_contain_ids), |
| 563 | serialize($must_not_contain_ids), |
| 564 | serialize($must_exclude_one_ids), |
| 565 | $type, |
| 566 | $fields, |
| 567 | $terms, |
| 568 | $sort_days, |
| 569 | $sort_key, |
| 570 | $topic_id, |
| 571 | implode(',', $ex_fid_ary), |
| 572 | $post_visibility, |
| 573 | implode(',', $author_ary), |
| 574 | $author_name, |
| 575 | ); |
| 576 | |
| 577 | /** |
| 578 | * Allow changing the search_key for cached results |
| 579 | * |
| 580 | * @event core.search_native_by_keyword_modify_search_key |
| 581 | * @var array search_key_array Array with search parameters to generate the search_key |
| 582 | * @var array must_contain_ids Array with post ids of posts containing words that are to be included |
| 583 | * @var array must_not_contain_ids Array with post ids of posts containing words that should not be included |
| 584 | * @var array must_exclude_one_ids Array with post ids of posts containing at least one word that needs to be excluded |
| 585 | * @var string type Searching type ('posts', 'topics') |
| 586 | * @var string fields Searching fields ('titleonly', 'msgonly', 'firstpost', 'all') |
| 587 | * @var string terms Searching terms ('all', 'any') |
| 588 | * @var int sort_days Time, in days, of the oldest possible post to list |
| 589 | * @var string sort_key The sort type used from the possible sort types |
| 590 | * @var int topic_id Limit the search to this topic_id only |
| 591 | * @var array ex_fid_ary Which forums not to search on |
| 592 | * @var string post_visibility Post visibility data |
| 593 | * @var array author_ary Array of user_id containing the users to filter the results to |
| 594 | * @since 3.1.7-RC1 |
| 595 | */ |
| 596 | $vars = array( |
| 597 | 'search_key_array', |
| 598 | 'must_contain_ids', |
| 599 | 'must_not_contain_ids', |
| 600 | 'must_exclude_one_ids', |
| 601 | 'type', |
| 602 | 'fields', |
| 603 | 'terms', |
| 604 | 'sort_days', |
| 605 | 'sort_key', |
| 606 | 'topic_id', |
| 607 | 'ex_fid_ary', |
| 608 | 'post_visibility', |
| 609 | 'author_ary', |
| 610 | ); |
| 611 | extract($this->phpbb_dispatcher->trigger_event('core.search_native_by_keyword_modify_search_key', compact($vars))); |
| 612 | |
| 613 | $search_key = md5(implode('#', $search_key_array)); |
| 614 | |
| 615 | // try reading the results from cache |
| 616 | $total_results = 0; |
| 617 | if ($this->obtain_ids($search_key, $total_results, $id_ary, $start, $per_page, $sort_dir) == self::SEARCH_RESULT_IN_CACHE) |
| 618 | { |
| 619 | return $total_results; |
| 620 | } |
| 621 | |
| 622 | $id_ary = array(); |
| 623 | |
| 624 | $sql_where = array(); |
| 625 | $m_num = 0; |
| 626 | $w_num = 0; |
| 627 | |
| 628 | $sql_array = array( |
| 629 | 'SELECT' => ($type == 'posts') ? 'DISTINCT p.post_id' : 'DISTINCT p.topic_id', |
| 630 | 'FROM' => array( |
| 631 | $this->search_wordmatch_table => array(), |
| 632 | $this->search_wordlist_table => array(), |
| 633 | ), |
| 634 | 'LEFT_JOIN' => array(array( |
| 635 | 'FROM' => array(POSTS_TABLE => 'p'), |
| 636 | 'ON' => 'm0.post_id = p.post_id', |
| 637 | )), |
| 638 | ); |
| 639 | |
| 640 | $title_match = ''; |
| 641 | $left_join_topics = false; |
| 642 | $group_by = true; |
| 643 | // Build some display specific sql strings |
| 644 | switch ($fields) |
| 645 | { |
| 646 | case 'titleonly': |
| 647 | $title_match = 'title_match = 1'; |
| 648 | $group_by = false; |
| 649 | // no break |
| 650 | case 'firstpost': |
| 651 | $left_join_topics = true; |
| 652 | $sql_where[] = 'p.post_id = t.topic_first_post_id'; |
| 653 | break; |
| 654 | |
| 655 | case 'msgonly': |
| 656 | $title_match = 'title_match = 0'; |
| 657 | $group_by = false; |
| 658 | break; |
| 659 | } |
| 660 | |
| 661 | if ($type == 'topics') |
| 662 | { |
| 663 | $left_join_topics = true; |
| 664 | $group_by = true; |
| 665 | } |
| 666 | |
| 667 | /** |
| 668 | * @todo Add a query optimizer (handle stuff like "+(4|3) +4") |
| 669 | */ |
| 670 | |
| 671 | foreach ($this->must_contain_ids as $subquery) |
| 672 | { |
| 673 | if (is_array($subquery)) |
| 674 | { |
| 675 | $group_by = true; |
| 676 | |
| 677 | $word_id_sql = array(); |
| 678 | $word_ids = array(); |
| 679 | foreach ($subquery as $id) |
| 680 | { |
| 681 | if (is_string($id)) |
| 682 | { |
| 683 | $sql_array['LEFT_JOIN'][] = array( |
| 684 | 'FROM' => array($this->search_wordlist_table => 'w' . $w_num), |
| 685 | 'ON' => "w$w_num.word_text LIKE $id" |
| 686 | ); |
| 687 | $word_ids[] = "w$w_num.word_id"; |
| 688 | |
| 689 | $w_num++; |
| 690 | } |
| 691 | else |
| 692 | { |
| 693 | $word_ids[] = $id; |
| 694 | } |
| 695 | } |
| 696 | |
| 697 | $sql_where[] = $this->db->sql_in_set("m$m_num.word_id", $word_ids); |
| 698 | |
| 699 | unset($word_id_sql); |
| 700 | unset($word_ids); |
| 701 | } |
| 702 | else if (is_string($subquery)) |
| 703 | { |
| 704 | $sql_array['FROM'][$this->search_wordlist_table][] = 'w' . $w_num; |
| 705 | |
| 706 | $sql_where[] = "w$w_num.word_text LIKE $subquery"; |
| 707 | $sql_where[] = "m$m_num.word_id = w$w_num.word_id"; |
| 708 | |
| 709 | $group_by = true; |
| 710 | $w_num++; |
| 711 | } |
| 712 | else |
| 713 | { |
| 714 | $sql_where[] = "m$m_num.word_id = $subquery"; |
| 715 | } |
| 716 | |
| 717 | $sql_array['FROM'][$this->search_wordmatch_table][] = 'm' . $m_num; |
| 718 | |
| 719 | if ($title_match) |
| 720 | { |
| 721 | $sql_where[] = "m$m_num.$title_match"; |
| 722 | } |
| 723 | |
| 724 | if ($m_num != 0) |
| 725 | { |
| 726 | $sql_where[] = "m$m_num.post_id = m0.post_id"; |
| 727 | } |
| 728 | $m_num++; |
| 729 | } |
| 730 | |
| 731 | foreach ($this->must_not_contain_ids as $key => $subquery) |
| 732 | { |
| 733 | if (is_string($subquery)) |
| 734 | { |
| 735 | $sql_array['LEFT_JOIN'][] = array( |
| 736 | 'FROM' => array($this->search_wordlist_table => 'w' . $w_num), |
| 737 | 'ON' => "w$w_num.word_text LIKE $subquery" |
| 738 | ); |
| 739 | |
| 740 | $this->must_not_contain_ids[$key] = "w$w_num.word_id"; |
| 741 | |
| 742 | $group_by = true; |
| 743 | $w_num++; |
| 744 | } |
| 745 | } |
| 746 | |
| 747 | if (count($this->must_not_contain_ids)) |
| 748 | { |
| 749 | $sql_array['LEFT_JOIN'][] = array( |
| 750 | 'FROM' => array($this->search_wordmatch_table => 'm' . $m_num), |
| 751 | 'ON' => $this->db->sql_in_set("m$m_num.word_id", $this->must_not_contain_ids) . (($title_match) ? " AND m$m_num.$title_match" : '') . " AND m$m_num.post_id = m0.post_id" |
| 752 | ); |
| 753 | |
| 754 | $sql_where[] = "m$m_num.word_id IS NULL"; |
| 755 | $m_num++; |
| 756 | } |
| 757 | |
| 758 | foreach ($this->must_exclude_one_ids as $ids) |
| 759 | { |
| 760 | $is_null_joins = array(); |
| 761 | foreach ($ids as $id) |
| 762 | { |
| 763 | if (is_string($id)) |
| 764 | { |
| 765 | $sql_array['LEFT_JOIN'][] = array( |
| 766 | 'FROM' => array($this->search_wordlist_table => 'w' . $w_num), |
| 767 | 'ON' => "w$w_num.word_text LIKE $id" |
| 768 | ); |
| 769 | $id = "w$w_num.word_id"; |
| 770 | |
| 771 | $group_by = true; |
| 772 | $w_num++; |
| 773 | } |
| 774 | |
| 775 | $sql_array['LEFT_JOIN'][] = array( |
| 776 | 'FROM' => array($this->search_wordmatch_table => 'm' . $m_num), |
| 777 | 'ON' => "m$m_num.word_id = $id AND m$m_num.post_id = m0.post_id" . (($title_match) ? " AND m$m_num.$title_match" : '') |
| 778 | ); |
| 779 | $is_null_joins[] = "m$m_num.word_id IS NULL"; |
| 780 | |
| 781 | $m_num++; |
| 782 | } |
| 783 | $sql_where[] = '(' . implode(' OR ', $is_null_joins) . ')'; |
| 784 | } |
| 785 | |
| 786 | $sql_where[] = $post_visibility; |
| 787 | |
| 788 | $search_query = $this->search_query; |
| 789 | $must_exclude_one_ids = $this->must_exclude_one_ids; |
| 790 | $must_not_contain_ids = $this->must_not_contain_ids; |
| 791 | $must_contain_ids = $this->must_contain_ids; |
| 792 | |
| 793 | $sql_sort_table = $sql_sort_join = $sql_match = $sql_match_where = $sql_sort = ''; |
| 794 | |
| 795 | /** |
| 796 | * Allow changing the query used for counting for posts using fulltext_native |
| 797 | * |
| 798 | * @event core.search_native_keywords_count_query_before |
| 799 | * @var string search_query The parsed keywords used for this search |
| 800 | * @var array must_not_contain_ids Ids that cannot be taken into account for the results |
| 801 | * @var array must_exclude_one_ids Ids that cannot be on the results |
| 802 | * @var array must_contain_ids Ids that must be on the results |
| 803 | * @var int total_results The previous result count for the format of the query |
| 804 | * Set to 0 to force a re-count |
| 805 | * @var array sql_array The data on how to search in the DB at this point |
| 806 | * @var bool left_join_topics Whether or not TOPICS_TABLE should be CROSS JOIN'ED |
| 807 | * @var array author_ary Array of user_id containing the users to filter the results to |
| 808 | * @var string author_name An extra username to search on (!empty(author_ary) must be true, to be relevant) |
| 809 | * @var array ex_fid_ary Which forums not to search on |
| 810 | * @var int topic_id Limit the search to this topic_id only |
| 811 | * @var string sql_sort_table Extra tables to include in the SQL query. |
| 812 | * Used in conjunction with sql_sort_join |
| 813 | * @var string sql_sort_join SQL conditions to join all the tables used together. |
| 814 | * Used in conjunction with sql_sort_table |
| 815 | * @var int sort_days Time, in days, of the oldest possible post to list |
| 816 | * @var string sql_where An array of the current WHERE clause conditions |
| 817 | * @var string sql_match Which columns to do the search on |
| 818 | * @var string sql_match_where Extra conditions to use to properly filter the matching process |
| 819 | * @var bool group_by Whether or not the SQL query requires a GROUP BY for the elements in the SELECT clause |
| 820 | * @var string sort_by_sql The possible predefined sort types |
| 821 | * @var string sort_key The sort type used from the possible sort types |
| 822 | * @var string sort_dir "a" for ASC or "d" dor DESC for the sort order used |
| 823 | * @var string sql_sort The result SQL when processing sort_by_sql + sort_key + sort_dir |
| 824 | * @var int start How many posts to skip in the search results (used for pagination) |
| 825 | * @since 3.1.5-RC1 |
| 826 | */ |
| 827 | $vars = array( |
| 828 | 'search_query', |
| 829 | 'must_not_contain_ids', |
| 830 | 'must_exclude_one_ids', |
| 831 | 'must_contain_ids', |
| 832 | 'total_results', |
| 833 | 'sql_array', |
| 834 | 'left_join_topics', |
| 835 | 'author_ary', |
| 836 | 'author_name', |
| 837 | 'ex_fid_ary', |
| 838 | 'topic_id', |
| 839 | 'sql_sort_table', |
| 840 | 'sql_sort_join', |
| 841 | 'sort_days', |
| 842 | 'sql_where', |
| 843 | 'sql_match', |
| 844 | 'sql_match_where', |
| 845 | 'group_by', |
| 846 | 'sort_by_sql', |
| 847 | 'sort_key', |
| 848 | 'sort_dir', |
| 849 | 'sql_sort', |
| 850 | 'start', |
| 851 | ); |
| 852 | extract($this->phpbb_dispatcher->trigger_event('core.search_native_keywords_count_query_before', compact($vars))); |
| 853 | |
| 854 | if ($topic_id) |
| 855 | { |
| 856 | $sql_where[] = 'p.topic_id = ' . $topic_id; |
| 857 | } |
| 858 | |
| 859 | if (count($author_ary)) |
| 860 | { |
| 861 | if ($author_name) |
| 862 | { |
| 863 | // first one matches post of registered users, second one guests and deleted users |
| 864 | $sql_author = '(' . $this->db->sql_in_set('p.poster_id', array_diff($author_ary, array(ANONYMOUS)), false, true) . ' OR p.post_username ' . $author_name . ')'; |
| 865 | } |
| 866 | else |
| 867 | { |
| 868 | $sql_author = $this->db->sql_in_set('p.poster_id', $author_ary); |
| 869 | } |
| 870 | $sql_where[] = $sql_author; |
| 871 | } |
| 872 | |
| 873 | if (count($ex_fid_ary)) |
| 874 | { |
| 875 | $sql_where[] = $this->db->sql_in_set('p.forum_id', $ex_fid_ary, true); |
| 876 | } |
| 877 | |
| 878 | if ($sort_days) |
| 879 | { |
| 880 | $sql_where[] = 'p.post_time >= ' . (time() - ($sort_days * 86400)); |
| 881 | } |
| 882 | |
| 883 | $sql_array['WHERE'] = implode(' AND ', $sql_where); |
| 884 | |
| 885 | $is_mysql = false; |
| 886 | // if the total result count is not cached yet, retrieve it from the db |
| 887 | if (!$total_results) |
| 888 | { |
| 889 | $sql = ''; |
| 890 | $sql_array_count = $sql_array; |
| 891 | |
| 892 | if ($left_join_topics) |
| 893 | { |
| 894 | $sql_array_count['LEFT_JOIN'][] = array( |
| 895 | 'FROM' => array(TOPICS_TABLE => 't'), |
| 896 | 'ON' => 'p.topic_id = t.topic_id' |
| 897 | ); |
| 898 | } |
| 899 | |
| 900 | switch ($this->db->get_sql_layer()) |
| 901 | { |
| 902 | case 'mysqli': |
| 903 | $is_mysql = true; |
| 904 | |
| 905 | break; |
| 906 | |
| 907 | case 'sqlite3': |
| 908 | $sql_array_count['SELECT'] = ($type == 'posts') ? 'DISTINCT p.post_id' : 'DISTINCT p.topic_id'; |
| 909 | $sql = 'SELECT COUNT(' . (($type == 'posts') ? 'post_id' : 'topic_id') . ') as total_results |
| 910 | FROM (' . $this->db->sql_build_query('SELECT', $sql_array_count) . ')'; |
| 911 | |
| 912 | // no break |
| 913 | |
| 914 | default: |
| 915 | $sql_array_count['SELECT'] = ($type == 'posts') ? 'COUNT(DISTINCT p.post_id) AS total_results' : 'COUNT(DISTINCT p.topic_id) AS total_results'; |
| 916 | $sql = (!$sql) ? $this->db->sql_build_query('SELECT', $sql_array_count) : $sql; |
| 917 | |
| 918 | $result = $this->db->sql_query($sql); |
| 919 | $total_results = (int) $this->db->sql_fetchfield('total_results'); |
| 920 | $this->db->sql_freeresult($result); |
| 921 | |
| 922 | if (!$total_results) |
| 923 | { |
| 924 | return false; |
| 925 | } |
| 926 | break; |
| 927 | } |
| 928 | |
| 929 | unset($sql_array_count, $sql); |
| 930 | } |
| 931 | |
| 932 | // Build sql strings for sorting |
| 933 | $sql_sort = $sort_by_sql[$sort_key] . (($sort_dir == 'a') ? ' ASC' : ' DESC'); |
| 934 | |
| 935 | switch ($sql_sort[0]) |
| 936 | { |
| 937 | case 'u': |
| 938 | $sql_array['FROM'][USERS_TABLE] = 'u'; |
| 939 | $sql_where[] = 'u.user_id = p.poster_id '; |
| 940 | break; |
| 941 | |
| 942 | case 't': |
| 943 | $left_join_topics = true; |
| 944 | break; |
| 945 | |
| 946 | case 'f': |
| 947 | $sql_array['FROM'][FORUMS_TABLE] = 'f'; |
| 948 | $sql_where[] = 'f.forum_id = p.forum_id'; |
| 949 | break; |
| 950 | } |
| 951 | |
| 952 | if ($left_join_topics) |
| 953 | { |
| 954 | $sql_array['LEFT_JOIN'][] = array( |
| 955 | 'FROM' => array(TOPICS_TABLE => 't'), |
| 956 | 'ON' => 'p.topic_id = t.topic_id' |
| 957 | ); |
| 958 | } |
| 959 | |
| 960 | $sql_array['WHERE'] = implode(' AND ', $sql_where); |
| 961 | $sql_array['GROUP_BY'] = ($group_by) ? (($type == 'posts') ? 'p.post_id' : 'p.topic_id') . ', ' . $sort_by_sql[$sort_key] : ''; |
| 962 | $sql_array['ORDER_BY'] = $sql_sort; |
| 963 | $sql_array['SELECT'] .= $sort_by_sql[$sort_key] ? ", {$sort_by_sql[$sort_key]}" : ''; |
| 964 | |
| 965 | unset($sql_where, $sql_sort, $group_by); |
| 966 | |
| 967 | $sql = $this->db->sql_build_query('SELECT', $sql_array); |
| 968 | $result = $this->db->sql_query_limit($sql, $this->config['search_block_size'], $start); |
| 969 | |
| 970 | while ($row = $this->db->sql_fetchrow($result)) |
| 971 | { |
| 972 | $id_ary[] = (int) $row[(($type == 'posts') ? 'post_id' : 'topic_id')]; |
| 973 | } |
| 974 | $this->db->sql_freeresult($result); |
| 975 | |
| 976 | // If using mysql and the total result count is not calculated yet, get it from the db |
| 977 | if (!$total_results && $is_mysql) |
| 978 | { |
| 979 | $sql_count = str_replace("SELECT {$sql_array['SELECT']}", "SELECT COUNT({$sql_array['SELECT']}) as total_results", $sql); |
| 980 | $result = $this->db->sql_query($sql_count); |
| 981 | $total_results = $sql_array['GROUP_BY'] ? count($this->db->sql_fetchrowset($result)) : $this->db->sql_fetchfield('total_results'); |
| 982 | $this->db->sql_freeresult($result); |
| 983 | |
| 984 | if (!$total_results) |
| 985 | { |
| 986 | return false; |
| 987 | } |
| 988 | } |
| 989 | |
| 990 | if ($start >= $total_results) |
| 991 | { |
| 992 | $start = floor(($total_results - 1) / $per_page) * $per_page; |
| 993 | |
| 994 | $result = $this->db->sql_query_limit($sql, $this->config['search_block_size'], $start); |
| 995 | |
| 996 | while ($row = $this->db->sql_fetchrow($result)) |
| 997 | { |
| 998 | $id_ary[] = (int) $row[(($type == 'posts') ? 'post_id' : 'topic_id')]; |
| 999 | } |
| 1000 | $this->db->sql_freeresult($result); |
| 1001 | } |
| 1002 | |
| 1003 | // store the ids, from start on then delete anything that isn't on the current page because we only need ids for one page |
| 1004 | $this->save_ids($search_key, $this->search_query, $author_ary, $total_results, $id_ary, $start, $sort_dir); |
| 1005 | $id_ary = array_slice($id_ary, 0, (int) $per_page); |
| 1006 | |
| 1007 | return $total_results; |
| 1008 | } |
| 1009 | |
| 1010 | /** |
| 1011 | * {@inheritdoc} |
| 1012 | */ |
| 1013 | public function author_search(string $type, bool $firstpost_only, array $sort_by_sql, string $sort_key, string $sort_dir, string $sort_days, array $ex_fid_ary, string $post_visibility, int $topic_id, array $author_ary, string $author_name, array &$id_ary, int &$start, int $per_page) |
| 1014 | { |
| 1015 | // No author? No posts |
| 1016 | if (!count($author_ary)) |
| 1017 | { |
| 1018 | return 0; |
| 1019 | } |
| 1020 | |
| 1021 | // generate a search_key from all the options to identify the results |
| 1022 | $search_key_array = array( |
| 1023 | '', |
| 1024 | $type, |
| 1025 | ($firstpost_only) ? 'firstpost' : '', |
| 1026 | '', |
| 1027 | '', |
| 1028 | $sort_days, |
| 1029 | $sort_key, |
| 1030 | $topic_id, |
| 1031 | implode(',', $ex_fid_ary), |
| 1032 | $post_visibility, |
| 1033 | implode(',', $author_ary), |
| 1034 | $author_name, |
| 1035 | ); |
| 1036 | |
| 1037 | /** |
| 1038 | * Allow changing the search_key for cached results |
| 1039 | * |
| 1040 | * @event core.search_native_by_author_modify_search_key |
| 1041 | * @var array search_key_array Array with search parameters to generate the search_key |
| 1042 | * @var string type Searching type ('posts', 'topics') |
| 1043 | * @var boolean firstpost_only Flag indicating if only topic starting posts are considered |
| 1044 | * @var int sort_days Time, in days, of the oldest possible post to list |
| 1045 | * @var string sort_key The sort type used from the possible sort types |
| 1046 | * @var int topic_id Limit the search to this topic_id only |
| 1047 | * @var array ex_fid_ary Which forums not to search on |
| 1048 | * @var string post_visibility Post visibility data |
| 1049 | * @var array author_ary Array of user_id containing the users to filter the results to |
| 1050 | * @var string author_name The username to search on |
| 1051 | * @since 3.1.7-RC1 |
| 1052 | */ |
| 1053 | $vars = array( |
| 1054 | 'search_key_array', |
| 1055 | 'type', |
| 1056 | 'firstpost_only', |
| 1057 | 'sort_days', |
| 1058 | 'sort_key', |
| 1059 | 'topic_id', |
| 1060 | 'ex_fid_ary', |
| 1061 | 'post_visibility', |
| 1062 | 'author_ary', |
| 1063 | 'author_name', |
| 1064 | ); |
| 1065 | extract($this->phpbb_dispatcher->trigger_event('core.search_native_by_author_modify_search_key', compact($vars))); |
| 1066 | |
| 1067 | $search_key = md5(implode('#', $search_key_array)); |
| 1068 | |
| 1069 | // try reading the results from cache |
| 1070 | $total_results = 0; |
| 1071 | if ($this->obtain_ids($search_key, $total_results, $id_ary, $start, $per_page, $sort_dir) == self::SEARCH_RESULT_IN_CACHE) |
| 1072 | { |
| 1073 | return $total_results; |
| 1074 | } |
| 1075 | |
| 1076 | $id_ary = array(); |
| 1077 | |
| 1078 | // Create some display specific sql strings |
| 1079 | if ($author_name) |
| 1080 | { |
| 1081 | // first one matches post of registered users, second one guests and deleted users |
| 1082 | $sql_author = '(' . $this->db->sql_in_set('p.poster_id', array_diff($author_ary, array(ANONYMOUS)), false, true) . ' OR p.post_username ' . $author_name . ')'; |
| 1083 | } |
| 1084 | else |
| 1085 | { |
| 1086 | $sql_author = $this->db->sql_in_set('p.poster_id', $author_ary); |
| 1087 | } |
| 1088 | $sql_fora = (count($ex_fid_ary)) ? ' AND ' . $this->db->sql_in_set('p.forum_id', $ex_fid_ary, true) : ''; |
| 1089 | $sql_time = ($sort_days) ? ' AND p.post_time >= ' . (time() - ($sort_days * 86400)) : ''; |
| 1090 | $sql_topic_id = ($topic_id) ? ' AND p.topic_id = ' . (int) $topic_id : ''; |
| 1091 | $sql_firstpost = ($firstpost_only) ? ' AND p.post_id = t.topic_first_post_id' : ''; |
| 1092 | $post_visibility = ($post_visibility) ? ' AND ' . $post_visibility : ''; |
| 1093 | |
| 1094 | // Build sql strings for sorting |
| 1095 | $sql_sort = $sort_by_sql[$sort_key] . (($sort_dir == 'a') ? ' ASC' : ' DESC'); |
| 1096 | $sql_sort_table = $sql_sort_join = ''; |
| 1097 | switch ($sql_sort[0]) |
| 1098 | { |
| 1099 | case 'u': |
| 1100 | $sql_sort_table = USERS_TABLE . ' u, '; |
| 1101 | $sql_sort_join = ' AND u.user_id = p.poster_id '; |
| 1102 | break; |
| 1103 | |
| 1104 | case 't': |
| 1105 | $sql_sort_table = ($type == 'posts' && !$firstpost_only) ? TOPICS_TABLE . ' t, ' : ''; |
| 1106 | $sql_sort_join = ($type == 'posts' && !$firstpost_only) ? ' AND t.topic_id = p.topic_id ' : ''; |
| 1107 | break; |
| 1108 | |
| 1109 | case 'f': |
| 1110 | $sql_sort_table = FORUMS_TABLE . ' f, '; |
| 1111 | $sql_sort_join = ' AND f.forum_id = p.forum_id '; |
| 1112 | break; |
| 1113 | } |
| 1114 | |
| 1115 | $select = ($type == 'posts') ? 'p.post_id' : 't.topic_id'; |
| 1116 | $select .= $sort_by_sql[$sort_key] ? ", {$sort_by_sql[$sort_key]}" : ''; |
| 1117 | $is_mysql = false; |
| 1118 | |
| 1119 | /** |
| 1120 | * Allow changing the query used to search for posts by author in fulltext_native |
| 1121 | * |
| 1122 | * @event core.search_native_author_count_query_before |
| 1123 | * @var int total_results The previous result count for the format of the query. |
| 1124 | * Set to 0 to force a re-count |
| 1125 | * @var string type The type of search being made |
| 1126 | * @var string select SQL SELECT clause for what to get |
| 1127 | * @var string sql_sort_table CROSS JOIN'ed table to allow doing the sort chosen |
| 1128 | * @var string sql_sort_join Condition to define how to join the CROSS JOIN'ed table specifyed in sql_sort_table |
| 1129 | * @var array sql_author SQL WHERE condition for the post author ids |
| 1130 | * @var int topic_id Limit the search to this topic_id only |
| 1131 | * @var string sort_by_sql The possible predefined sort types |
| 1132 | * @var string sort_key The sort type used from the possible sort types |
| 1133 | * @var string sort_dir "a" for ASC or "d" dor DESC for the sort order used |
| 1134 | * @var string sql_sort The result SQL when processing sort_by_sql + sort_key + sort_dir |
| 1135 | * @var string sort_days Time, in days, that the oldest post showing can have |
| 1136 | * @var string sql_time The SQL to search on the time specifyed by sort_days |
| 1137 | * @var bool firstpost_only Wether or not to search only on the first post of the topics |
| 1138 | * @var string sql_firstpost The SQL used in the WHERE claused to filter by firstpost. |
| 1139 | * @var array ex_fid_ary Forum ids that must not be searched on |
| 1140 | * @var array sql_fora SQL query for ex_fid_ary |
| 1141 | * @var int start How many posts to skip in the search results (used for pagination) |
| 1142 | * @since 3.1.5-RC1 |
| 1143 | */ |
| 1144 | $vars = array( |
| 1145 | 'total_results', |
| 1146 | 'type', |
| 1147 | 'select', |
| 1148 | 'sql_sort_table', |
| 1149 | 'sql_sort_join', |
| 1150 | 'sql_author', |
| 1151 | 'topic_id', |
| 1152 | 'sort_by_sql', |
| 1153 | 'sort_key', |
| 1154 | 'sort_dir', |
| 1155 | 'sql_sort', |
| 1156 | 'sort_days', |
| 1157 | 'sql_time', |
| 1158 | 'firstpost_only', |
| 1159 | 'sql_firstpost', |
| 1160 | 'ex_fid_ary', |
| 1161 | 'sql_fora', |
| 1162 | 'start', |
| 1163 | ); |
| 1164 | extract($this->phpbb_dispatcher->trigger_event('core.search_native_author_count_query_before', compact($vars))); |
| 1165 | |
| 1166 | // If the cache was completely empty count the results |
| 1167 | if (!$total_results) |
| 1168 | { |
| 1169 | switch ($this->db->get_sql_layer()) |
| 1170 | { |
| 1171 | case 'mysqli': |
| 1172 | $is_mysql = true; |
| 1173 | break; |
| 1174 | |
| 1175 | default: |
| 1176 | if ($type == 'posts') |
| 1177 | { |
| 1178 | $sql = 'SELECT COUNT(p.post_id) as total_results |
| 1179 | FROM ' . POSTS_TABLE . ' p' . (($firstpost_only) ? ', ' . TOPICS_TABLE . ' t ' : ' ') . " |
| 1180 | WHERE $sql_author |
| 1181 | $sql_topic_id |
| 1182 | $sql_firstpost |
| 1183 | $post_visibility |
| 1184 | $sql_fora |
| 1185 | $sql_time"; |
| 1186 | } |
| 1187 | else |
| 1188 | { |
| 1189 | if ($this->db->get_sql_layer() == 'sqlite3') |
| 1190 | { |
| 1191 | $sql = 'SELECT COUNT(topic_id) as total_results |
| 1192 | FROM (SELECT DISTINCT t.topic_id'; |
| 1193 | } |
| 1194 | else |
| 1195 | { |
| 1196 | $sql = 'SELECT COUNT(DISTINCT t.topic_id) as total_results'; |
| 1197 | } |
| 1198 | |
| 1199 | $sql .= ' FROM ' . TOPICS_TABLE . ' t, ' . POSTS_TABLE . " p |
| 1200 | WHERE $sql_author |
| 1201 | $sql_topic_id |
| 1202 | $sql_firstpost |
| 1203 | $post_visibility |
| 1204 | $sql_fora |
| 1205 | AND t.topic_id = p.topic_id |
| 1206 | $sql_time" . ($this->db->get_sql_layer() == 'sqlite3' ? ')' : ''); |
| 1207 | } |
| 1208 | $result = $this->db->sql_query($sql); |
| 1209 | |
| 1210 | $total_results = (int) $this->db->sql_fetchfield('total_results'); |
| 1211 | $this->db->sql_freeresult($result); |
| 1212 | |
| 1213 | if (!$total_results) |
| 1214 | { |
| 1215 | return false; |
| 1216 | } |
| 1217 | break; |
| 1218 | } |
| 1219 | } |
| 1220 | |
| 1221 | // Build the query for really selecting the post_ids |
| 1222 | if ($type == 'posts') |
| 1223 | { |
| 1224 | $sql = "SELECT $select |
| 1225 | FROM " . $sql_sort_table . POSTS_TABLE . ' p' . (($firstpost_only) ? ', ' . TOPICS_TABLE . ' t' : '') . " |
| 1226 | WHERE $sql_author |
| 1227 | $sql_topic_id |
| 1228 | $sql_firstpost |
| 1229 | $post_visibility |
| 1230 | $sql_fora |
| 1231 | $sql_sort_join |
| 1232 | $sql_time |
| 1233 | ORDER BY $sql_sort"; |
| 1234 | $field = 'post_id'; |
| 1235 | } |
| 1236 | else |
| 1237 | { |
| 1238 | $sql = "SELECT $select |
| 1239 | FROM " . $sql_sort_table . TOPICS_TABLE . ' t, ' . POSTS_TABLE . " p |
| 1240 | WHERE $sql_author |
| 1241 | $sql_topic_id |
| 1242 | $sql_firstpost |
| 1243 | $post_visibility |
| 1244 | $sql_fora |
| 1245 | AND t.topic_id = p.topic_id |
| 1246 | $sql_sort_join |
| 1247 | $sql_time |
| 1248 | GROUP BY t.topic_id, " . $sort_by_sql[$sort_key] . ' |
| 1249 | ORDER BY ' . $sql_sort; |
| 1250 | $field = 'topic_id'; |
| 1251 | } |
| 1252 | |
| 1253 | // Only read one block of posts from the db and then cache it |
| 1254 | $result = $this->db->sql_query_limit($sql, $this->config['search_block_size'], $start); |
| 1255 | |
| 1256 | while ($row = $this->db->sql_fetchrow($result)) |
| 1257 | { |
| 1258 | $id_ary[] = (int) $row[$field]; |
| 1259 | } |
| 1260 | $this->db->sql_freeresult($result); |
| 1261 | |
| 1262 | if (!$total_results && $is_mysql) |
| 1263 | { |
| 1264 | $sql_count = str_replace("SELECT $select", "SELECT COUNT(*) as total_results", $sql); |
| 1265 | $result = $this->db->sql_query($sql_count); |
| 1266 | $total_results = ($type == 'posts') ? (int) $this->db->sql_fetchfield('total_results') : count($this->db->sql_fetchrowset($result)); |
| 1267 | $this->db->sql_freeresult($result); |
| 1268 | |
| 1269 | if (!$total_results) |
| 1270 | { |
| 1271 | return false; |
| 1272 | } |
| 1273 | } |
| 1274 | |
| 1275 | if ($start >= $total_results) |
| 1276 | { |
| 1277 | $start = floor(($total_results - 1) / $per_page) * $per_page; |
| 1278 | |
| 1279 | $result = $this->db->sql_query_limit($sql, $this->config['search_block_size'], $start); |
| 1280 | |
| 1281 | while ($row = $this->db->sql_fetchrow($result)) |
| 1282 | { |
| 1283 | $id_ary[] = (int) $row[$field]; |
| 1284 | } |
| 1285 | $this->db->sql_freeresult($result); |
| 1286 | } |
| 1287 | |
| 1288 | if (count($id_ary)) |
| 1289 | { |
| 1290 | $this->save_ids($search_key, '', $author_ary, $total_results, $id_ary, $start, $sort_dir); |
| 1291 | $id_ary = array_slice($id_ary, 0, $per_page); |
| 1292 | |
| 1293 | return $total_results; |
| 1294 | } |
| 1295 | return false; |
| 1296 | } |
| 1297 | |
| 1298 | /** |
| 1299 | * {@inheritdoc} |
| 1300 | */ |
| 1301 | public function supports_phrase_search(): bool |
| 1302 | { |
| 1303 | return false; |
| 1304 | } |
| 1305 | |
| 1306 | /** |
| 1307 | * {@inheritdoc} |
| 1308 | */ |
| 1309 | public function index(string $mode, int $post_id, string &$message, string &$subject, int $poster_id, int $forum_id) |
| 1310 | { |
| 1311 | if (!$this->config['fulltext_native_load_upd']) |
| 1312 | { |
| 1313 | /** |
| 1314 | * The search indexer is disabled, return |
| 1315 | */ |
| 1316 | return; |
| 1317 | } |
| 1318 | |
| 1319 | // Split old and new post/subject to obtain array of 'words' |
| 1320 | $split_text = $this->split_message($message); |
| 1321 | $split_title = $this->split_message($subject); |
| 1322 | |
| 1323 | $cur_words = array('post' => array(), 'title' => array()); |
| 1324 | |
| 1325 | $words = array(); |
| 1326 | if ($mode == 'edit') |
| 1327 | { |
| 1328 | $words['add']['post'] = array(); |
| 1329 | $words['add']['title'] = array(); |
| 1330 | $words['del']['post'] = array(); |
| 1331 | $words['del']['title'] = array(); |
| 1332 | |
| 1333 | $sql = 'SELECT w.word_id, w.word_text, m.title_match |
| 1334 | FROM ' . $this->search_wordlist_table . ' w, ' . $this->search_wordmatch_table . " m |
| 1335 | WHERE m.post_id = $post_id |
| 1336 | AND w.word_id = m.word_id"; |
| 1337 | $result = $this->db->sql_query($sql); |
| 1338 | |
| 1339 | while ($row = $this->db->sql_fetchrow($result)) |
| 1340 | { |
| 1341 | $which = ($row['title_match']) ? 'title' : 'post'; |
| 1342 | $cur_words[$which][$row['word_text']] = $row['word_id']; |
| 1343 | } |
| 1344 | $this->db->sql_freeresult($result); |
| 1345 | |
| 1346 | $words['add']['post'] = array_diff($split_text, array_keys($cur_words['post'])); |
| 1347 | $words['add']['title'] = array_diff($split_title, array_keys($cur_words['title'])); |
| 1348 | $words['del']['post'] = array_diff(array_keys($cur_words['post']), $split_text); |
| 1349 | $words['del']['title'] = array_diff(array_keys($cur_words['title']), $split_title); |
| 1350 | } |
| 1351 | else |
| 1352 | { |
| 1353 | $words['add']['post'] = $split_text; |
| 1354 | $words['add']['title'] = $split_title; |
| 1355 | $words['del']['post'] = array(); |
| 1356 | $words['del']['title'] = array(); |
| 1357 | } |
| 1358 | |
| 1359 | /** |
| 1360 | * Event to modify method arguments and words before the native search index is updated |
| 1361 | * |
| 1362 | * @event core.search_native_index_before |
| 1363 | * @var string mode Contains the post mode: edit, post, reply, quote |
| 1364 | * @var int post_id The id of the post which is modified/created |
| 1365 | * @var string message New or updated post content |
| 1366 | * @var string subject New or updated post subject |
| 1367 | * @var int poster_id Post author's user id |
| 1368 | * @var int forum_id The id of the forum in which the post is located |
| 1369 | * @var array words Grouped lists of words added to or remove from the index |
| 1370 | * @var array split_text Array of words from the message |
| 1371 | * @var array split_title Array of words from the title |
| 1372 | * @var array cur_words Array of words currently in the index for comparing to new words |
| 1373 | * when mode is edit. Empty for other modes. |
| 1374 | * @since 3.2.3-RC1 |
| 1375 | */ |
| 1376 | $vars = array( |
| 1377 | 'mode', |
| 1378 | 'post_id', |
| 1379 | 'message', |
| 1380 | 'subject', |
| 1381 | 'poster_id', |
| 1382 | 'forum_id', |
| 1383 | 'words', |
| 1384 | 'split_text', |
| 1385 | 'split_title', |
| 1386 | 'cur_words', |
| 1387 | ); |
| 1388 | extract($this->phpbb_dispatcher->trigger_event('core.search_native_index_before', compact($vars))); |
| 1389 | |
| 1390 | unset($split_text); |
| 1391 | unset($split_title); |
| 1392 | |
| 1393 | // Get unique words from the above arrays |
| 1394 | $unique_add_words = array_unique(array_merge($words['add']['post'], $words['add']['title'])); |
| 1395 | |
| 1396 | // We now have unique arrays of all words to be added and removed and |
| 1397 | // individual arrays of added and removed words for text and title. What |
| 1398 | // we need to do now is add the new words (if they don't already exist) |
| 1399 | // and then add (or remove) matches between the words and this post |
| 1400 | if (count($unique_add_words)) |
| 1401 | { |
| 1402 | $sql = 'SELECT word_id, word_text |
| 1403 | FROM ' . $this->search_wordlist_table . ' |
| 1404 | WHERE ' . $this->db->sql_in_set('word_text', $unique_add_words); |
| 1405 | $result = $this->db->sql_query($sql); |
| 1406 | |
| 1407 | $word_ids = array(); |
| 1408 | while ($row = $this->db->sql_fetchrow($result)) |
| 1409 | { |
| 1410 | $word_ids[$row['word_text']] = $row['word_id']; |
| 1411 | } |
| 1412 | $this->db->sql_freeresult($result); |
| 1413 | $new_words = array_diff($unique_add_words, array_keys($word_ids)); |
| 1414 | |
| 1415 | $this->db->sql_transaction('begin'); |
| 1416 | if (count($new_words)) |
| 1417 | { |
| 1418 | $sql_ary = array(); |
| 1419 | |
| 1420 | foreach ($new_words as $word) |
| 1421 | { |
| 1422 | $sql_ary[] = array('word_text' => (string) $word, 'word_count' => 0); |
| 1423 | } |
| 1424 | $this->db->sql_return_on_error(true); |
| 1425 | $this->db->sql_multi_insert($this->search_wordlist_table, $sql_ary); |
| 1426 | $this->db->sql_return_on_error(false); |
| 1427 | } |
| 1428 | unset($new_words, $sql_ary); |
| 1429 | } |
| 1430 | else |
| 1431 | { |
| 1432 | $this->db->sql_transaction('begin'); |
| 1433 | } |
| 1434 | |
| 1435 | // now update the search match table, remove links to removed words and add links to new words |
| 1436 | foreach ($words['del'] as $word_in => $word_ary) |
| 1437 | { |
| 1438 | $title_match = ($word_in == 'title') ? 1 : 0; |
| 1439 | |
| 1440 | if (count($word_ary)) |
| 1441 | { |
| 1442 | $sql_in = array(); |
| 1443 | foreach ($word_ary as $word) |
| 1444 | { |
| 1445 | $sql_in[] = $cur_words[$word_in][$word]; |
| 1446 | } |
| 1447 | |
| 1448 | $sql = 'DELETE FROM ' . $this->search_wordmatch_table . ' |
| 1449 | WHERE ' . $this->db->sql_in_set('word_id', $sql_in) . ' |
| 1450 | AND post_id = ' . intval($post_id) . " |
| 1451 | AND title_match = $title_match"; |
| 1452 | $this->db->sql_query($sql); |
| 1453 | |
| 1454 | $sql = 'UPDATE ' . $this->search_wordlist_table . ' |
| 1455 | SET word_count = word_count - 1 |
| 1456 | WHERE ' . $this->db->sql_in_set('word_id', $sql_in) . ' |
| 1457 | AND word_count > 0'; |
| 1458 | $this->db->sql_query($sql); |
| 1459 | |
| 1460 | unset($sql_in); |
| 1461 | } |
| 1462 | } |
| 1463 | |
| 1464 | $this->db->sql_return_on_error(true); |
| 1465 | foreach ($words['add'] as $word_in => $word_ary) |
| 1466 | { |
| 1467 | $title_match = ($word_in == 'title') ? 1 : 0; |
| 1468 | |
| 1469 | if (count($word_ary)) |
| 1470 | { |
| 1471 | $sql = 'INSERT INTO ' . $this->search_wordmatch_table . ' (post_id, word_id, title_match) |
| 1472 | SELECT ' . (int) $post_id . ', word_id, ' . (int) $title_match . ' |
| 1473 | FROM ' . $this->search_wordlist_table . ' |
| 1474 | WHERE ' . $this->db->sql_in_set('word_text', $word_ary); |
| 1475 | $this->db->sql_query($sql); |
| 1476 | |
| 1477 | $sql = 'UPDATE ' . $this->search_wordlist_table . ' |
| 1478 | SET word_count = word_count + 1 |
| 1479 | WHERE ' . $this->db->sql_in_set('word_text', $word_ary); |
| 1480 | $this->db->sql_query($sql); |
| 1481 | } |
| 1482 | } |
| 1483 | $this->db->sql_return_on_error(false); |
| 1484 | |
| 1485 | $this->db->sql_transaction('commit'); |
| 1486 | |
| 1487 | // destroy cached search results containing any of the words removed or added |
| 1488 | $this->destroy_cache(array_unique(array_merge($words['add']['post'], $words['add']['title'], $words['del']['post'], $words['del']['title'])), array($poster_id)); |
| 1489 | |
| 1490 | unset($unique_add_words); |
| 1491 | unset($words); |
| 1492 | unset($cur_words); |
| 1493 | } |
| 1494 | |
| 1495 | /** |
| 1496 | * {@inheritdoc} |
| 1497 | */ |
| 1498 | public function index_remove(array $post_ids, array $author_ids, array $forum_ids): void |
| 1499 | { |
| 1500 | if (count($post_ids)) |
| 1501 | { |
| 1502 | $sql = 'SELECT w.word_id, w.word_text, m.title_match |
| 1503 | FROM ' . $this->search_wordmatch_table . ' m, ' . $this->search_wordlist_table . ' w |
| 1504 | WHERE ' . $this->db->sql_in_set('m.post_id', $post_ids) . ' |
| 1505 | AND w.word_id = m.word_id'; |
| 1506 | $result = $this->db->sql_query($sql); |
| 1507 | |
| 1508 | $message_word_ids = $title_word_ids = $word_texts = array(); |
| 1509 | while ($row = $this->db->sql_fetchrow($result)) |
| 1510 | { |
| 1511 | if ($row['title_match']) |
| 1512 | { |
| 1513 | $title_word_ids[] = $row['word_id']; |
| 1514 | } |
| 1515 | else |
| 1516 | { |
| 1517 | $message_word_ids[] = $row['word_id']; |
| 1518 | } |
| 1519 | $word_texts[] = $row['word_text']; |
| 1520 | } |
| 1521 | $this->db->sql_freeresult($result); |
| 1522 | |
| 1523 | if (count($title_word_ids)) |
| 1524 | { |
| 1525 | $sql = 'UPDATE ' . $this->search_wordlist_table . ' |
| 1526 | SET word_count = word_count - 1 |
| 1527 | WHERE ' . $this->db->sql_in_set('word_id', $title_word_ids) . ' |
| 1528 | AND word_count > 0'; |
| 1529 | $this->db->sql_query($sql); |
| 1530 | } |
| 1531 | |
| 1532 | if (count($message_word_ids)) |
| 1533 | { |
| 1534 | $sql = 'UPDATE ' . $this->search_wordlist_table . ' |
| 1535 | SET word_count = word_count - 1 |
| 1536 | WHERE ' . $this->db->sql_in_set('word_id', $message_word_ids) . ' |
| 1537 | AND word_count > 0'; |
| 1538 | $this->db->sql_query($sql); |
| 1539 | } |
| 1540 | |
| 1541 | unset($title_word_ids); |
| 1542 | unset($message_word_ids); |
| 1543 | |
| 1544 | $sql = 'DELETE FROM ' . $this->search_wordmatch_table . ' |
| 1545 | WHERE ' . $this->db->sql_in_set('post_id', $post_ids); |
| 1546 | $this->db->sql_query($sql); |
| 1547 | } |
| 1548 | |
| 1549 | $this->destroy_cache(array_unique($word_texts), array_unique($author_ids)); |
| 1550 | } |
| 1551 | |
| 1552 | /** |
| 1553 | * {@inheritdoc} |
| 1554 | */ |
| 1555 | public function tidy(): void |
| 1556 | { |
| 1557 | // Is the fulltext indexer disabled? If yes then we need not |
| 1558 | // carry on ... it's okay ... I know when I'm not wanted boo hoo |
| 1559 | if (!$this->config['fulltext_native_load_upd']) |
| 1560 | { |
| 1561 | $this->config->set('search_last_gc', time(), false); |
| 1562 | return; |
| 1563 | } |
| 1564 | |
| 1565 | $destroy_cache_words = array(); |
| 1566 | |
| 1567 | // Remove common words |
| 1568 | if ($this->config['num_posts'] >= 100 && $this->config['fulltext_native_common_thres']) |
| 1569 | { |
| 1570 | $common_threshold = ((double) $this->config['fulltext_native_common_thres']) / 100.0; |
| 1571 | // First, get the IDs of common words |
| 1572 | $sql = 'SELECT word_id, word_text |
| 1573 | FROM ' . $this->search_wordlist_table . ' |
| 1574 | WHERE word_count > ' . floor($this->config['num_posts'] * $common_threshold) . ' |
| 1575 | OR word_common = 1'; |
| 1576 | $result = $this->db->sql_query($sql); |
| 1577 | |
| 1578 | $sql_in = array(); |
| 1579 | while ($row = $this->db->sql_fetchrow($result)) |
| 1580 | { |
| 1581 | $sql_in[] = $row['word_id']; |
| 1582 | $destroy_cache_words[] = $row['word_text']; |
| 1583 | } |
| 1584 | $this->db->sql_freeresult($result); |
| 1585 | |
| 1586 | if (count($sql_in)) |
| 1587 | { |
| 1588 | // Flag the words |
| 1589 | $sql = 'UPDATE ' . $this->search_wordlist_table . ' |
| 1590 | SET word_common = 1 |
| 1591 | WHERE ' . $this->db->sql_in_set('word_id', $sql_in); |
| 1592 | $this->db->sql_query($sql); |
| 1593 | |
| 1594 | // by setting search_last_gc to the new time here we make sure that if a user reloads because the |
| 1595 | // following query takes too long, he won't run into it again |
| 1596 | $this->config->set('search_last_gc', time(), false); |
| 1597 | |
| 1598 | // Delete the matches |
| 1599 | $sql = 'DELETE FROM ' . $this->search_wordmatch_table . ' |
| 1600 | WHERE ' . $this->db->sql_in_set('word_id', $sql_in); |
| 1601 | $this->db->sql_query($sql); |
| 1602 | } |
| 1603 | unset($sql_in); |
| 1604 | } |
| 1605 | |
| 1606 | if (count($destroy_cache_words)) |
| 1607 | { |
| 1608 | // destroy cached search results containing any of the words that are now common or were removed |
| 1609 | $this->destroy_cache(array_unique($destroy_cache_words)); |
| 1610 | } |
| 1611 | |
| 1612 | $this->config->set('search_last_gc', time(), false); |
| 1613 | } |
| 1614 | |
| 1615 | // create_index is inherited from base.php |
| 1616 | |
| 1617 | /** |
| 1618 | * {@inheritdoc} |
| 1619 | */ |
| 1620 | public function delete_index(int &$post_counter = null): ?array |
| 1621 | { |
| 1622 | $truncate_tables = [ |
| 1623 | $this->search_wordlist_table, |
| 1624 | $this->search_wordmatch_table, |
| 1625 | $this->search_results_table, |
| 1626 | ]; |
| 1627 | |
| 1628 | $stats = $this->stats; |
| 1629 | |
| 1630 | /** |
| 1631 | * Event to modify SQL queries before the native search index is deleted |
| 1632 | * |
| 1633 | * @event core.search_native_delete_index_before |
| 1634 | * |
| 1635 | * @var array stats Array with statistics of the current index (read only) |
| 1636 | * @var array truncate_tables Array with tables that will be truncated |
| 1637 | * |
| 1638 | * @since 3.2.3-RC1 |
| 1639 | * @changed 4.0.0-a1 Removed sql_queries, only add/remove tables to truncate to truncate_tables |
| 1640 | */ |
| 1641 | $vars = array( |
| 1642 | 'stats', |
| 1643 | 'truncate_tables', |
| 1644 | ); |
| 1645 | extract($this->phpbb_dispatcher->trigger_event('core.search_native_delete_index_before', compact($vars))); |
| 1646 | |
| 1647 | foreach ($truncate_tables as $table) |
| 1648 | { |
| 1649 | $this->db_tools->sql_truncate_table($table); |
| 1650 | } |
| 1651 | |
| 1652 | return null; |
| 1653 | } |
| 1654 | |
| 1655 | /** |
| 1656 | * {@inheritdoc} |
| 1657 | */ |
| 1658 | public function index_created(): bool |
| 1659 | { |
| 1660 | if (!count($this->stats)) |
| 1661 | { |
| 1662 | $this->get_stats(); |
| 1663 | } |
| 1664 | |
| 1665 | return $this->stats['total_words'] && $this->stats['total_matches']; |
| 1666 | } |
| 1667 | |
| 1668 | /** |
| 1669 | * {@inheritdoc} |
| 1670 | */ |
| 1671 | public function index_stats() |
| 1672 | { |
| 1673 | if (!count($this->stats)) |
| 1674 | { |
| 1675 | $this->get_stats(); |
| 1676 | } |
| 1677 | |
| 1678 | return array( |
| 1679 | $this->language->lang('TOTAL_WORDS') => $this->stats['total_words'], |
| 1680 | $this->language->lang('TOTAL_MATCHES') => $this->stats['total_matches']); |
| 1681 | } |
| 1682 | |
| 1683 | /** |
| 1684 | * Computes the stats and store them in the $this->stats associative array |
| 1685 | */ |
| 1686 | protected function get_stats() |
| 1687 | { |
| 1688 | $this->stats['total_words'] = $this->db->get_estimated_row_count($this->search_wordlist_table); |
| 1689 | $this->stats['total_matches'] = $this->db->get_estimated_row_count($this->search_wordmatch_table); |
| 1690 | } |
| 1691 | |
| 1692 | /** |
| 1693 | * Split a text into words of a given length |
| 1694 | * |
| 1695 | * The text is converted to UTF-8, cleaned up, and split. Then, words that |
| 1696 | * conform to the defined length range are returned in an array. |
| 1697 | * |
| 1698 | * NOTE: duplicates are NOT removed from the return array |
| 1699 | * |
| 1700 | * @param string $text Text to split, encoded in UTF-8 |
| 1701 | * @return array Array of UTF-8 words |
| 1702 | */ |
| 1703 | protected function split_message($text) |
| 1704 | { |
| 1705 | $match = $words = array(); |
| 1706 | |
| 1707 | /** |
| 1708 | * Taken from the original code |
| 1709 | */ |
| 1710 | // Do not index code |
| 1711 | $match[] = '#\[code(?:=.*?)?(\:?[0-9a-z]{5,})\].*?\[\/code(\:?[0-9a-z]{5,})\]#is'; |
| 1712 | // BBcode |
| 1713 | $match[] = '#\[\/?[a-z0-9\*\+\-]+(?:=.*?)?(?::[a-z])?(\:?[0-9a-z]{5,})\]#'; |
| 1714 | |
| 1715 | $min = $this->word_length['min']; |
| 1716 | |
| 1717 | $isset_min = $min - 1; |
| 1718 | |
| 1719 | /** |
| 1720 | * Clean up the string, remove HTML tags, remove BBCodes |
| 1721 | */ |
| 1722 | $word = strtok($this->cleanup(preg_replace($match, ' ', strip_tags($text)), '-1'), ' '); |
| 1723 | |
| 1724 | while (strlen($word)) |
| 1725 | { |
| 1726 | if (strlen($word) > 255 || strlen($word) <= $isset_min) |
| 1727 | { |
| 1728 | /** |
| 1729 | * Words longer than 255 bytes are ignored. This will have to be |
| 1730 | * changed whenever we change the length of search_wordlist.word_text |
| 1731 | * |
| 1732 | * Words shorter than $isset_min bytes are ignored, too |
| 1733 | */ |
| 1734 | $word = strtok(' '); |
| 1735 | continue; |
| 1736 | } |
| 1737 | |
| 1738 | $len = utf8_strlen($word); |
| 1739 | |
| 1740 | /** |
| 1741 | * Test whether the word is too short to be indexed. |
| 1742 | * |
| 1743 | * Note that this limit does NOT apply to CJK and Hangul |
| 1744 | */ |
| 1745 | if ($len < $min) |
| 1746 | { |
| 1747 | /** |
| 1748 | * Note: this could be optimized. If the codepoint is lower than Hangul's range |
| 1749 | * we know that it will also be lower than CJK ranges |
| 1750 | */ |
| 1751 | if ((strncmp($word, self::UTF8_HANGUL_FIRST, 3) < 0 || strncmp($word, self::UTF8_HANGUL_LAST, 3) > 0) |
| 1752 | && (strncmp($word, self::UTF8_CJK_FIRST, 3) < 0 || strncmp($word, self::UTF8_CJK_LAST, 3) > 0) |
| 1753 | && (strncmp($word, self::UTF8_CJK_B_FIRST, 4) < 0 || strncmp($word, self::UTF8_CJK_B_LAST, 4) > 0)) |
| 1754 | { |
| 1755 | $word = strtok(' '); |
| 1756 | continue; |
| 1757 | } |
| 1758 | } |
| 1759 | |
| 1760 | $words[] = $word; |
| 1761 | $word = strtok(' '); |
| 1762 | } |
| 1763 | |
| 1764 | return $words; |
| 1765 | } |
| 1766 | |
| 1767 | /** |
| 1768 | * Clean up a text to remove non-alphanumeric characters |
| 1769 | * |
| 1770 | * This method receives a UTF-8 string, normalizes and validates it, replaces all |
| 1771 | * non-alphanumeric characters with strings then returns the result. |
| 1772 | * |
| 1773 | * Any number of "allowed chars" can be passed as a UTF-8 string in NFC. |
| 1774 | * |
| 1775 | * @param string $text Text to split, in UTF-8 (not normalized or sanitized) |
| 1776 | * @param string $allowed_chars String of special chars to allow |
| 1777 | * @param string $encoding Text encoding |
| 1778 | * @return string Cleaned up text, only alphanumeric chars are left |
| 1779 | */ |
| 1780 | protected function cleanup($text, $allowed_chars = null, $encoding = 'utf-8') |
| 1781 | { |
| 1782 | static $conv = array(), $conv_loaded = array(); |
| 1783 | $allow = array(); |
| 1784 | |
| 1785 | // Convert the text to UTF-8 |
| 1786 | $encoding = strtolower($encoding); |
| 1787 | if ($encoding != 'utf-8') |
| 1788 | { |
| 1789 | $text = utf8_recode($text, $encoding); |
| 1790 | } |
| 1791 | |
| 1792 | $utf_len_mask = array( |
| 1793 | "\xC0" => 2, |
| 1794 | "\xD0" => 2, |
| 1795 | "\xE0" => 3, |
| 1796 | "\xF0" => 4 |
| 1797 | ); |
| 1798 | |
| 1799 | /** |
| 1800 | * Replace HTML entities and NCRs |
| 1801 | */ |
| 1802 | $text = html_entity_decode(utf8_decode_ncr($text), ENT_QUOTES); |
| 1803 | |
| 1804 | /** |
| 1805 | * Normalize to NFC |
| 1806 | */ |
| 1807 | $text = \Normalizer::normalize($text); |
| 1808 | |
| 1809 | /** |
| 1810 | * The first thing we do is: |
| 1811 | * |
| 1812 | * - convert ASCII-7 letters to lowercase |
| 1813 | * - remove the ASCII-7 non-alpha characters |
| 1814 | * - remove the bytes that should not appear in a valid UTF-8 string: 0xC0, |
| 1815 | * 0xC1 and 0xF5-0xFF |
| 1816 | * |
| 1817 | * @todo in theory, the third one is already taken care of during normalization and those chars should have been replaced by Unicode replacement chars |
| 1818 | */ |
| 1819 | $sb_match = "ISTCPAMELRDOJBNHFGVWUQKYXZ\r\n\t!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0B\x0C\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F\xC0\xC1\xF5\xF6\xF7\xF8\xF9\xFA\xFB\xFC\xFD\xFE\xFF"; |
| 1820 | $sb_replace = 'istcpamelrdojbnhfgvwuqkyxz '; |
| 1821 | |
| 1822 | /** |
| 1823 | * This is the list of legal ASCII chars, it is automatically extended |
| 1824 | * with ASCII chars from $allowed_chars |
| 1825 | */ |
| 1826 | $legal_ascii = ' eaisntroludcpmghbfvq10xy2j9kw354867z'; |
| 1827 | |
| 1828 | /** |
| 1829 | * Prepare an array containing the extra chars to allow |
| 1830 | */ |
| 1831 | if (isset($allowed_chars[0])) |
| 1832 | { |
| 1833 | $pos = 0; |
| 1834 | $len = strlen($allowed_chars); |
| 1835 | do |
| 1836 | { |
| 1837 | $c = $allowed_chars[$pos]; |
| 1838 | |
| 1839 | if ($c < "\x80") |
| 1840 | { |
| 1841 | /** |
| 1842 | * ASCII char |
| 1843 | */ |
| 1844 | $sb_pos = strpos($sb_match, $c); |
| 1845 | if (is_int($sb_pos)) |
| 1846 | { |
| 1847 | /** |
| 1848 | * Remove the char from $sb_match and its corresponding |
| 1849 | * replacement in $sb_replace |
| 1850 | */ |
| 1851 | $sb_match = substr($sb_match, 0, $sb_pos) . substr($sb_match, $sb_pos + 1); |
| 1852 | $sb_replace = substr($sb_replace, 0, $sb_pos) . substr($sb_replace, $sb_pos + 1); |
| 1853 | $legal_ascii .= $c; |
| 1854 | } |
| 1855 | |
| 1856 | ++$pos; |
| 1857 | } |
| 1858 | else |
| 1859 | { |
| 1860 | /** |
| 1861 | * UTF-8 char |
| 1862 | */ |
| 1863 | $utf_len = $utf_len_mask[$c & "\xF0"]; |
| 1864 | $allow[substr($allowed_chars, $pos, $utf_len)] = 1; |
| 1865 | $pos += $utf_len; |
| 1866 | } |
| 1867 | } |
| 1868 | while ($pos < $len); |
| 1869 | } |
| 1870 | |
| 1871 | $text = strtr($text, $sb_match, $sb_replace); |
| 1872 | $ret = ''; |
| 1873 | |
| 1874 | $pos = 0; |
| 1875 | $len = strlen($text); |
| 1876 | |
| 1877 | do |
| 1878 | { |
| 1879 | /** |
| 1880 | * Do all consecutive ASCII chars at once |
| 1881 | */ |
| 1882 | if ($spn = strspn($text, $legal_ascii, $pos)) |
| 1883 | { |
| 1884 | $ret .= substr($text, $pos, $spn); |
| 1885 | $pos += $spn; |
| 1886 | } |
| 1887 | |
| 1888 | if ($pos >= $len) |
| 1889 | { |
| 1890 | return $ret; |
| 1891 | } |
| 1892 | |
| 1893 | /** |
| 1894 | * Capture the UTF char |
| 1895 | */ |
| 1896 | $utf_len = $utf_len_mask[$text[$pos] & "\xF0"]; |
| 1897 | $utf_char = substr($text, $pos, $utf_len); |
| 1898 | $pos += $utf_len; |
| 1899 | |
| 1900 | if (($utf_char >= self::UTF8_HANGUL_FIRST && $utf_char <= self::UTF8_HANGUL_LAST) |
| 1901 | || ($utf_char >= self::UTF8_CJK_FIRST && $utf_char <= self::UTF8_CJK_LAST) |
| 1902 | || ($utf_char >= self::UTF8_CJK_B_FIRST && $utf_char <= self::UTF8_CJK_B_LAST)) |
| 1903 | { |
| 1904 | /** |
| 1905 | * All characters within these ranges are valid |
| 1906 | * |
| 1907 | * We separate them with a space in order to index each character |
| 1908 | * individually |
| 1909 | */ |
| 1910 | $ret .= ' ' . $utf_char . ' '; |
| 1911 | continue; |
| 1912 | } |
| 1913 | |
| 1914 | if (isset($allow[$utf_char])) |
| 1915 | { |
| 1916 | /** |
| 1917 | * The char is explicitly allowed |
| 1918 | */ |
| 1919 | $ret .= $utf_char; |
| 1920 | continue; |
| 1921 | } |
| 1922 | |
| 1923 | if (isset($conv[$utf_char])) |
| 1924 | { |
| 1925 | /** |
| 1926 | * The char is mapped to something, maybe to itself actually |
| 1927 | */ |
| 1928 | $ret .= $conv[$utf_char]; |
| 1929 | continue; |
| 1930 | } |
| 1931 | |
| 1932 | /** |
| 1933 | * The char isn't mapped, but did we load its conversion table? |
| 1934 | * |
| 1935 | * The search indexer table is split into blocks. The block number of |
| 1936 | * each char is equal to its codepoint right-shifted for 11 bits. It |
| 1937 | * means that out of the 11, 16 or 21 meaningful bits of a 2-, 3- or |
| 1938 | * 4- byte sequence we only keep the leftmost 0, 5 or 10 bits. Thus, |
| 1939 | * all UTF chars encoded in 2 bytes are in the same first block. |
| 1940 | */ |
| 1941 | if (isset($utf_char[2])) |
| 1942 | { |
| 1943 | if (isset($utf_char[3])) |
| 1944 | { |
| 1945 | /** |
| 1946 | * 1111 0nnn 10nn nnnn 10nx xxxx 10xx xxxx |
| 1947 | * 0000 0111 0011 1111 0010 0000 |
| 1948 | */ |
| 1949 | $idx = ((ord($utf_char[0]) & 0x07) << 7) | ((ord($utf_char[1]) & 0x3F) << 1) | ((ord($utf_char[2]) & 0x20) >> 5); |
| 1950 | } |
| 1951 | else |
| 1952 | { |
| 1953 | /** |
| 1954 | * 1110 nnnn 10nx xxxx 10xx xxxx |
| 1955 | * 0000 0111 0010 0000 |
| 1956 | */ |
| 1957 | $idx = ((ord($utf_char[0]) & 0x07) << 1) | ((ord($utf_char[1]) & 0x20) >> 5); |
| 1958 | } |
| 1959 | } |
| 1960 | else |
| 1961 | { |
| 1962 | /** |
| 1963 | * 110x xxxx 10xx xxxx |
| 1964 | * 0000 0000 0000 0000 |
| 1965 | */ |
| 1966 | $idx = 0; |
| 1967 | } |
| 1968 | |
| 1969 | /** |
| 1970 | * Check if the required conv table has been loaded already |
| 1971 | */ |
| 1972 | if (!isset($conv_loaded[$idx])) |
| 1973 | { |
| 1974 | $conv_loaded[$idx] = 1; |
| 1975 | $file = $this->phpbb_root_path . 'includes/utf/data/search_indexer_' . $idx . '.' . $this->php_ext; |
| 1976 | |
| 1977 | if (file_exists($file)) |
| 1978 | { |
| 1979 | $conv += include($file); |
| 1980 | } |
| 1981 | } |
| 1982 | |
| 1983 | if (isset($conv[$utf_char])) |
| 1984 | { |
| 1985 | $ret .= $conv[$utf_char]; |
| 1986 | } |
| 1987 | else |
| 1988 | { |
| 1989 | /** |
| 1990 | * We add an entry to the conversion table so that we |
| 1991 | * don't have to convert to codepoint and perform the checks |
| 1992 | * that are above this block |
| 1993 | */ |
| 1994 | $conv[$utf_char] = ' '; |
| 1995 | $ret .= ' '; |
| 1996 | } |
| 1997 | } |
| 1998 | while (1); |
| 1999 | |
| 2000 | return $ret; |
| 2001 | } |
| 2002 | |
| 2003 | /** |
| 2004 | * {@inheritdoc} |
| 2005 | */ |
| 2006 | public function get_acp_options(): array |
| 2007 | { |
| 2008 | /** |
| 2009 | * if we need any options, copied from fulltext_native for now, will have to be adjusted or removed |
| 2010 | */ |
| 2011 | |
| 2012 | $tpl = ' |
| 2013 | <dl> |
| 2014 | <dt><label for="fulltext_native_load_upd">' . $this->language->lang('YES_SEARCH_UPDATE') . $this->language->lang('COLON') . '</label><br /><span>' . $this->language->lang('YES_SEARCH_UPDATE_EXPLAIN') . '</span></dt> |
| 2015 | <dd><label><input type="radio" id="fulltext_native_load_upd" name="config[fulltext_native_load_upd]" value="1"' . (($this->config['fulltext_native_load_upd']) ? ' checked="checked"' : '') . ' class="radio" /> ' . $this->language->lang('YES') . '</label><label><input type="radio" name="config[fulltext_native_load_upd]" value="0"' . ((!$this->config['fulltext_native_load_upd']) ? ' checked="checked"' : '') . ' class="radio" /> ' . $this->language->lang('NO') . '</label></dd> |
| 2016 | </dl> |
| 2017 | <dl> |
| 2018 | <dt><label for="fulltext_native_min_chars">' . $this->language->lang('MIN_SEARCH_CHARS') . $this->language->lang('COLON') . '</label><br /><span>' . $this->language->lang('MIN_SEARCH_CHARS_EXPLAIN') . '</span></dt> |
| 2019 | <dd><input id="fulltext_native_min_chars" type="number" min="0" max="255" name="config[fulltext_native_min_chars]" value="' . (int) $this->config['fulltext_native_min_chars'] . '" /></dd> |
| 2020 | </dl> |
| 2021 | <dl> |
| 2022 | <dt><label for="fulltext_native_max_chars">' . $this->language->lang('MAX_SEARCH_CHARS') . $this->language->lang('COLON') . '</label><br /><span>' . $this->language->lang('MAX_SEARCH_CHARS_EXPLAIN') . '</span></dt> |
| 2023 | <dd><input id="fulltext_native_max_chars" type="number" min="0" max="255" name="config[fulltext_native_max_chars]" value="' . (int) $this->config['fulltext_native_max_chars'] . '" /></dd> |
| 2024 | </dl> |
| 2025 | <dl> |
| 2026 | <dt><label for="fulltext_native_common_thres">' . $this->language->lang('COMMON_WORD_THRESHOLD') . $this->language->lang('COLON') . '</label><br /><span>' . $this->language->lang('COMMON_WORD_THRESHOLD_EXPLAIN') . '</span></dt> |
| 2027 | <dd><input id="fulltext_native_common_thres" type="text" name="config[fulltext_native_common_thres]" value="' . (double) $this->config['fulltext_native_common_thres'] . '" /> %</dd> |
| 2028 | </dl> |
| 2029 | '; |
| 2030 | |
| 2031 | // These are fields required in the config table |
| 2032 | return array( |
| 2033 | 'tpl' => $tpl, |
| 2034 | 'config' => array('fulltext_native_load_upd' => 'bool', 'fulltext_native_min_chars' => 'integer:0:255', 'fulltext_native_max_chars' => 'integer:0:255', 'fulltext_native_common_thres' => 'double:0:100') |
| 2035 | ); |
| 2036 | } |
| 2037 | } |