Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
19.41% |
171 / 881 |
|
4.76% |
1 / 21 |
CRAP | |
0.00% |
0 / 1 |
| fulltext_native | |
19.41% |
171 / 881 |
|
4.76% |
1 / 21 |
30137.07 | |
0.00% |
0 / 1 |
| __construct | |
91.67% |
11 / 12 |
|
0.00% |
0 / 1 |
2.00 | |||
| get_name | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| is_available | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| init | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| get_search_query | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| get_common_words | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| get_word_length | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| split_keywords | |
74.10% |
123 / 166 |
|
0.00% |
0 / 1 |
138.44 | |||
| keyword_search | |
0.00% |
0 / 255 |
|
0.00% |
0 / 1 |
3192 | |||
| author_search | |
0.00% |
0 / 158 |
|
0.00% |
0 / 1 |
1482 | |||
| supports_phrase_search | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| index | |
0.00% |
0 / 98 |
|
0.00% |
0 / 1 |
272 | |||
| index_remove | |
0.00% |
0 / 29 |
|
0.00% |
0 / 1 |
42 | |||
| tidy | |
0.00% |
0 / 28 |
|
0.00% |
0 / 1 |
56 | |||
| delete_index | |
0.00% |
0 / 14 |
|
0.00% |
0 / 1 |
6 | |||
| index_created | |
0.00% |
0 / 3 |
|
0.00% |
0 / 1 |
12 | |||
| index_stats | |
0.00% |
0 / 5 |
|
0.00% |
0 / 1 |
6 | |||
| get_stats | |
0.00% |
0 / 2 |
|
0.00% |
0 / 1 |
2 | |||
| split_message | |
0.00% |
0 / 20 |
|
0.00% |
0 / 1 |
132 | |||
| cleanup | |
51.43% |
36 / 70 |
|
0.00% |
0 / 1 |
65.84 | |||
| get_acp_options | |
0.00% |
0 / 14 |
|
0.00% |
0 / 1 |
12 | |||
| 1 | <?php |
| 2 | /** |
| 3 | * |
| 4 | * This file is part of the phpBB Forum Software package. |
| 5 | * |
| 6 | * @copyright (c) phpBB Limited <https://www.phpbb.com> |
| 7 | * @license GNU General Public License, version 2 (GPL-2.0) |
| 8 | * |
| 9 | * For full copyright and license information, please see |
| 10 | * the docs/CREDITS.txt file. |
| 11 | * |
| 12 | */ |
| 13 | |
| 14 | namespace phpbb\search\backend; |
| 15 | |
| 16 | use phpbb\config\config; |
| 17 | use phpbb\db\driver\driver_interface; |
| 18 | use phpbb\db\tools\tools_interface; |
| 19 | use phpbb\event\dispatcher_interface; |
| 20 | use phpbb\language\language; |
| 21 | use phpbb\user; |
| 22 | |
| 23 | /** |
| 24 | * phpBB's own db driven fulltext search, version 2 |
| 25 | */ |
| 26 | class fulltext_native extends base implements search_backend_interface |
| 27 | { |
| 28 | protected const UTF8_HANGUL_FIRST = "\xEA\xB0\x80"; |
| 29 | protected const UTF8_HANGUL_LAST = "\xED\x9E\xA3"; |
| 30 | protected const UTF8_CJK_FIRST = "\xE4\xB8\x80"; |
| 31 | protected const UTF8_CJK_LAST = "\xE9\xBE\xBB"; |
| 32 | protected const UTF8_CJK_B_FIRST = "\xF0\xA0\x80\x80"; |
| 33 | protected const UTF8_CJK_B_LAST = "\xF0\xAA\x9B\x96"; |
| 34 | |
| 35 | /** |
| 36 | * Associative array holding index stats |
| 37 | * @var array |
| 38 | */ |
| 39 | protected $stats = array(); |
| 40 | |
| 41 | /** |
| 42 | * Associative array stores the min and max word length to be searched |
| 43 | * @var array |
| 44 | */ |
| 45 | protected $word_length = array(); |
| 46 | |
| 47 | /** |
| 48 | * Contains tidied search query. |
| 49 | * Operators are prefixed in search query and common words excluded |
| 50 | * @var string |
| 51 | */ |
| 52 | protected $search_query = ''; |
| 53 | |
| 54 | /** |
| 55 | * Contains common words. |
| 56 | * Common words are words with length less/more than min/max length |
| 57 | * @var array |
| 58 | */ |
| 59 | protected $common_words = array(); |
| 60 | |
| 61 | /** |
| 62 | * Post ids of posts containing words that are to be included |
| 63 | * @var array |
| 64 | */ |
| 65 | protected $must_contain_ids = array(); |
| 66 | |
| 67 | /** |
| 68 | * Post ids of posts containing words that should not be included |
| 69 | * @var array |
| 70 | */ |
| 71 | protected $must_not_contain_ids = array(); |
| 72 | |
| 73 | /** |
| 74 | * Post ids of posts containing at least one word that needs to be excluded |
| 75 | * @var array |
| 76 | */ |
| 77 | protected $must_exclude_one_ids = array(); |
| 78 | |
| 79 | /** |
| 80 | * Relative path to board root |
| 81 | * @var string |
| 82 | */ |
| 83 | protected $phpbb_root_path; |
| 84 | |
| 85 | /** |
| 86 | * PHP Extension |
| 87 | * @var string |
| 88 | */ |
| 89 | protected $php_ext; |
| 90 | |
| 91 | /** |
| 92 | * DBAL tools |
| 93 | * @var tools_interface |
| 94 | */ |
| 95 | protected $db_tools; |
| 96 | |
| 97 | /** |
| 98 | * phpBB event dispatcher object |
| 99 | * @var dispatcher_interface |
| 100 | */ |
| 101 | protected $phpbb_dispatcher; |
| 102 | |
| 103 | /** @var language */ |
| 104 | protected $language; |
| 105 | |
| 106 | /** @var string */ |
| 107 | protected $search_wordlist_table; |
| 108 | |
| 109 | /** @var string */ |
| 110 | protected $search_wordmatch_table; |
| 111 | |
| 112 | /** |
| 113 | * Initialises the fulltext_native search backend with min/max word length |
| 114 | * |
| 115 | * @param config $config Config object |
| 116 | * @param driver_interface $db Database object |
| 117 | * @param tools_interface $db_tools Database tools |
| 118 | * @param dispatcher_interface $phpbb_dispatcher Event dispatcher object |
| 119 | * @param language $language |
| 120 | * @param user $user User object |
| 121 | * @param string $search_results_table |
| 122 | * @param string $search_wordlist_table |
| 123 | * @param string $search_wordmatch_table |
| 124 | * @param string $phpbb_root_path phpBB root path |
| 125 | * @param string $phpEx PHP file extension |
| 126 | */ |
| 127 | public function __construct(config $config, driver_interface $db, tools_interface $db_tools, dispatcher_interface $phpbb_dispatcher, |
| 128 | language $language, user $user, string $search_results_table, string $search_wordlist_table, |
| 129 | string $search_wordmatch_table, string $phpbb_root_path, string $phpEx) |
| 130 | { |
| 131 | global $cache; |
| 132 | |
| 133 | parent::__construct($cache, $config, $db, $user, $search_results_table); |
| 134 | $this->db_tools = $db_tools; |
| 135 | $this->phpbb_dispatcher = $phpbb_dispatcher; |
| 136 | $this->language = $language; |
| 137 | |
| 138 | $this->search_wordlist_table = $search_wordlist_table; |
| 139 | $this->search_wordmatch_table = $search_wordmatch_table; |
| 140 | |
| 141 | $this->phpbb_root_path = $phpbb_root_path; |
| 142 | $this->php_ext = $phpEx; |
| 143 | |
| 144 | $this->word_length = array('min' => (int) $this->config['fulltext_native_min_chars'], 'max' => (int) $this->config['fulltext_native_max_chars']); |
| 145 | |
| 146 | /** |
| 147 | * Load the UTF tools |
| 148 | */ |
| 149 | if (!function_exists('utf8_decode_ncr')) |
| 150 | { |
| 151 | include($this->phpbb_root_path . 'includes/utf/utf_tools.' . $this->php_ext); |
| 152 | } |
| 153 | } |
| 154 | |
| 155 | /** |
| 156 | * {@inheritdoc} |
| 157 | */ |
| 158 | public function get_name(): string |
| 159 | { |
| 160 | return 'phpBB Native Fulltext'; |
| 161 | } |
| 162 | |
| 163 | /** |
| 164 | * {@inheritdoc} |
| 165 | */ |
| 166 | public function is_available(): bool |
| 167 | { |
| 168 | return true; |
| 169 | } |
| 170 | |
| 171 | /** |
| 172 | * {@inheritdoc} |
| 173 | */ |
| 174 | public function init() |
| 175 | { |
| 176 | return false; |
| 177 | } |
| 178 | |
| 179 | /** |
| 180 | * {@inheritdoc} |
| 181 | */ |
| 182 | public function get_search_query(): string |
| 183 | { |
| 184 | return $this->search_query; |
| 185 | } |
| 186 | |
| 187 | /** |
| 188 | * {@inheritdoc} |
| 189 | */ |
| 190 | public function get_common_words(): array |
| 191 | { |
| 192 | return $this->common_words; |
| 193 | } |
| 194 | |
| 195 | /** |
| 196 | * {@inheritdoc} |
| 197 | */ |
| 198 | public function get_word_length() |
| 199 | { |
| 200 | return $this->word_length; |
| 201 | } |
| 202 | |
| 203 | /** |
| 204 | * {@inheritdoc} |
| 205 | */ |
| 206 | public function split_keywords(string &$keywords, string $terms): bool |
| 207 | { |
| 208 | $tokens = '+-|()* '; |
| 209 | |
| 210 | $keywords = trim($this->cleanup($keywords, $tokens)); |
| 211 | |
| 212 | // allow word|word|word without brackets |
| 213 | if ((strpos($keywords, ' ') === false) && (strpos($keywords, '|') !== false) && (strpos($keywords, '(') === false)) |
| 214 | { |
| 215 | $keywords = '(' . $keywords . ')'; |
| 216 | } |
| 217 | |
| 218 | $open_bracket = $space = false; |
| 219 | for ($i = 0, $n = strlen($keywords); $i < $n; $i++) |
| 220 | { |
| 221 | if ($open_bracket !== false) |
| 222 | { |
| 223 | switch ($keywords[$i]) |
| 224 | { |
| 225 | case ')': |
| 226 | if ($open_bracket + 1 == $i) |
| 227 | { |
| 228 | $keywords[$i - 1] = '|'; |
| 229 | $keywords[$i] = '|'; |
| 230 | } |
| 231 | $open_bracket = false; |
| 232 | break; |
| 233 | case '(': |
| 234 | $keywords[$i] = '|'; |
| 235 | break; |
| 236 | case '+': |
| 237 | case '-': |
| 238 | case ' ': |
| 239 | $keywords[$i] = '|'; |
| 240 | break; |
| 241 | case '*': |
| 242 | // $i can never be 0 here since $open_bracket is initialised to false |
| 243 | if (strpos($tokens, $keywords[$i - 1]) !== false && ($i + 1 === $n || strpos($tokens, $keywords[$i + 1]) !== false)) |
| 244 | { |
| 245 | $keywords[$i] = '|'; |
| 246 | } |
| 247 | break; |
| 248 | } |
| 249 | } |
| 250 | else |
| 251 | { |
| 252 | switch ($keywords[$i]) |
| 253 | { |
| 254 | case ')': |
| 255 | $keywords[$i] = ' '; |
| 256 | break; |
| 257 | case '(': |
| 258 | $open_bracket = $i; |
| 259 | $space = false; |
| 260 | break; |
| 261 | case '|': |
| 262 | $keywords[$i] = ' '; |
| 263 | break; |
| 264 | case '-': |
| 265 | // Ignore hyphen if followed by a space |
| 266 | if (isset($keywords[$i + 1]) && $keywords[$i + 1] == ' ') |
| 267 | { |
| 268 | $keywords[$i] = ' '; |
| 269 | } |
| 270 | else |
| 271 | { |
| 272 | $space = $keywords[$i]; |
| 273 | } |
| 274 | break; |
| 275 | case '+': |
| 276 | $space = $keywords[$i]; |
| 277 | break; |
| 278 | case ' ': |
| 279 | if ($space !== false) |
| 280 | { |
| 281 | $keywords[$i] = $space; |
| 282 | } |
| 283 | break; |
| 284 | default: |
| 285 | $space = false; |
| 286 | } |
| 287 | } |
| 288 | } |
| 289 | |
| 290 | if ($open_bracket !== false) |
| 291 | { |
| 292 | $keywords .= ')'; |
| 293 | } |
| 294 | |
| 295 | $match = array( |
| 296 | '# +#', |
| 297 | '#\|\|+#', |
| 298 | '#(\+|\-)(?:\+|\-)+#', |
| 299 | '#\(\|#', |
| 300 | '#\|\)#', |
| 301 | ); |
| 302 | $replace = array( |
| 303 | ' ', |
| 304 | '|', |
| 305 | '$1', |
| 306 | '(', |
| 307 | ')', |
| 308 | ); |
| 309 | |
| 310 | $keywords = preg_replace($match, $replace, $keywords); |
| 311 | |
| 312 | // Ensure a space exists before +, - and | to make the split and count work correctly |
| 313 | $countable_keywords = preg_replace('/(?<!\s)(\+|\-|\|)/', ' $1', $keywords); |
| 314 | |
| 315 | $num_keywords = count(explode(' ', $countable_keywords)); |
| 316 | |
| 317 | // We limit the number of allowed keywords to minimize load on the database |
| 318 | if ($this->config['max_num_search_keywords'] && $num_keywords > $this->config['max_num_search_keywords']) |
| 319 | { |
| 320 | trigger_error($this->language->lang('MAX_NUM_SEARCH_KEYWORDS_REFINE', (int) $this->config['max_num_search_keywords'], $num_keywords)); |
| 321 | } |
| 322 | |
| 323 | // $keywords input format: each word separated by a space, words in a bracket are not separated |
| 324 | |
| 325 | // the user wants to search for any word, convert the search query |
| 326 | if ($terms == 'any') |
| 327 | { |
| 328 | $words = array(); |
| 329 | |
| 330 | preg_match_all('#([^\\s+\\-|()]+)(?:$|[\\s+\\-|()])#u', $keywords, $words); |
| 331 | if (count($words[1])) |
| 332 | { |
| 333 | $keywords = '(' . implode('|', $words[1]) . ')'; |
| 334 | } |
| 335 | } |
| 336 | |
| 337 | // Remove non trailing wildcards from each word to prevent a full table scan (it's now using the database index) |
| 338 | $match = '#\*(?!$|\s)#'; |
| 339 | $replace = '$1'; |
| 340 | $keywords = preg_replace($match, $replace, $keywords); |
| 341 | |
| 342 | // Only allow one wildcard in the search query to limit the database load |
| 343 | $match = '#\*#'; |
| 344 | $replace = '$1'; |
| 345 | $count_wildcards = substr_count($keywords, '*'); |
| 346 | |
| 347 | // Reverse the string to remove all wildcards except the first one |
| 348 | $keywords = strrev(preg_replace($match, $replace, strrev($keywords), $count_wildcards - 1)); |
| 349 | unset($count_wildcards); |
| 350 | |
| 351 | // set the search_query which is shown to the user |
| 352 | $this->search_query = $keywords; |
| 353 | |
| 354 | $exact_words = array(); |
| 355 | preg_match_all('#([^\\s+\\-|()]+)(?:$|[\\s+\\-|()])#u', $keywords, $exact_words); |
| 356 | $exact_words = $exact_words[1]; |
| 357 | |
| 358 | $common_ids = $words = array(); |
| 359 | |
| 360 | if (count($exact_words)) |
| 361 | { |
| 362 | $sql = 'SELECT word_id, word_text, word_common |
| 363 | FROM ' . $this->search_wordlist_table . ' |
| 364 | WHERE ' . $this->db->sql_in_set('word_text', $exact_words) . ' |
| 365 | ORDER BY word_count ASC'; |
| 366 | $result = $this->db->sql_query($sql); |
| 367 | |
| 368 | // store an array of words and ids, remove common words |
| 369 | while ($row = $this->db->sql_fetchrow($result)) |
| 370 | { |
| 371 | if ($row['word_common']) |
| 372 | { |
| 373 | $this->common_words[] = $row['word_text']; |
| 374 | $common_ids[$row['word_text']] = (int) $row['word_id']; |
| 375 | continue; |
| 376 | } |
| 377 | |
| 378 | $words[$row['word_text']] = (int) $row['word_id']; |
| 379 | } |
| 380 | $this->db->sql_freeresult($result); |
| 381 | } |
| 382 | |
| 383 | // Handle +, - without preceding whitespace character |
| 384 | $match = array('#(\S)\+#', '#(\S)-#'); |
| 385 | $replace = array('$1 +', '$1 +'); |
| 386 | |
| 387 | $keywords = preg_replace($match, $replace, $keywords); |
| 388 | |
| 389 | // now analyse the search query, first split it using the spaces |
| 390 | $query = explode(' ', $keywords); |
| 391 | |
| 392 | $this->must_contain_ids = array(); |
| 393 | $this->must_not_contain_ids = array(); |
| 394 | $this->must_exclude_one_ids = array(); |
| 395 | |
| 396 | foreach ($query as $word) |
| 397 | { |
| 398 | if (empty($word)) |
| 399 | { |
| 400 | continue; |
| 401 | } |
| 402 | |
| 403 | // words which should not be included |
| 404 | if ($word[0] == '-') |
| 405 | { |
| 406 | $word = substr($word, 1); |
| 407 | |
| 408 | // a group of which at least one may not be in the resulting posts |
| 409 | if (isset($word[0]) && $word[0] == '(') |
| 410 | { |
| 411 | $word = array_unique(explode('|', substr($word, 1, -1))); |
| 412 | $mode = 'must_exclude_one'; |
| 413 | } |
| 414 | // one word which should not be in the resulting posts |
| 415 | else |
| 416 | { |
| 417 | $mode = 'must_not_contain'; |
| 418 | } |
| 419 | $ignore_no_id = true; |
| 420 | } |
| 421 | // words which have to be included |
| 422 | else |
| 423 | { |
| 424 | // no prefix is the same as a +prefix |
| 425 | if ($word[0] == '+') |
| 426 | { |
| 427 | $word = substr($word, 1); |
| 428 | } |
| 429 | |
| 430 | // a group of words of which at least one word should be in every resulting post |
| 431 | if (isset($word[0]) && $word[0] == '(') |
| 432 | { |
| 433 | $word = array_unique(explode('|', substr($word, 1, -1))); |
| 434 | } |
| 435 | $ignore_no_id = false; |
| 436 | $mode = 'must_contain'; |
| 437 | } |
| 438 | |
| 439 | if (empty($word)) |
| 440 | { |
| 441 | continue; |
| 442 | } |
| 443 | |
| 444 | // if this is an array of words then retrieve an id for each |
| 445 | if (is_array($word)) |
| 446 | { |
| 447 | $non_common_words = array(); |
| 448 | $id_words = array(); |
| 449 | foreach ($word as $i => $word_part) |
| 450 | { |
| 451 | if (strpos($word_part, '*') !== false) |
| 452 | { |
| 453 | $len = utf8_strlen(str_replace('*', '', $word_part)); |
| 454 | if ($len >= $this->word_length['min'] && $len <= $this->word_length['max']) |
| 455 | { |
| 456 | $id_words[] = '\'' . $this->db->sql_escape(str_replace('*', '%', $word_part)) . '\''; |
| 457 | $non_common_words[] = $word_part; |
| 458 | } |
| 459 | else |
| 460 | { |
| 461 | $this->common_words[] = $word_part; |
| 462 | } |
| 463 | } |
| 464 | else if (isset($words[$word_part])) |
| 465 | { |
| 466 | $id_words[] = $words[$word_part]; |
| 467 | $non_common_words[] = $word_part; |
| 468 | } |
| 469 | else |
| 470 | { |
| 471 | $len = utf8_strlen($word_part); |
| 472 | if ($len < $this->word_length['min'] || $len > $this->word_length['max']) |
| 473 | { |
| 474 | $this->common_words[] = $word_part; |
| 475 | } |
| 476 | } |
| 477 | } |
| 478 | if (count($id_words)) |
| 479 | { |
| 480 | sort($id_words); |
| 481 | if (count($id_words) > 1) |
| 482 | { |
| 483 | $this->{$mode . '_ids'}[] = $id_words; |
| 484 | } |
| 485 | else |
| 486 | { |
| 487 | $mode = ($mode == 'must_exclude_one') ? 'must_not_contain' : $mode; |
| 488 | $this->{$mode . '_ids'}[] = $id_words[0]; |
| 489 | } |
| 490 | } |
| 491 | // throw an error if we shall not ignore unexistant words |
| 492 | else if (!$ignore_no_id && count($non_common_words)) |
| 493 | { |
| 494 | trigger_error(sprintf($this->language->lang('WORDS_IN_NO_POST'), implode($this->language->lang('COMMA_SEPARATOR'), $non_common_words))); |
| 495 | } |
| 496 | unset($non_common_words); |
| 497 | } |
| 498 | // else we only need one id |
| 499 | else if (($wildcard = strpos($word, '*') !== false) || isset($words[$word])) |
| 500 | { |
| 501 | if ($wildcard) |
| 502 | { |
| 503 | $len = utf8_strlen(str_replace('*', '', $word)); |
| 504 | if ($len >= $this->word_length['min'] && $len <= $this->word_length['max']) |
| 505 | { |
| 506 | $this->{$mode . '_ids'}[] = '\'' . $this->db->sql_escape(str_replace('*', '%', $word)) . '\''; |
| 507 | } |
| 508 | else |
| 509 | { |
| 510 | $this->common_words[] = $word; |
| 511 | } |
| 512 | } |
| 513 | else |
| 514 | { |
| 515 | $this->{$mode . '_ids'}[] = $words[$word]; |
| 516 | } |
| 517 | } |
| 518 | else |
| 519 | { |
| 520 | if (!isset($common_ids[$word])) |
| 521 | { |
| 522 | $len = utf8_strlen($word); |
| 523 | if ($len < $this->word_length['min'] || $len > $this->word_length['max']) |
| 524 | { |
| 525 | $this->common_words[] = $word; |
| 526 | } |
| 527 | } |
| 528 | } |
| 529 | } |
| 530 | |
| 531 | // Return true if all words are not common words |
| 532 | if (count($exact_words) - count($this->common_words) > 0) |
| 533 | { |
| 534 | return true; |
| 535 | } |
| 536 | return false; |
| 537 | } |
| 538 | |
| 539 | /** |
| 540 | * {@inheritdoc} |
| 541 | */ |
| 542 | public function keyword_search(string $type, string $fields, string $terms, array $sort_by_sql, string $sort_key, string $sort_dir, string $sort_days, array $ex_fid_ary, string $post_visibility, int $topic_id, array $author_ary, string $author_name, array &$id_ary, int &$start, int $per_page) |
| 543 | { |
| 544 | // No keywords? No posts. |
| 545 | if (empty($this->search_query)) |
| 546 | { |
| 547 | return false; |
| 548 | } |
| 549 | |
| 550 | // we can't search for negatives only |
| 551 | if (empty($this->must_contain_ids)) |
| 552 | { |
| 553 | return false; |
| 554 | } |
| 555 | |
| 556 | $must_contain_ids = $this->must_contain_ids; |
| 557 | $must_not_contain_ids = $this->must_not_contain_ids; |
| 558 | $must_exclude_one_ids = $this->must_exclude_one_ids; |
| 559 | |
| 560 | sort($must_contain_ids); |
| 561 | sort($must_not_contain_ids); |
| 562 | sort($must_exclude_one_ids); |
| 563 | |
| 564 | // generate a search_key from all the options to identify the results |
| 565 | $search_key_array = array( |
| 566 | serialize($must_contain_ids), |
| 567 | serialize($must_not_contain_ids), |
| 568 | serialize($must_exclude_one_ids), |
| 569 | $type, |
| 570 | $fields, |
| 571 | $terms, |
| 572 | $sort_days, |
| 573 | $sort_key, |
| 574 | $topic_id, |
| 575 | implode(',', $ex_fid_ary), |
| 576 | $post_visibility, |
| 577 | implode(',', $author_ary), |
| 578 | $author_name, |
| 579 | ); |
| 580 | |
| 581 | /** |
| 582 | * Allow changing the search_key for cached results |
| 583 | * |
| 584 | * @event core.search_native_by_keyword_modify_search_key |
| 585 | * @var array search_key_array Array with search parameters to generate the search_key |
| 586 | * @var array must_contain_ids Array with post ids of posts containing words that are to be included |
| 587 | * @var array must_not_contain_ids Array with post ids of posts containing words that should not be included |
| 588 | * @var array must_exclude_one_ids Array with post ids of posts containing at least one word that needs to be excluded |
| 589 | * @var string type Searching type ('posts', 'topics') |
| 590 | * @var string fields Searching fields ('titleonly', 'msgonly', 'firstpost', 'all') |
| 591 | * @var string terms Searching terms ('all', 'any') |
| 592 | * @var int sort_days Time, in days, of the oldest possible post to list |
| 593 | * @var string sort_key The sort type used from the possible sort types |
| 594 | * @var int topic_id Limit the search to this topic_id only |
| 595 | * @var array ex_fid_ary Which forums not to search on |
| 596 | * @var string post_visibility Post visibility data |
| 597 | * @var array author_ary Array of user_id containing the users to filter the results to |
| 598 | * @since 3.1.7-RC1 |
| 599 | */ |
| 600 | $vars = array( |
| 601 | 'search_key_array', |
| 602 | 'must_contain_ids', |
| 603 | 'must_not_contain_ids', |
| 604 | 'must_exclude_one_ids', |
| 605 | 'type', |
| 606 | 'fields', |
| 607 | 'terms', |
| 608 | 'sort_days', |
| 609 | 'sort_key', |
| 610 | 'topic_id', |
| 611 | 'ex_fid_ary', |
| 612 | 'post_visibility', |
| 613 | 'author_ary', |
| 614 | ); |
| 615 | extract($this->phpbb_dispatcher->trigger_event('core.search_native_by_keyword_modify_search_key', compact($vars))); |
| 616 | |
| 617 | $search_key = md5(implode('#', $search_key_array)); |
| 618 | |
| 619 | // try reading the results from cache |
| 620 | $total_results = 0; |
| 621 | if ($this->obtain_ids($search_key, $total_results, $id_ary, $start, $per_page, $sort_dir) == self::SEARCH_RESULT_IN_CACHE) |
| 622 | { |
| 623 | return $total_results; |
| 624 | } |
| 625 | |
| 626 | $id_ary = array(); |
| 627 | |
| 628 | $sql_where = array(); |
| 629 | $m_num = 0; |
| 630 | $w_num = 0; |
| 631 | |
| 632 | $sql_array = array( |
| 633 | 'SELECT' => ($type == 'posts') ? 'DISTINCT p.post_id' : 'DISTINCT p.topic_id', |
| 634 | 'FROM' => array( |
| 635 | $this->search_wordmatch_table => array(), |
| 636 | $this->search_wordlist_table => array(), |
| 637 | ), |
| 638 | 'LEFT_JOIN' => array(array( |
| 639 | 'FROM' => array(POSTS_TABLE => 'p'), |
| 640 | 'ON' => 'm0.post_id = p.post_id', |
| 641 | )), |
| 642 | ); |
| 643 | |
| 644 | $title_match = ''; |
| 645 | $left_join_topics = false; |
| 646 | $group_by = true; |
| 647 | // Build some display specific sql strings |
| 648 | switch ($fields) |
| 649 | { |
| 650 | case 'titleonly': |
| 651 | $title_match = 'title_match = 1'; |
| 652 | $group_by = false; |
| 653 | // no break |
| 654 | case 'firstpost': |
| 655 | $left_join_topics = true; |
| 656 | $sql_where[] = 'p.post_id = t.topic_first_post_id'; |
| 657 | break; |
| 658 | |
| 659 | case 'msgonly': |
| 660 | $title_match = 'title_match = 0'; |
| 661 | $group_by = false; |
| 662 | break; |
| 663 | } |
| 664 | |
| 665 | if ($type == 'topics') |
| 666 | { |
| 667 | $left_join_topics = true; |
| 668 | $group_by = true; |
| 669 | } |
| 670 | |
| 671 | /** |
| 672 | * @todo Add a query optimizer (handle stuff like "+(4|3) +4") |
| 673 | */ |
| 674 | |
| 675 | foreach ($this->must_contain_ids as $subquery) |
| 676 | { |
| 677 | if (is_array($subquery)) |
| 678 | { |
| 679 | $group_by = true; |
| 680 | |
| 681 | $word_id_sql = array(); |
| 682 | $word_ids = array(); |
| 683 | foreach ($subquery as $id) |
| 684 | { |
| 685 | if (is_string($id)) |
| 686 | { |
| 687 | $sql_array['LEFT_JOIN'][] = array( |
| 688 | 'FROM' => array($this->search_wordlist_table => 'w' . $w_num), |
| 689 | 'ON' => "w$w_num.word_text LIKE $id" |
| 690 | ); |
| 691 | $word_ids[] = "w$w_num.word_id"; |
| 692 | |
| 693 | $w_num++; |
| 694 | } |
| 695 | else |
| 696 | { |
| 697 | $word_ids[] = $id; |
| 698 | } |
| 699 | } |
| 700 | |
| 701 | $sql_where[] = $this->db->sql_in_set("m$m_num.word_id", $word_ids); |
| 702 | |
| 703 | unset($word_id_sql); |
| 704 | unset($word_ids); |
| 705 | } |
| 706 | else if (is_string($subquery)) |
| 707 | { |
| 708 | $sql_array['FROM'][$this->search_wordlist_table][] = 'w' . $w_num; |
| 709 | |
| 710 | $sql_where[] = "w$w_num.word_text LIKE $subquery"; |
| 711 | $sql_where[] = "m$m_num.word_id = w$w_num.word_id"; |
| 712 | |
| 713 | $group_by = true; |
| 714 | $w_num++; |
| 715 | } |
| 716 | else |
| 717 | { |
| 718 | $sql_where[] = "m$m_num.word_id = $subquery"; |
| 719 | } |
| 720 | |
| 721 | $sql_array['FROM'][$this->search_wordmatch_table][] = 'm' . $m_num; |
| 722 | |
| 723 | if ($title_match) |
| 724 | { |
| 725 | $sql_where[] = "m$m_num.$title_match"; |
| 726 | } |
| 727 | |
| 728 | if ($m_num != 0) |
| 729 | { |
| 730 | $sql_where[] = "m$m_num.post_id = m0.post_id"; |
| 731 | } |
| 732 | $m_num++; |
| 733 | } |
| 734 | |
| 735 | foreach ($this->must_not_contain_ids as $key => $subquery) |
| 736 | { |
| 737 | if (is_string($subquery)) |
| 738 | { |
| 739 | $sql_array['LEFT_JOIN'][] = array( |
| 740 | 'FROM' => array($this->search_wordlist_table => 'w' . $w_num), |
| 741 | 'ON' => "w$w_num.word_text LIKE $subquery" |
| 742 | ); |
| 743 | |
| 744 | $this->must_not_contain_ids[$key] = "w$w_num.word_id"; |
| 745 | |
| 746 | $group_by = true; |
| 747 | $w_num++; |
| 748 | } |
| 749 | } |
| 750 | |
| 751 | if (count($this->must_not_contain_ids)) |
| 752 | { |
| 753 | $sql_array['LEFT_JOIN'][] = array( |
| 754 | 'FROM' => array($this->search_wordmatch_table => 'm' . $m_num), |
| 755 | 'ON' => $this->db->sql_in_set("m$m_num.word_id", $this->must_not_contain_ids) . (($title_match) ? " AND m$m_num.$title_match" : '') . " AND m$m_num.post_id = m0.post_id" |
| 756 | ); |
| 757 | |
| 758 | $sql_where[] = "m$m_num.word_id IS NULL"; |
| 759 | $m_num++; |
| 760 | } |
| 761 | |
| 762 | foreach ($this->must_exclude_one_ids as $ids) |
| 763 | { |
| 764 | $is_null_joins = array(); |
| 765 | foreach ($ids as $id) |
| 766 | { |
| 767 | if (is_string($id)) |
| 768 | { |
| 769 | $sql_array['LEFT_JOIN'][] = array( |
| 770 | 'FROM' => array($this->search_wordlist_table => 'w' . $w_num), |
| 771 | 'ON' => "w$w_num.word_text LIKE $id" |
| 772 | ); |
| 773 | $id = "w$w_num.word_id"; |
| 774 | |
| 775 | $group_by = true; |
| 776 | $w_num++; |
| 777 | } |
| 778 | |
| 779 | $sql_array['LEFT_JOIN'][] = array( |
| 780 | 'FROM' => array($this->search_wordmatch_table => 'm' . $m_num), |
| 781 | 'ON' => "m$m_num.word_id = $id AND m$m_num.post_id = m0.post_id" . (($title_match) ? " AND m$m_num.$title_match" : '') |
| 782 | ); |
| 783 | $is_null_joins[] = "m$m_num.word_id IS NULL"; |
| 784 | |
| 785 | $m_num++; |
| 786 | } |
| 787 | $sql_where[] = '(' . implode(' OR ', $is_null_joins) . ')'; |
| 788 | } |
| 789 | |
| 790 | $sql_where[] = $post_visibility; |
| 791 | |
| 792 | $search_query = $this->search_query; |
| 793 | $must_exclude_one_ids = $this->must_exclude_one_ids; |
| 794 | $must_not_contain_ids = $this->must_not_contain_ids; |
| 795 | $must_contain_ids = $this->must_contain_ids; |
| 796 | |
| 797 | $sql_sort_table = $sql_sort_join = $sql_match = $sql_match_where = $sql_sort = ''; |
| 798 | |
| 799 | /** |
| 800 | * Allow changing the query used for counting for posts using fulltext_native |
| 801 | * |
| 802 | * @event core.search_native_keywords_count_query_before |
| 803 | * @var string search_query The parsed keywords used for this search |
| 804 | * @var array must_not_contain_ids Ids that cannot be taken into account for the results |
| 805 | * @var array must_exclude_one_ids Ids that cannot be on the results |
| 806 | * @var array must_contain_ids Ids that must be on the results |
| 807 | * @var int total_results The previous result count for the format of the query |
| 808 | * Set to 0 to force a re-count |
| 809 | * @var array sql_array The data on how to search in the DB at this point |
| 810 | * @var bool left_join_topics Whether or not TOPICS_TABLE should be CROSS JOIN'ED |
| 811 | * @var array author_ary Array of user_id containing the users to filter the results to |
| 812 | * @var string author_name An extra username to search on (!empty(author_ary) must be true, to be relevant) |
| 813 | * @var array ex_fid_ary Which forums not to search on |
| 814 | * @var int topic_id Limit the search to this topic_id only |
| 815 | * @var string sql_sort_table Extra tables to include in the SQL query. |
| 816 | * Used in conjunction with sql_sort_join |
| 817 | * @var string sql_sort_join SQL conditions to join all the tables used together. |
| 818 | * Used in conjunction with sql_sort_table |
| 819 | * @var int sort_days Time, in days, of the oldest possible post to list |
| 820 | * @var string sql_where An array of the current WHERE clause conditions |
| 821 | * @var string sql_match Which columns to do the search on |
| 822 | * @var string sql_match_where Extra conditions to use to properly filter the matching process |
| 823 | * @var bool group_by Whether or not the SQL query requires a GROUP BY for the elements in the SELECT clause |
| 824 | * @var string sort_by_sql The possible predefined sort types |
| 825 | * @var string sort_key The sort type used from the possible sort types |
| 826 | * @var string sort_dir "a" for ASC or "d" dor DESC for the sort order used |
| 827 | * @var string sql_sort The result SQL when processing sort_by_sql + sort_key + sort_dir |
| 828 | * @var int start How many posts to skip in the search results (used for pagination) |
| 829 | * @since 3.1.5-RC1 |
| 830 | */ |
| 831 | $vars = array( |
| 832 | 'search_query', |
| 833 | 'must_not_contain_ids', |
| 834 | 'must_exclude_one_ids', |
| 835 | 'must_contain_ids', |
| 836 | 'total_results', |
| 837 | 'sql_array', |
| 838 | 'left_join_topics', |
| 839 | 'author_ary', |
| 840 | 'author_name', |
| 841 | 'ex_fid_ary', |
| 842 | 'topic_id', |
| 843 | 'sql_sort_table', |
| 844 | 'sql_sort_join', |
| 845 | 'sort_days', |
| 846 | 'sql_where', |
| 847 | 'sql_match', |
| 848 | 'sql_match_where', |
| 849 | 'group_by', |
| 850 | 'sort_by_sql', |
| 851 | 'sort_key', |
| 852 | 'sort_dir', |
| 853 | 'sql_sort', |
| 854 | 'start', |
| 855 | ); |
| 856 | extract($this->phpbb_dispatcher->trigger_event('core.search_native_keywords_count_query_before', compact($vars))); |
| 857 | |
| 858 | if ($topic_id) |
| 859 | { |
| 860 | $sql_where[] = 'p.topic_id = ' . $topic_id; |
| 861 | } |
| 862 | |
| 863 | if (count($author_ary)) |
| 864 | { |
| 865 | if ($author_name) |
| 866 | { |
| 867 | // first one matches post of registered users, second one guests and deleted users |
| 868 | $sql_author = '(' . $this->db->sql_in_set('p.poster_id', array_diff($author_ary, array(ANONYMOUS)), false, true) . ' OR p.post_username ' . $author_name . ')'; |
| 869 | } |
| 870 | else |
| 871 | { |
| 872 | $sql_author = $this->db->sql_in_set('p.poster_id', $author_ary); |
| 873 | } |
| 874 | $sql_where[] = $sql_author; |
| 875 | } |
| 876 | |
| 877 | if (count($ex_fid_ary)) |
| 878 | { |
| 879 | $sql_where[] = $this->db->sql_in_set('p.forum_id', $ex_fid_ary, true); |
| 880 | } |
| 881 | |
| 882 | if ($sort_days) |
| 883 | { |
| 884 | $sql_where[] = 'p.post_time >= ' . (time() - ($sort_days * 86400)); |
| 885 | } |
| 886 | |
| 887 | $sql_array['WHERE'] = implode(' AND ', $sql_where); |
| 888 | |
| 889 | $is_mysql = false; |
| 890 | // if the total result count is not cached yet, retrieve it from the db |
| 891 | if (!$total_results) |
| 892 | { |
| 893 | $sql = ''; |
| 894 | $sql_array_count = $sql_array; |
| 895 | |
| 896 | if ($left_join_topics) |
| 897 | { |
| 898 | $sql_array_count['LEFT_JOIN'][] = array( |
| 899 | 'FROM' => array(TOPICS_TABLE => 't'), |
| 900 | 'ON' => 'p.topic_id = t.topic_id' |
| 901 | ); |
| 902 | } |
| 903 | |
| 904 | switch ($this->db->get_sql_layer()) |
| 905 | { |
| 906 | case 'mysqli': |
| 907 | $is_mysql = true; |
| 908 | |
| 909 | break; |
| 910 | |
| 911 | case 'sqlite3': |
| 912 | $sql_array_count['SELECT'] = ($type == 'posts') ? 'DISTINCT p.post_id' : 'DISTINCT p.topic_id'; |
| 913 | $sql = 'SELECT COUNT(' . (($type == 'posts') ? 'post_id' : 'topic_id') . ') as total_results |
| 914 | FROM (' . $this->db->sql_build_query('SELECT', $sql_array_count) . ')'; |
| 915 | |
| 916 | // no break |
| 917 | |
| 918 | default: |
| 919 | $sql_array_count['SELECT'] = ($type == 'posts') ? 'COUNT(DISTINCT p.post_id) AS total_results' : 'COUNT(DISTINCT p.topic_id) AS total_results'; |
| 920 | $sql = (!$sql) ? $this->db->sql_build_query('SELECT', $sql_array_count) : $sql; |
| 921 | |
| 922 | $result = $this->db->sql_query($sql); |
| 923 | $total_results = (int) $this->db->sql_fetchfield('total_results'); |
| 924 | $this->db->sql_freeresult($result); |
| 925 | |
| 926 | if (!$total_results) |
| 927 | { |
| 928 | return false; |
| 929 | } |
| 930 | break; |
| 931 | } |
| 932 | |
| 933 | unset($sql_array_count, $sql); |
| 934 | } |
| 935 | |
| 936 | // Build sql strings for sorting |
| 937 | $sql_sort = $sort_by_sql[$sort_key] . (($sort_dir == 'a') ? ' ASC' : ' DESC'); |
| 938 | |
| 939 | switch ($sql_sort[0]) |
| 940 | { |
| 941 | case 'u': |
| 942 | $sql_array['FROM'][USERS_TABLE] = 'u'; |
| 943 | $sql_where[] = 'u.user_id = p.poster_id '; |
| 944 | break; |
| 945 | |
| 946 | case 't': |
| 947 | $left_join_topics = true; |
| 948 | break; |
| 949 | |
| 950 | case 'f': |
| 951 | $sql_array['FROM'][FORUMS_TABLE] = 'f'; |
| 952 | $sql_where[] = 'f.forum_id = p.forum_id'; |
| 953 | break; |
| 954 | } |
| 955 | |
| 956 | if ($left_join_topics) |
| 957 | { |
| 958 | $sql_array['LEFT_JOIN'][] = array( |
| 959 | 'FROM' => array(TOPICS_TABLE => 't'), |
| 960 | 'ON' => 'p.topic_id = t.topic_id' |
| 961 | ); |
| 962 | } |
| 963 | |
| 964 | $sql_array['WHERE'] = implode(' AND ', $sql_where); |
| 965 | $sql_array['GROUP_BY'] = ($group_by) ? (($type == 'posts') ? 'p.post_id' : 'p.topic_id') . ', ' . $sort_by_sql[$sort_key] : ''; |
| 966 | $sql_array['ORDER_BY'] = $sql_sort; |
| 967 | $sql_array['SELECT'] .= $sort_by_sql[$sort_key] ? ", {$sort_by_sql[$sort_key]}" : ''; |
| 968 | |
| 969 | unset($sql_where, $sql_sort, $group_by); |
| 970 | |
| 971 | $sql = $this->db->sql_build_query('SELECT', $sql_array); |
| 972 | $result = $this->db->sql_query_limit($sql, $this->config['search_block_size'], $start); |
| 973 | |
| 974 | while ($row = $this->db->sql_fetchrow($result)) |
| 975 | { |
| 976 | $id_ary[] = (int) $row[(($type == 'posts') ? 'post_id' : 'topic_id')]; |
| 977 | } |
| 978 | $this->db->sql_freeresult($result); |
| 979 | |
| 980 | // If using mysql and the total result count is not calculated yet, get it from the db |
| 981 | if (!$total_results && $is_mysql) |
| 982 | { |
| 983 | $sql_count = str_replace("SELECT {$sql_array['SELECT']}", "SELECT COUNT({$sql_array['SELECT']}) as total_results", $sql); |
| 984 | $result = $this->db->sql_query($sql_count); |
| 985 | $total_results = $sql_array['GROUP_BY'] ? count($this->db->sql_fetchrowset($result)) : $this->db->sql_fetchfield('total_results'); |
| 986 | $this->db->sql_freeresult($result); |
| 987 | |
| 988 | if (!$total_results) |
| 989 | { |
| 990 | return false; |
| 991 | } |
| 992 | } |
| 993 | |
| 994 | if ($start >= $total_results) |
| 995 | { |
| 996 | $start = floor(($total_results - 1) / $per_page) * $per_page; |
| 997 | |
| 998 | $result = $this->db->sql_query_limit($sql, $this->config['search_block_size'], $start); |
| 999 | |
| 1000 | while ($row = $this->db->sql_fetchrow($result)) |
| 1001 | { |
| 1002 | $id_ary[] = (int) $row[(($type == 'posts') ? 'post_id' : 'topic_id')]; |
| 1003 | } |
| 1004 | $this->db->sql_freeresult($result); |
| 1005 | } |
| 1006 | |
| 1007 | // store the ids, from start on then delete anything that isn't on the current page because we only need ids for one page |
| 1008 | $this->save_ids($search_key, $this->search_query, $author_ary, $total_results, $id_ary, $start, $sort_dir); |
| 1009 | $id_ary = array_slice($id_ary, 0, (int) $per_page); |
| 1010 | |
| 1011 | return $total_results; |
| 1012 | } |
| 1013 | |
| 1014 | /** |
| 1015 | * {@inheritdoc} |
| 1016 | */ |
| 1017 | public function author_search(string $type, bool $firstpost_only, array $sort_by_sql, string $sort_key, string $sort_dir, string $sort_days, array $ex_fid_ary, string $post_visibility, int $topic_id, array $author_ary, string $author_name, array &$id_ary, int &$start, int $per_page) |
| 1018 | { |
| 1019 | // No author? No posts |
| 1020 | if (!count($author_ary)) |
| 1021 | { |
| 1022 | return 0; |
| 1023 | } |
| 1024 | |
| 1025 | // generate a search_key from all the options to identify the results |
| 1026 | $search_key_array = array( |
| 1027 | '', |
| 1028 | $type, |
| 1029 | ($firstpost_only) ? 'firstpost' : '', |
| 1030 | '', |
| 1031 | '', |
| 1032 | $sort_days, |
| 1033 | $sort_key, |
| 1034 | $topic_id, |
| 1035 | implode(',', $ex_fid_ary), |
| 1036 | $post_visibility, |
| 1037 | implode(',', $author_ary), |
| 1038 | $author_name, |
| 1039 | ); |
| 1040 | |
| 1041 | /** |
| 1042 | * Allow changing the search_key for cached results |
| 1043 | * |
| 1044 | * @event core.search_native_by_author_modify_search_key |
| 1045 | * @var array search_key_array Array with search parameters to generate the search_key |
| 1046 | * @var string type Searching type ('posts', 'topics') |
| 1047 | * @var boolean firstpost_only Flag indicating if only topic starting posts are considered |
| 1048 | * @var int sort_days Time, in days, of the oldest possible post to list |
| 1049 | * @var string sort_key The sort type used from the possible sort types |
| 1050 | * @var int topic_id Limit the search to this topic_id only |
| 1051 | * @var array ex_fid_ary Which forums not to search on |
| 1052 | * @var string post_visibility Post visibility data |
| 1053 | * @var array author_ary Array of user_id containing the users to filter the results to |
| 1054 | * @var string author_name The username to search on |
| 1055 | * @since 3.1.7-RC1 |
| 1056 | */ |
| 1057 | $vars = array( |
| 1058 | 'search_key_array', |
| 1059 | 'type', |
| 1060 | 'firstpost_only', |
| 1061 | 'sort_days', |
| 1062 | 'sort_key', |
| 1063 | 'topic_id', |
| 1064 | 'ex_fid_ary', |
| 1065 | 'post_visibility', |
| 1066 | 'author_ary', |
| 1067 | 'author_name', |
| 1068 | ); |
| 1069 | extract($this->phpbb_dispatcher->trigger_event('core.search_native_by_author_modify_search_key', compact($vars))); |
| 1070 | |
| 1071 | $search_key = md5(implode('#', $search_key_array)); |
| 1072 | |
| 1073 | // try reading the results from cache |
| 1074 | $total_results = 0; |
| 1075 | if ($this->obtain_ids($search_key, $total_results, $id_ary, $start, $per_page, $sort_dir) == self::SEARCH_RESULT_IN_CACHE) |
| 1076 | { |
| 1077 | return $total_results; |
| 1078 | } |
| 1079 | |
| 1080 | $id_ary = array(); |
| 1081 | |
| 1082 | // Create some display specific sql strings |
| 1083 | if ($author_name) |
| 1084 | { |
| 1085 | // first one matches post of registered users, second one guests and deleted users |
| 1086 | $sql_author = '(' . $this->db->sql_in_set('p.poster_id', array_diff($author_ary, array(ANONYMOUS)), false, true) . ' OR p.post_username ' . $author_name . ')'; |
| 1087 | } |
| 1088 | else |
| 1089 | { |
| 1090 | $sql_author = $this->db->sql_in_set('p.poster_id', $author_ary); |
| 1091 | } |
| 1092 | $sql_fora = (count($ex_fid_ary)) ? ' AND ' . $this->db->sql_in_set('p.forum_id', $ex_fid_ary, true) : ''; |
| 1093 | $sql_time = ($sort_days) ? ' AND p.post_time >= ' . (time() - ($sort_days * 86400)) : ''; |
| 1094 | $sql_topic_id = ($topic_id) ? ' AND p.topic_id = ' . (int) $topic_id : ''; |
| 1095 | $sql_firstpost = ($firstpost_only) ? ' AND p.post_id = t.topic_first_post_id' : ''; |
| 1096 | $post_visibility = ($post_visibility) ? ' AND ' . $post_visibility : ''; |
| 1097 | |
| 1098 | // Build sql strings for sorting |
| 1099 | $sql_sort = $sort_by_sql[$sort_key] . (($sort_dir == 'a') ? ' ASC' : ' DESC'); |
| 1100 | $sql_sort_table = $sql_sort_join = ''; |
| 1101 | switch ($sql_sort[0]) |
| 1102 | { |
| 1103 | case 'u': |
| 1104 | $sql_sort_table = USERS_TABLE . ' u, '; |
| 1105 | $sql_sort_join = ' AND u.user_id = p.poster_id '; |
| 1106 | break; |
| 1107 | |
| 1108 | case 't': |
| 1109 | $sql_sort_table = ($type == 'posts' && !$firstpost_only) ? TOPICS_TABLE . ' t, ' : ''; |
| 1110 | $sql_sort_join = ($type == 'posts' && !$firstpost_only) ? ' AND t.topic_id = p.topic_id ' : ''; |
| 1111 | break; |
| 1112 | |
| 1113 | case 'f': |
| 1114 | $sql_sort_table = FORUMS_TABLE . ' f, '; |
| 1115 | $sql_sort_join = ' AND f.forum_id = p.forum_id '; |
| 1116 | break; |
| 1117 | } |
| 1118 | |
| 1119 | $select = ($type == 'posts') ? 'p.post_id' : 't.topic_id'; |
| 1120 | $select .= $sort_by_sql[$sort_key] ? ", {$sort_by_sql[$sort_key]}" : ''; |
| 1121 | $is_mysql = false; |
| 1122 | |
| 1123 | /** |
| 1124 | * Allow changing the query used to search for posts by author in fulltext_native |
| 1125 | * |
| 1126 | * @event core.search_native_author_count_query_before |
| 1127 | * @var int total_results The previous result count for the format of the query. |
| 1128 | * Set to 0 to force a re-count |
| 1129 | * @var string type The type of search being made |
| 1130 | * @var string select SQL SELECT clause for what to get |
| 1131 | * @var string sql_sort_table CROSS JOIN'ed table to allow doing the sort chosen |
| 1132 | * @var string sql_sort_join Condition to define how to join the CROSS JOIN'ed table specifyed in sql_sort_table |
| 1133 | * @var array sql_author SQL WHERE condition for the post author ids |
| 1134 | * @var int topic_id Limit the search to this topic_id only |
| 1135 | * @var string sort_by_sql The possible predefined sort types |
| 1136 | * @var string sort_key The sort type used from the possible sort types |
| 1137 | * @var string sort_dir "a" for ASC or "d" dor DESC for the sort order used |
| 1138 | * @var string sql_sort The result SQL when processing sort_by_sql + sort_key + sort_dir |
| 1139 | * @var string sort_days Time, in days, that the oldest post showing can have |
| 1140 | * @var string sql_time The SQL to search on the time specifyed by sort_days |
| 1141 | * @var bool firstpost_only Wether or not to search only on the first post of the topics |
| 1142 | * @var string sql_firstpost The SQL used in the WHERE claused to filter by firstpost. |
| 1143 | * @var array ex_fid_ary Forum ids that must not be searched on |
| 1144 | * @var array sql_fora SQL query for ex_fid_ary |
| 1145 | * @var int start How many posts to skip in the search results (used for pagination) |
| 1146 | * @since 3.1.5-RC1 |
| 1147 | */ |
| 1148 | $vars = array( |
| 1149 | 'total_results', |
| 1150 | 'type', |
| 1151 | 'select', |
| 1152 | 'sql_sort_table', |
| 1153 | 'sql_sort_join', |
| 1154 | 'sql_author', |
| 1155 | 'topic_id', |
| 1156 | 'sort_by_sql', |
| 1157 | 'sort_key', |
| 1158 | 'sort_dir', |
| 1159 | 'sql_sort', |
| 1160 | 'sort_days', |
| 1161 | 'sql_time', |
| 1162 | 'firstpost_only', |
| 1163 | 'sql_firstpost', |
| 1164 | 'ex_fid_ary', |
| 1165 | 'sql_fora', |
| 1166 | 'start', |
| 1167 | ); |
| 1168 | extract($this->phpbb_dispatcher->trigger_event('core.search_native_author_count_query_before', compact($vars))); |
| 1169 | |
| 1170 | // If the cache was completely empty count the results |
| 1171 | if (!$total_results) |
| 1172 | { |
| 1173 | switch ($this->db->get_sql_layer()) |
| 1174 | { |
| 1175 | case 'mysqli': |
| 1176 | $is_mysql = true; |
| 1177 | break; |
| 1178 | |
| 1179 | default: |
| 1180 | if ($type == 'posts') |
| 1181 | { |
| 1182 | $sql = 'SELECT COUNT(p.post_id) as total_results |
| 1183 | FROM ' . POSTS_TABLE . ' p' . (($firstpost_only) ? ', ' . TOPICS_TABLE . ' t ' : ' ') . " |
| 1184 | WHERE $sql_author |
| 1185 | $sql_topic_id |
| 1186 | $sql_firstpost |
| 1187 | $post_visibility |
| 1188 | $sql_fora |
| 1189 | $sql_time"; |
| 1190 | } |
| 1191 | else |
| 1192 | { |
| 1193 | if ($this->db->get_sql_layer() == 'sqlite3') |
| 1194 | { |
| 1195 | $sql = 'SELECT COUNT(topic_id) as total_results |
| 1196 | FROM (SELECT DISTINCT t.topic_id'; |
| 1197 | } |
| 1198 | else |
| 1199 | { |
| 1200 | $sql = 'SELECT COUNT(DISTINCT t.topic_id) as total_results'; |
| 1201 | } |
| 1202 | |
| 1203 | $sql .= ' FROM ' . TOPICS_TABLE . ' t, ' . POSTS_TABLE . " p |
| 1204 | WHERE $sql_author |
| 1205 | $sql_topic_id |
| 1206 | $sql_firstpost |
| 1207 | $post_visibility |
| 1208 | $sql_fora |
| 1209 | AND t.topic_id = p.topic_id |
| 1210 | $sql_time" . ($this->db->get_sql_layer() == 'sqlite3' ? ')' : ''); |
| 1211 | } |
| 1212 | $result = $this->db->sql_query($sql); |
| 1213 | |
| 1214 | $total_results = (int) $this->db->sql_fetchfield('total_results'); |
| 1215 | $this->db->sql_freeresult($result); |
| 1216 | |
| 1217 | if (!$total_results) |
| 1218 | { |
| 1219 | return false; |
| 1220 | } |
| 1221 | break; |
| 1222 | } |
| 1223 | } |
| 1224 | |
| 1225 | // Build the query for really selecting the post_ids |
| 1226 | if ($type == 'posts') |
| 1227 | { |
| 1228 | $sql = "SELECT $select |
| 1229 | FROM " . $sql_sort_table . POSTS_TABLE . ' p' . (($firstpost_only) ? ', ' . TOPICS_TABLE . ' t' : '') . " |
| 1230 | WHERE $sql_author |
| 1231 | $sql_topic_id |
| 1232 | $sql_firstpost |
| 1233 | $post_visibility |
| 1234 | $sql_fora |
| 1235 | $sql_sort_join |
| 1236 | $sql_time |
| 1237 | ORDER BY $sql_sort"; |
| 1238 | $field = 'post_id'; |
| 1239 | } |
| 1240 | else |
| 1241 | { |
| 1242 | $sql = "SELECT $select |
| 1243 | FROM " . $sql_sort_table . TOPICS_TABLE . ' t, ' . POSTS_TABLE . " p |
| 1244 | WHERE $sql_author |
| 1245 | $sql_topic_id |
| 1246 | $sql_firstpost |
| 1247 | $post_visibility |
| 1248 | $sql_fora |
| 1249 | AND t.topic_id = p.topic_id |
| 1250 | $sql_sort_join |
| 1251 | $sql_time |
| 1252 | GROUP BY t.topic_id, " . $sort_by_sql[$sort_key] . ' |
| 1253 | ORDER BY ' . $sql_sort; |
| 1254 | $field = 'topic_id'; |
| 1255 | } |
| 1256 | |
| 1257 | // Only read one block of posts from the db and then cache it |
| 1258 | $result = $this->db->sql_query_limit($sql, $this->config['search_block_size'], $start); |
| 1259 | |
| 1260 | while ($row = $this->db->sql_fetchrow($result)) |
| 1261 | { |
| 1262 | $id_ary[] = (int) $row[$field]; |
| 1263 | } |
| 1264 | $this->db->sql_freeresult($result); |
| 1265 | |
| 1266 | if (!$total_results && $is_mysql) |
| 1267 | { |
| 1268 | $sql_count = str_replace("SELECT $select", "SELECT COUNT(*) as total_results", $sql); |
| 1269 | $result = $this->db->sql_query($sql_count); |
| 1270 | $total_results = ($type == 'posts') ? (int) $this->db->sql_fetchfield('total_results') : count($this->db->sql_fetchrowset($result)); |
| 1271 | $this->db->sql_freeresult($result); |
| 1272 | |
| 1273 | if (!$total_results) |
| 1274 | { |
| 1275 | return false; |
| 1276 | } |
| 1277 | } |
| 1278 | |
| 1279 | if ($start >= $total_results) |
| 1280 | { |
| 1281 | $start = floor(($total_results - 1) / $per_page) * $per_page; |
| 1282 | |
| 1283 | $result = $this->db->sql_query_limit($sql, $this->config['search_block_size'], $start); |
| 1284 | |
| 1285 | while ($row = $this->db->sql_fetchrow($result)) |
| 1286 | { |
| 1287 | $id_ary[] = (int) $row[$field]; |
| 1288 | } |
| 1289 | $this->db->sql_freeresult($result); |
| 1290 | } |
| 1291 | |
| 1292 | if (count($id_ary)) |
| 1293 | { |
| 1294 | $this->save_ids($search_key, '', $author_ary, $total_results, $id_ary, $start, $sort_dir); |
| 1295 | $id_ary = array_slice($id_ary, 0, $per_page); |
| 1296 | |
| 1297 | return $total_results; |
| 1298 | } |
| 1299 | return false; |
| 1300 | } |
| 1301 | |
| 1302 | /** |
| 1303 | * {@inheritdoc} |
| 1304 | */ |
| 1305 | public function supports_phrase_search(): bool |
| 1306 | { |
| 1307 | return false; |
| 1308 | } |
| 1309 | |
| 1310 | /** |
| 1311 | * {@inheritdoc} |
| 1312 | */ |
| 1313 | public function index(string $mode, int $post_id, string &$message, string &$subject, int $poster_id, int $forum_id) |
| 1314 | { |
| 1315 | if (!$this->config['fulltext_native_load_upd']) |
| 1316 | { |
| 1317 | /** |
| 1318 | * The search indexer is disabled, return |
| 1319 | */ |
| 1320 | return; |
| 1321 | } |
| 1322 | |
| 1323 | // Split old and new post/subject to obtain array of 'words' |
| 1324 | $split_text = $this->split_message($message); |
| 1325 | $split_title = $this->split_message($subject); |
| 1326 | |
| 1327 | $cur_words = array('post' => array(), 'title' => array()); |
| 1328 | |
| 1329 | $words = array(); |
| 1330 | if ($mode == 'edit') |
| 1331 | { |
| 1332 | $words['add']['post'] = array(); |
| 1333 | $words['add']['title'] = array(); |
| 1334 | $words['del']['post'] = array(); |
| 1335 | $words['del']['title'] = array(); |
| 1336 | |
| 1337 | $sql = 'SELECT w.word_id, w.word_text, m.title_match |
| 1338 | FROM ' . $this->search_wordlist_table . ' w, ' . $this->search_wordmatch_table . " m |
| 1339 | WHERE m.post_id = $post_id |
| 1340 | AND w.word_id = m.word_id"; |
| 1341 | $result = $this->db->sql_query($sql); |
| 1342 | |
| 1343 | while ($row = $this->db->sql_fetchrow($result)) |
| 1344 | { |
| 1345 | $which = ($row['title_match']) ? 'title' : 'post'; |
| 1346 | $cur_words[$which][$row['word_text']] = $row['word_id']; |
| 1347 | } |
| 1348 | $this->db->sql_freeresult($result); |
| 1349 | |
| 1350 | $words['add']['post'] = array_diff($split_text, array_keys($cur_words['post'])); |
| 1351 | $words['add']['title'] = array_diff($split_title, array_keys($cur_words['title'])); |
| 1352 | $words['del']['post'] = array_diff(array_keys($cur_words['post']), $split_text); |
| 1353 | $words['del']['title'] = array_diff(array_keys($cur_words['title']), $split_title); |
| 1354 | } |
| 1355 | else |
| 1356 | { |
| 1357 | $words['add']['post'] = $split_text; |
| 1358 | $words['add']['title'] = $split_title; |
| 1359 | $words['del']['post'] = array(); |
| 1360 | $words['del']['title'] = array(); |
| 1361 | } |
| 1362 | |
| 1363 | /** |
| 1364 | * Event to modify method arguments and words before the native search index is updated |
| 1365 | * |
| 1366 | * @event core.search_native_index_before |
| 1367 | * @var string mode Contains the post mode: edit, post, reply, quote |
| 1368 | * @var int post_id The id of the post which is modified/created |
| 1369 | * @var string message New or updated post content |
| 1370 | * @var string subject New or updated post subject |
| 1371 | * @var int poster_id Post author's user id |
| 1372 | * @var int forum_id The id of the forum in which the post is located |
| 1373 | * @var array words Grouped lists of words added to or remove from the index |
| 1374 | * @var array split_text Array of words from the message |
| 1375 | * @var array split_title Array of words from the title |
| 1376 | * @var array cur_words Array of words currently in the index for comparing to new words |
| 1377 | * when mode is edit. Empty for other modes. |
| 1378 | * @since 3.2.3-RC1 |
| 1379 | */ |
| 1380 | $vars = array( |
| 1381 | 'mode', |
| 1382 | 'post_id', |
| 1383 | 'message', |
| 1384 | 'subject', |
| 1385 | 'poster_id', |
| 1386 | 'forum_id', |
| 1387 | 'words', |
| 1388 | 'split_text', |
| 1389 | 'split_title', |
| 1390 | 'cur_words', |
| 1391 | ); |
| 1392 | extract($this->phpbb_dispatcher->trigger_event('core.search_native_index_before', compact($vars))); |
| 1393 | |
| 1394 | unset($split_text); |
| 1395 | unset($split_title); |
| 1396 | |
| 1397 | // Get unique words from the above arrays |
| 1398 | $unique_add_words = array_unique(array_merge($words['add']['post'], $words['add']['title'])); |
| 1399 | |
| 1400 | // We now have unique arrays of all words to be added and removed and |
| 1401 | // individual arrays of added and removed words for text and title. What |
| 1402 | // we need to do now is add the new words (if they don't already exist) |
| 1403 | // and then add (or remove) matches between the words and this post |
| 1404 | if (count($unique_add_words)) |
| 1405 | { |
| 1406 | $sql = 'SELECT word_id, word_text |
| 1407 | FROM ' . $this->search_wordlist_table . ' |
| 1408 | WHERE ' . $this->db->sql_in_set('word_text', $unique_add_words); |
| 1409 | $result = $this->db->sql_query($sql); |
| 1410 | |
| 1411 | $word_ids = array(); |
| 1412 | while ($row = $this->db->sql_fetchrow($result)) |
| 1413 | { |
| 1414 | $word_ids[$row['word_text']] = $row['word_id']; |
| 1415 | } |
| 1416 | $this->db->sql_freeresult($result); |
| 1417 | $new_words = array_diff($unique_add_words, array_keys($word_ids)); |
| 1418 | |
| 1419 | $this->db->sql_transaction('begin'); |
| 1420 | if (count($new_words)) |
| 1421 | { |
| 1422 | $sql_ary = array(); |
| 1423 | |
| 1424 | foreach ($new_words as $word) |
| 1425 | { |
| 1426 | $sql_ary[] = array('word_text' => (string) $word, 'word_count' => 0); |
| 1427 | } |
| 1428 | $this->db->sql_return_on_error(true); |
| 1429 | $this->db->sql_multi_insert($this->search_wordlist_table, $sql_ary); |
| 1430 | $this->db->sql_return_on_error(false); |
| 1431 | } |
| 1432 | unset($new_words, $sql_ary); |
| 1433 | } |
| 1434 | else |
| 1435 | { |
| 1436 | $this->db->sql_transaction('begin'); |
| 1437 | } |
| 1438 | |
| 1439 | // now update the search match table, remove links to removed words and add links to new words |
| 1440 | foreach ($words['del'] as $word_in => $word_ary) |
| 1441 | { |
| 1442 | $title_match = ($word_in == 'title') ? 1 : 0; |
| 1443 | |
| 1444 | if (count($word_ary)) |
| 1445 | { |
| 1446 | $sql_in = array(); |
| 1447 | foreach ($word_ary as $word) |
| 1448 | { |
| 1449 | $sql_in[] = $cur_words[$word_in][$word]; |
| 1450 | } |
| 1451 | |
| 1452 | $sql = 'DELETE FROM ' . $this->search_wordmatch_table . ' |
| 1453 | WHERE ' . $this->db->sql_in_set('word_id', $sql_in) . ' |
| 1454 | AND post_id = ' . intval($post_id) . " |
| 1455 | AND title_match = $title_match"; |
| 1456 | $this->db->sql_query($sql); |
| 1457 | |
| 1458 | $sql = 'UPDATE ' . $this->search_wordlist_table . ' |
| 1459 | SET word_count = word_count - 1 |
| 1460 | WHERE ' . $this->db->sql_in_set('word_id', $sql_in) . ' |
| 1461 | AND word_count > 0'; |
| 1462 | $this->db->sql_query($sql); |
| 1463 | |
| 1464 | unset($sql_in); |
| 1465 | } |
| 1466 | } |
| 1467 | |
| 1468 | $this->db->sql_return_on_error(true); |
| 1469 | foreach ($words['add'] as $word_in => $word_ary) |
| 1470 | { |
| 1471 | $title_match = ($word_in == 'title') ? 1 : 0; |
| 1472 | |
| 1473 | if (count($word_ary)) |
| 1474 | { |
| 1475 | $sql = 'INSERT INTO ' . $this->search_wordmatch_table . ' (post_id, word_id, title_match) |
| 1476 | SELECT ' . (int) $post_id . ', word_id, ' . (int) $title_match . ' |
| 1477 | FROM ' . $this->search_wordlist_table . ' |
| 1478 | WHERE ' . $this->db->sql_in_set('word_text', $word_ary); |
| 1479 | $this->db->sql_query($sql); |
| 1480 | |
| 1481 | $sql = 'UPDATE ' . $this->search_wordlist_table . ' |
| 1482 | SET word_count = word_count + 1 |
| 1483 | WHERE ' . $this->db->sql_in_set('word_text', $word_ary); |
| 1484 | $this->db->sql_query($sql); |
| 1485 | } |
| 1486 | } |
| 1487 | $this->db->sql_return_on_error(false); |
| 1488 | |
| 1489 | $this->db->sql_transaction('commit'); |
| 1490 | |
| 1491 | // destroy cached search results containing any of the words removed or added |
| 1492 | $this->destroy_cache(array_unique(array_merge($words['add']['post'], $words['add']['title'], $words['del']['post'], $words['del']['title'])), array($poster_id)); |
| 1493 | |
| 1494 | unset($unique_add_words); |
| 1495 | unset($words); |
| 1496 | unset($cur_words); |
| 1497 | } |
| 1498 | |
| 1499 | /** |
| 1500 | * {@inheritdoc} |
| 1501 | */ |
| 1502 | public function index_remove(array $post_ids, array $author_ids, array $forum_ids): void |
| 1503 | { |
| 1504 | if (count($post_ids)) |
| 1505 | { |
| 1506 | $sql = 'SELECT w.word_id, w.word_text, m.title_match |
| 1507 | FROM ' . $this->search_wordmatch_table . ' m, ' . $this->search_wordlist_table . ' w |
| 1508 | WHERE ' . $this->db->sql_in_set('m.post_id', $post_ids) . ' |
| 1509 | AND w.word_id = m.word_id'; |
| 1510 | $result = $this->db->sql_query($sql); |
| 1511 | |
| 1512 | $message_word_ids = $title_word_ids = $word_texts = array(); |
| 1513 | while ($row = $this->db->sql_fetchrow($result)) |
| 1514 | { |
| 1515 | if ($row['title_match']) |
| 1516 | { |
| 1517 | $title_word_ids[] = $row['word_id']; |
| 1518 | } |
| 1519 | else |
| 1520 | { |
| 1521 | $message_word_ids[] = $row['word_id']; |
| 1522 | } |
| 1523 | $word_texts[] = $row['word_text']; |
| 1524 | } |
| 1525 | $this->db->sql_freeresult($result); |
| 1526 | |
| 1527 | if (count($title_word_ids)) |
| 1528 | { |
| 1529 | $sql = 'UPDATE ' . $this->search_wordlist_table . ' |
| 1530 | SET word_count = word_count - 1 |
| 1531 | WHERE ' . $this->db->sql_in_set('word_id', $title_word_ids) . ' |
| 1532 | AND word_count > 0'; |
| 1533 | $this->db->sql_query($sql); |
| 1534 | } |
| 1535 | |
| 1536 | if (count($message_word_ids)) |
| 1537 | { |
| 1538 | $sql = 'UPDATE ' . $this->search_wordlist_table . ' |
| 1539 | SET word_count = word_count - 1 |
| 1540 | WHERE ' . $this->db->sql_in_set('word_id', $message_word_ids) . ' |
| 1541 | AND word_count > 0'; |
| 1542 | $this->db->sql_query($sql); |
| 1543 | } |
| 1544 | |
| 1545 | unset($title_word_ids); |
| 1546 | unset($message_word_ids); |
| 1547 | |
| 1548 | $sql = 'DELETE FROM ' . $this->search_wordmatch_table . ' |
| 1549 | WHERE ' . $this->db->sql_in_set('post_id', $post_ids); |
| 1550 | $this->db->sql_query($sql); |
| 1551 | } |
| 1552 | |
| 1553 | $this->destroy_cache(array_unique($word_texts), array_unique($author_ids)); |
| 1554 | } |
| 1555 | |
| 1556 | /** |
| 1557 | * {@inheritdoc} |
| 1558 | */ |
| 1559 | public function tidy(): void |
| 1560 | { |
| 1561 | // Is the fulltext indexer disabled? If yes then we need not |
| 1562 | // carry on ... it's okay ... I know when I'm not wanted boo hoo |
| 1563 | if (!$this->config['fulltext_native_load_upd']) |
| 1564 | { |
| 1565 | $this->config->set('search_last_gc', time(), false); |
| 1566 | return; |
| 1567 | } |
| 1568 | |
| 1569 | $destroy_cache_words = array(); |
| 1570 | |
| 1571 | // Remove common words |
| 1572 | if ($this->config['num_posts'] >= 100 && $this->config['fulltext_native_common_thres']) |
| 1573 | { |
| 1574 | $common_threshold = ((double) $this->config['fulltext_native_common_thres']) / 100.0; |
| 1575 | // First, get the IDs of common words |
| 1576 | $sql = 'SELECT word_id, word_text |
| 1577 | FROM ' . $this->search_wordlist_table . ' |
| 1578 | WHERE word_count > ' . floor($this->config['num_posts'] * $common_threshold) . ' |
| 1579 | OR word_common = 1'; |
| 1580 | $result = $this->db->sql_query($sql); |
| 1581 | |
| 1582 | $sql_in = array(); |
| 1583 | while ($row = $this->db->sql_fetchrow($result)) |
| 1584 | { |
| 1585 | $sql_in[] = $row['word_id']; |
| 1586 | $destroy_cache_words[] = $row['word_text']; |
| 1587 | } |
| 1588 | $this->db->sql_freeresult($result); |
| 1589 | |
| 1590 | if (count($sql_in)) |
| 1591 | { |
| 1592 | // Flag the words |
| 1593 | $sql = 'UPDATE ' . $this->search_wordlist_table . ' |
| 1594 | SET word_common = 1 |
| 1595 | WHERE ' . $this->db->sql_in_set('word_id', $sql_in); |
| 1596 | $this->db->sql_query($sql); |
| 1597 | |
| 1598 | // by setting search_last_gc to the new time here we make sure that if a user reloads because the |
| 1599 | // following query takes too long, he won't run into it again |
| 1600 | $this->config->set('search_last_gc', time(), false); |
| 1601 | |
| 1602 | // Delete the matches |
| 1603 | $sql = 'DELETE FROM ' . $this->search_wordmatch_table . ' |
| 1604 | WHERE ' . $this->db->sql_in_set('word_id', $sql_in); |
| 1605 | $this->db->sql_query($sql); |
| 1606 | } |
| 1607 | unset($sql_in); |
| 1608 | } |
| 1609 | |
| 1610 | if (count($destroy_cache_words)) |
| 1611 | { |
| 1612 | // destroy cached search results containing any of the words that are now common or were removed |
| 1613 | $this->destroy_cache(array_unique($destroy_cache_words)); |
| 1614 | } |
| 1615 | |
| 1616 | $this->config->set('search_last_gc', time(), false); |
| 1617 | } |
| 1618 | |
| 1619 | // create_index is inherited from base.php |
| 1620 | |
| 1621 | /** |
| 1622 | * {@inheritdoc} |
| 1623 | */ |
| 1624 | public function delete_index(int &$post_counter = null): ?array |
| 1625 | { |
| 1626 | $truncate_tables = [ |
| 1627 | $this->search_wordlist_table, |
| 1628 | $this->search_wordmatch_table, |
| 1629 | $this->search_results_table, |
| 1630 | ]; |
| 1631 | |
| 1632 | $stats = $this->stats; |
| 1633 | |
| 1634 | /** |
| 1635 | * Event to modify SQL queries before the native search index is deleted |
| 1636 | * |
| 1637 | * @event core.search_native_delete_index_before |
| 1638 | * |
| 1639 | * @var array stats Array with statistics of the current index (read only) |
| 1640 | * @var array truncate_tables Array with tables that will be truncated |
| 1641 | * |
| 1642 | * @since 3.2.3-RC1 |
| 1643 | * @changed 4.0.0-a1 Removed sql_queries, only add/remove tables to truncate to truncate_tables |
| 1644 | */ |
| 1645 | $vars = array( |
| 1646 | 'stats', |
| 1647 | 'truncate_tables', |
| 1648 | ); |
| 1649 | extract($this->phpbb_dispatcher->trigger_event('core.search_native_delete_index_before', compact($vars))); |
| 1650 | |
| 1651 | foreach ($truncate_tables as $table) |
| 1652 | { |
| 1653 | $this->db_tools->sql_truncate_table($table); |
| 1654 | } |
| 1655 | |
| 1656 | return null; |
| 1657 | } |
| 1658 | |
| 1659 | /** |
| 1660 | * {@inheritdoc} |
| 1661 | */ |
| 1662 | public function index_created(): bool |
| 1663 | { |
| 1664 | if (!count($this->stats)) |
| 1665 | { |
| 1666 | $this->get_stats(); |
| 1667 | } |
| 1668 | |
| 1669 | return $this->stats['total_words'] && $this->stats['total_matches']; |
| 1670 | } |
| 1671 | |
| 1672 | /** |
| 1673 | * {@inheritdoc} |
| 1674 | */ |
| 1675 | public function index_stats() |
| 1676 | { |
| 1677 | if (!count($this->stats)) |
| 1678 | { |
| 1679 | $this->get_stats(); |
| 1680 | } |
| 1681 | |
| 1682 | return array( |
| 1683 | $this->language->lang('TOTAL_WORDS') => $this->stats['total_words'], |
| 1684 | $this->language->lang('TOTAL_MATCHES') => $this->stats['total_matches']); |
| 1685 | } |
| 1686 | |
| 1687 | /** |
| 1688 | * Computes the stats and store them in the $this->stats associative array |
| 1689 | */ |
| 1690 | protected function get_stats() |
| 1691 | { |
| 1692 | $this->stats['total_words'] = $this->db->get_estimated_row_count($this->search_wordlist_table); |
| 1693 | $this->stats['total_matches'] = $this->db->get_estimated_row_count($this->search_wordmatch_table); |
| 1694 | } |
| 1695 | |
| 1696 | /** |
| 1697 | * Split a text into words of a given length |
| 1698 | * |
| 1699 | * The text is converted to UTF-8, cleaned up, and split. Then, words that |
| 1700 | * conform to the defined length range are returned in an array. |
| 1701 | * |
| 1702 | * NOTE: duplicates are NOT removed from the return array |
| 1703 | * |
| 1704 | * @param string $text Text to split, encoded in UTF-8 |
| 1705 | * @return array Array of UTF-8 words |
| 1706 | */ |
| 1707 | protected function split_message($text) |
| 1708 | { |
| 1709 | $match = $words = array(); |
| 1710 | |
| 1711 | /** |
| 1712 | * Taken from the original code |
| 1713 | */ |
| 1714 | // Do not index code |
| 1715 | $match[] = '#\[code(?:=.*?)?(\:?[0-9a-z]{5,})\].*?\[\/code(\:?[0-9a-z]{5,})\]#is'; |
| 1716 | // BBcode |
| 1717 | $match[] = '#\[\/?[a-z0-9\*\+\-]+(?:=.*?)?(?::[a-z])?(\:?[0-9a-z]{5,})\]#'; |
| 1718 | |
| 1719 | $min = $this->word_length['min']; |
| 1720 | |
| 1721 | $isset_min = $min - 1; |
| 1722 | |
| 1723 | /** |
| 1724 | * Clean up the string, remove HTML tags, remove BBCodes |
| 1725 | */ |
| 1726 | $word = strtok($this->cleanup(preg_replace($match, ' ', strip_tags($text)), '-1'), ' '); |
| 1727 | |
| 1728 | while (strlen($word)) |
| 1729 | { |
| 1730 | if (strlen($word) > 255 || strlen($word) <= $isset_min) |
| 1731 | { |
| 1732 | /** |
| 1733 | * Words longer than 255 bytes are ignored. This will have to be |
| 1734 | * changed whenever we change the length of search_wordlist.word_text |
| 1735 | * |
| 1736 | * Words shorter than $isset_min bytes are ignored, too |
| 1737 | */ |
| 1738 | $word = strtok(' '); |
| 1739 | continue; |
| 1740 | } |
| 1741 | |
| 1742 | $len = utf8_strlen($word); |
| 1743 | |
| 1744 | /** |
| 1745 | * Test whether the word is too short to be indexed. |
| 1746 | * |
| 1747 | * Note that this limit does NOT apply to CJK and Hangul |
| 1748 | */ |
| 1749 | if ($len < $min) |
| 1750 | { |
| 1751 | /** |
| 1752 | * Note: this could be optimized. If the codepoint is lower than Hangul's range |
| 1753 | * we know that it will also be lower than CJK ranges |
| 1754 | */ |
| 1755 | if ((strncmp($word, self::UTF8_HANGUL_FIRST, 3) < 0 || strncmp($word, self::UTF8_HANGUL_LAST, 3) > 0) |
| 1756 | && (strncmp($word, self::UTF8_CJK_FIRST, 3) < 0 || strncmp($word, self::UTF8_CJK_LAST, 3) > 0) |
| 1757 | && (strncmp($word, self::UTF8_CJK_B_FIRST, 4) < 0 || strncmp($word, self::UTF8_CJK_B_LAST, 4) > 0)) |
| 1758 | { |
| 1759 | $word = strtok(' '); |
| 1760 | continue; |
| 1761 | } |
| 1762 | } |
| 1763 | |
| 1764 | $words[] = $word; |
| 1765 | $word = strtok(' '); |
| 1766 | } |
| 1767 | |
| 1768 | return $words; |
| 1769 | } |
| 1770 | |
| 1771 | /** |
| 1772 | * Clean up a text to remove non-alphanumeric characters |
| 1773 | * |
| 1774 | * This method receives a UTF-8 string, normalizes and validates it, replaces all |
| 1775 | * non-alphanumeric characters with strings then returns the result. |
| 1776 | * |
| 1777 | * Any number of "allowed chars" can be passed as a UTF-8 string in NFC. |
| 1778 | * |
| 1779 | * @param string $text Text to split, in UTF-8 (not normalized or sanitized) |
| 1780 | * @param string $allowed_chars String of special chars to allow |
| 1781 | * @param string $encoding Text encoding |
| 1782 | * @return string Cleaned up text, only alphanumeric chars are left |
| 1783 | */ |
| 1784 | protected function cleanup($text, $allowed_chars = null, $encoding = 'utf-8') |
| 1785 | { |
| 1786 | static $conv = array(), $conv_loaded = array(); |
| 1787 | $allow = array(); |
| 1788 | |
| 1789 | // Convert the text to UTF-8 |
| 1790 | $encoding = strtolower($encoding); |
| 1791 | if ($encoding != 'utf-8') |
| 1792 | { |
| 1793 | $text = utf8_recode($text, $encoding); |
| 1794 | } |
| 1795 | |
| 1796 | $utf_len_mask = array( |
| 1797 | "\xC0" => 2, |
| 1798 | "\xD0" => 2, |
| 1799 | "\xE0" => 3, |
| 1800 | "\xF0" => 4 |
| 1801 | ); |
| 1802 | |
| 1803 | /** |
| 1804 | * Replace HTML entities and NCRs |
| 1805 | */ |
| 1806 | $text = html_entity_decode(utf8_decode_ncr($text), ENT_QUOTES); |
| 1807 | |
| 1808 | /** |
| 1809 | * Normalize to NFC |
| 1810 | */ |
| 1811 | $text = \Normalizer::normalize($text); |
| 1812 | |
| 1813 | /** |
| 1814 | * The first thing we do is: |
| 1815 | * |
| 1816 | * - convert ASCII-7 letters to lowercase |
| 1817 | * - remove the ASCII-7 non-alpha characters |
| 1818 | * - remove the bytes that should not appear in a valid UTF-8 string: 0xC0, |
| 1819 | * 0xC1 and 0xF5-0xFF |
| 1820 | * |
| 1821 | * @todo in theory, the third one is already taken care of during normalization and those chars should have been replaced by Unicode replacement chars |
| 1822 | */ |
| 1823 | $sb_match = "ISTCPAMELRDOJBNHFGVWUQKYXZ\r\n\t!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0B\x0C\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F\xC0\xC1\xF5\xF6\xF7\xF8\xF9\xFA\xFB\xFC\xFD\xFE\xFF"; |
| 1824 | $sb_replace = 'istcpamelrdojbnhfgvwuqkyxz '; |
| 1825 | |
| 1826 | /** |
| 1827 | * This is the list of legal ASCII chars, it is automatically extended |
| 1828 | * with ASCII chars from $allowed_chars |
| 1829 | */ |
| 1830 | $legal_ascii = ' eaisntroludcpmghbfvq10xy2j9kw354867z'; |
| 1831 | |
| 1832 | /** |
| 1833 | * Prepare an array containing the extra chars to allow |
| 1834 | */ |
| 1835 | if (isset($allowed_chars[0])) |
| 1836 | { |
| 1837 | $pos = 0; |
| 1838 | $len = strlen($allowed_chars); |
| 1839 | do |
| 1840 | { |
| 1841 | $c = $allowed_chars[$pos]; |
| 1842 | |
| 1843 | if ($c < "\x80") |
| 1844 | { |
| 1845 | /** |
| 1846 | * ASCII char |
| 1847 | */ |
| 1848 | $sb_pos = strpos($sb_match, $c); |
| 1849 | if (is_int($sb_pos)) |
| 1850 | { |
| 1851 | /** |
| 1852 | * Remove the char from $sb_match and its corresponding |
| 1853 | * replacement in $sb_replace |
| 1854 | */ |
| 1855 | $sb_match = substr($sb_match, 0, $sb_pos) . substr($sb_match, $sb_pos + 1); |
| 1856 | $sb_replace = substr($sb_replace, 0, $sb_pos) . substr($sb_replace, $sb_pos + 1); |
| 1857 | $legal_ascii .= $c; |
| 1858 | } |
| 1859 | |
| 1860 | ++$pos; |
| 1861 | } |
| 1862 | else |
| 1863 | { |
| 1864 | /** |
| 1865 | * UTF-8 char |
| 1866 | */ |
| 1867 | $utf_len = $utf_len_mask[$c & "\xF0"]; |
| 1868 | $allow[substr($allowed_chars, $pos, $utf_len)] = 1; |
| 1869 | $pos += $utf_len; |
| 1870 | } |
| 1871 | } |
| 1872 | while ($pos < $len); |
| 1873 | } |
| 1874 | |
| 1875 | $text = strtr($text, $sb_match, $sb_replace); |
| 1876 | $ret = ''; |
| 1877 | |
| 1878 | $pos = 0; |
| 1879 | $len = strlen($text); |
| 1880 | |
| 1881 | do |
| 1882 | { |
| 1883 | /** |
| 1884 | * Do all consecutive ASCII chars at once |
| 1885 | */ |
| 1886 | if ($spn = strspn($text, $legal_ascii, $pos)) |
| 1887 | { |
| 1888 | $ret .= substr($text, $pos, $spn); |
| 1889 | $pos += $spn; |
| 1890 | } |
| 1891 | |
| 1892 | if ($pos >= $len) |
| 1893 | { |
| 1894 | return $ret; |
| 1895 | } |
| 1896 | |
| 1897 | /** |
| 1898 | * Capture the UTF char |
| 1899 | */ |
| 1900 | $utf_len = $utf_len_mask[$text[$pos] & "\xF0"]; |
| 1901 | $utf_char = substr($text, $pos, $utf_len); |
| 1902 | $pos += $utf_len; |
| 1903 | |
| 1904 | if (($utf_char >= self::UTF8_HANGUL_FIRST && $utf_char <= self::UTF8_HANGUL_LAST) |
| 1905 | || ($utf_char >= self::UTF8_CJK_FIRST && $utf_char <= self::UTF8_CJK_LAST) |
| 1906 | || ($utf_char >= self::UTF8_CJK_B_FIRST && $utf_char <= self::UTF8_CJK_B_LAST)) |
| 1907 | { |
| 1908 | /** |
| 1909 | * All characters within these ranges are valid |
| 1910 | * |
| 1911 | * We separate them with a space in order to index each character |
| 1912 | * individually |
| 1913 | */ |
| 1914 | $ret .= ' ' . $utf_char . ' '; |
| 1915 | continue; |
| 1916 | } |
| 1917 | |
| 1918 | if (isset($allow[$utf_char])) |
| 1919 | { |
| 1920 | /** |
| 1921 | * The char is explicitly allowed |
| 1922 | */ |
| 1923 | $ret .= $utf_char; |
| 1924 | continue; |
| 1925 | } |
| 1926 | |
| 1927 | if (isset($conv[$utf_char])) |
| 1928 | { |
| 1929 | /** |
| 1930 | * The char is mapped to something, maybe to itself actually |
| 1931 | */ |
| 1932 | $ret .= $conv[$utf_char]; |
| 1933 | continue; |
| 1934 | } |
| 1935 | |
| 1936 | /** |
| 1937 | * The char isn't mapped, but did we load its conversion table? |
| 1938 | * |
| 1939 | * The search indexer table is split into blocks. The block number of |
| 1940 | * each char is equal to its codepoint right-shifted for 11 bits. It |
| 1941 | * means that out of the 11, 16 or 21 meaningful bits of a 2-, 3- or |
| 1942 | * 4- byte sequence we only keep the leftmost 0, 5 or 10 bits. Thus, |
| 1943 | * all UTF chars encoded in 2 bytes are in the same first block. |
| 1944 | */ |
| 1945 | if (isset($utf_char[2])) |
| 1946 | { |
| 1947 | if (isset($utf_char[3])) |
| 1948 | { |
| 1949 | /** |
| 1950 | * 1111 0nnn 10nn nnnn 10nx xxxx 10xx xxxx |
| 1951 | * 0000 0111 0011 1111 0010 0000 |
| 1952 | */ |
| 1953 | $idx = ((ord($utf_char[0]) & 0x07) << 7) | ((ord($utf_char[1]) & 0x3F) << 1) | ((ord($utf_char[2]) & 0x20) >> 5); |
| 1954 | } |
| 1955 | else |
| 1956 | { |
| 1957 | /** |
| 1958 | * 1110 nnnn 10nx xxxx 10xx xxxx |
| 1959 | * 0000 0111 0010 0000 |
| 1960 | */ |
| 1961 | $idx = ((ord($utf_char[0]) & 0x07) << 1) | ((ord($utf_char[1]) & 0x20) >> 5); |
| 1962 | } |
| 1963 | } |
| 1964 | else |
| 1965 | { |
| 1966 | /** |
| 1967 | * 110x xxxx 10xx xxxx |
| 1968 | * 0000 0000 0000 0000 |
| 1969 | */ |
| 1970 | $idx = 0; |
| 1971 | } |
| 1972 | |
| 1973 | /** |
| 1974 | * Check if the required conv table has been loaded already |
| 1975 | */ |
| 1976 | if (!isset($conv_loaded[$idx])) |
| 1977 | { |
| 1978 | $conv_loaded[$idx] = 1; |
| 1979 | $file = $this->phpbb_root_path . 'includes/utf/data/search_indexer_' . $idx . '.' . $this->php_ext; |
| 1980 | |
| 1981 | if (file_exists($file)) |
| 1982 | { |
| 1983 | $conv += include($file); |
| 1984 | } |
| 1985 | } |
| 1986 | |
| 1987 | if (isset($conv[$utf_char])) |
| 1988 | { |
| 1989 | $ret .= $conv[$utf_char]; |
| 1990 | } |
| 1991 | else |
| 1992 | { |
| 1993 | /** |
| 1994 | * We add an entry to the conversion table so that we |
| 1995 | * don't have to convert to codepoint and perform the checks |
| 1996 | * that are above this block |
| 1997 | */ |
| 1998 | $conv[$utf_char] = ' '; |
| 1999 | $ret .= ' '; |
| 2000 | } |
| 2001 | } |
| 2002 | while (1); |
| 2003 | |
| 2004 | return $ret; |
| 2005 | } |
| 2006 | |
| 2007 | /** |
| 2008 | * {@inheritdoc} |
| 2009 | */ |
| 2010 | public function get_acp_options(): array |
| 2011 | { |
| 2012 | /** |
| 2013 | * if we need any options, copied from fulltext_native for now, will have to be adjusted or removed |
| 2014 | */ |
| 2015 | |
| 2016 | $tpl = ' |
| 2017 | <dl> |
| 2018 | <dt><label for="fulltext_native_load_upd">' . $this->language->lang('YES_SEARCH_UPDATE') . $this->language->lang('COLON') . '</label><br /><span>' . $this->language->lang('YES_SEARCH_UPDATE_EXPLAIN') . '</span></dt> |
| 2019 | <dd><label><input type="radio" id="fulltext_native_load_upd" name="config[fulltext_native_load_upd]" value="1"' . (($this->config['fulltext_native_load_upd']) ? ' checked="checked"' : '') . ' class="radio" /> ' . $this->language->lang('YES') . '</label><label><input type="radio" name="config[fulltext_native_load_upd]" value="0"' . ((!$this->config['fulltext_native_load_upd']) ? ' checked="checked"' : '') . ' class="radio" /> ' . $this->language->lang('NO') . '</label></dd> |
| 2020 | </dl> |
| 2021 | <dl> |
| 2022 | <dt><label for="fulltext_native_min_chars">' . $this->language->lang('MIN_SEARCH_CHARS') . $this->language->lang('COLON') . '</label><br /><span>' . $this->language->lang('MIN_SEARCH_CHARS_EXPLAIN') . '</span></dt> |
| 2023 | <dd><input id="fulltext_native_min_chars" type="number" min="0" max="255" name="config[fulltext_native_min_chars]" value="' . (int) $this->config['fulltext_native_min_chars'] . '" /></dd> |
| 2024 | </dl> |
| 2025 | <dl> |
| 2026 | <dt><label for="fulltext_native_max_chars">' . $this->language->lang('MAX_SEARCH_CHARS') . $this->language->lang('COLON') . '</label><br /><span>' . $this->language->lang('MAX_SEARCH_CHARS_EXPLAIN') . '</span></dt> |
| 2027 | <dd><input id="fulltext_native_max_chars" type="number" min="0" max="255" name="config[fulltext_native_max_chars]" value="' . (int) $this->config['fulltext_native_max_chars'] . '" /></dd> |
| 2028 | </dl> |
| 2029 | <dl> |
| 2030 | <dt><label for="fulltext_native_common_thres">' . $this->language->lang('COMMON_WORD_THRESHOLD') . $this->language->lang('COLON') . '</label><br /><span>' . $this->language->lang('COMMON_WORD_THRESHOLD_EXPLAIN') . '</span></dt> |
| 2031 | <dd><input id="fulltext_native_common_thres" type="text" name="config[fulltext_native_common_thres]" value="' . (double) $this->config['fulltext_native_common_thres'] . '" /> %</dd> |
| 2032 | </dl> |
| 2033 | '; |
| 2034 | |
| 2035 | // These are fields required in the config table |
| 2036 | return array( |
| 2037 | 'tpl' => $tpl, |
| 2038 | 'config' => array('fulltext_native_load_upd' => 'bool', 'fulltext_native_min_chars' => 'integer:0:255', 'fulltext_native_max_chars' => 'integer:0:255', 'fulltext_native_common_thres' => 'double:0:100') |
| 2039 | ); |
| 2040 | } |
| 2041 | } |