[25027] | 1 | <?php
|
---|
| 2 | /**
|
---|
| 3 | * Functions to create the fulltext search index
|
---|
| 4 | *
|
---|
| 5 | * @license GPL 2 (http://www.gnu.org/licenses/gpl.html)
|
---|
| 6 | * @author Andreas Gohr <[email protected]>
|
---|
| 7 | * @author Tom N Harris <[email protected]>
|
---|
| 8 | */
|
---|
| 9 |
|
---|
| 10 | if(!defined('DOKU_INC')) die('meh.');
|
---|
| 11 |
|
---|
| 12 | // Version tag used to force rebuild on upgrade
|
---|
| 13 | define('INDEXER_VERSION', 5);
|
---|
| 14 |
|
---|
| 15 | // set the minimum token length to use in the index (note, this doesn't apply to numeric tokens)
|
---|
| 16 | if (!defined('IDX_MINWORDLENGTH')) define('IDX_MINWORDLENGTH',2);
|
---|
| 17 |
|
---|
| 18 | // Asian characters are handled as words. The following regexp defines the
|
---|
| 19 | // Unicode-Ranges for Asian characters
|
---|
| 20 | // Ranges taken from http://en.wikipedia.org/wiki/Unicode_block
|
---|
| 21 | // I'm no language expert. If you think some ranges are wrongly chosen or
|
---|
| 22 | // a range is missing, please contact me
|
---|
| 23 | define('IDX_ASIAN1','[\x{0E00}-\x{0E7F}]'); // Thai
|
---|
| 24 | define('IDX_ASIAN2','['.
|
---|
| 25 | '\x{2E80}-\x{3040}'. // CJK -> Hangul
|
---|
| 26 | '\x{309D}-\x{30A0}'.
|
---|
| 27 | '\x{30FD}-\x{31EF}\x{3200}-\x{D7AF}'.
|
---|
| 28 | '\x{F900}-\x{FAFF}'. // CJK Compatibility Ideographs
|
---|
| 29 | '\x{FE30}-\x{FE4F}'. // CJK Compatibility Forms
|
---|
| 30 | "\xF0\xA0\x80\x80-\xF0\xAA\x9B\x9F". // CJK Extension B
|
---|
| 31 | "\xF0\xAA\x9C\x80-\xF0\xAB\x9C\xBF". // CJK Extension C
|
---|
| 32 | "\xF0\xAB\x9D\x80-\xF0\xAB\xA0\x9F". // CJK Extension D
|
---|
| 33 | "\xF0\xAF\xA0\x80-\xF0\xAF\xAB\xBF". // CJK Compatibility Supplement
|
---|
| 34 | ']');
|
---|
| 35 | define('IDX_ASIAN3','['. // Hiragana/Katakana (can be two characters)
|
---|
| 36 | '\x{3042}\x{3044}\x{3046}\x{3048}'.
|
---|
| 37 | '\x{304A}-\x{3062}\x{3064}-\x{3082}'.
|
---|
| 38 | '\x{3084}\x{3086}\x{3088}-\x{308D}'.
|
---|
| 39 | '\x{308F}-\x{3094}'.
|
---|
| 40 | '\x{30A2}\x{30A4}\x{30A6}\x{30A8}'.
|
---|
| 41 | '\x{30AA}-\x{30C2}\x{30C4}-\x{30E2}'.
|
---|
| 42 | '\x{30E4}\x{30E6}\x{30E8}-\x{30ED}'.
|
---|
| 43 | '\x{30EF}-\x{30F4}\x{30F7}-\x{30FA}'.
|
---|
| 44 | ']['.
|
---|
| 45 | '\x{3041}\x{3043}\x{3045}\x{3047}\x{3049}'.
|
---|
| 46 | '\x{3063}\x{3083}\x{3085}\x{3087}\x{308E}\x{3095}-\x{309C}'.
|
---|
| 47 | '\x{30A1}\x{30A3}\x{30A5}\x{30A7}\x{30A9}'.
|
---|
| 48 | '\x{30C3}\x{30E3}\x{30E5}\x{30E7}\x{30EE}\x{30F5}\x{30F6}\x{30FB}\x{30FC}'.
|
---|
| 49 | '\x{31F0}-\x{31FF}'.
|
---|
| 50 | ']?');
|
---|
| 51 | define('IDX_ASIAN', '(?:'.IDX_ASIAN1.'|'.IDX_ASIAN2.'|'.IDX_ASIAN3.')');
|
---|
| 52 |
|
---|
| 53 | /**
|
---|
| 54 | * Version of the indexer taking into consideration the external tokenizer.
|
---|
| 55 | * The indexer is only compatible with data written by the same version.
|
---|
| 56 | *
|
---|
| 57 | * @triggers INDEXER_VERSION_GET
|
---|
| 58 | * Plugins that modify what gets indexed should hook this event and
|
---|
| 59 | * add their version info to the event data like so:
|
---|
| 60 | * $data[$plugin_name] = $plugin_version;
|
---|
| 61 | *
|
---|
| 62 | * @author Tom N Harris <[email protected]>
|
---|
| 63 | * @author Michael Hamann <[email protected]>
|
---|
| 64 | */
|
---|
| 65 | function idx_get_version(){
|
---|
| 66 | static $indexer_version = null;
|
---|
| 67 | if ($indexer_version == null) {
|
---|
| 68 | global $conf;
|
---|
| 69 | $version = INDEXER_VERSION;
|
---|
| 70 |
|
---|
| 71 | // DokuWiki version is included for the convenience of plugins
|
---|
| 72 | $data = array('dokuwiki'=>$version);
|
---|
| 73 | trigger_event('INDEXER_VERSION_GET', $data, null, false);
|
---|
| 74 | unset($data['dokuwiki']); // this needs to be first
|
---|
| 75 | ksort($data);
|
---|
| 76 | foreach ($data as $plugin=>$vers)
|
---|
| 77 | $version .= '+'.$plugin.'='.$vers;
|
---|
| 78 | $indexer_version = $version;
|
---|
| 79 | }
|
---|
| 80 | return $indexer_version;
|
---|
| 81 | }
|
---|
| 82 |
|
---|
| 83 | /**
|
---|
| 84 | * Measure the length of a string.
|
---|
| 85 | * Differs from strlen in handling of asian characters.
|
---|
| 86 | *
|
---|
| 87 | * @author Tom N Harris <[email protected]>
|
---|
| 88 | */
|
---|
| 89 | function wordlen($w){
|
---|
| 90 | $l = strlen($w);
|
---|
| 91 | // If left alone, all chinese "words" will get put into w3.idx
|
---|
| 92 | // So the "length" of a "word" is faked
|
---|
| 93 | if(preg_match_all('/[\xE2-\xEF]/',$w,$leadbytes)) {
|
---|
| 94 | foreach($leadbytes[0] as $b)
|
---|
| 95 | $l += ord($b) - 0xE1;
|
---|
| 96 | }
|
---|
| 97 | return $l;
|
---|
| 98 | }
|
---|
| 99 |
|
---|
| 100 | /**
|
---|
| 101 | * Class that encapsulates operations on the indexer database.
|
---|
| 102 | *
|
---|
| 103 | * @author Tom N Harris <[email protected]>
|
---|
| 104 | */
|
---|
| 105 | class Doku_Indexer {
|
---|
| 106 |
|
---|
| 107 | /**
|
---|
| 108 | * Adds the contents of a page to the fulltext index
|
---|
| 109 | *
|
---|
| 110 | * The added text replaces previous words for the same page.
|
---|
| 111 | * An empty value erases the page.
|
---|
| 112 | *
|
---|
| 113 | * @param string $page a page name
|
---|
| 114 | * @param string $text the body of the page
|
---|
| 115 | * @return boolean the function completed successfully
|
---|
| 116 | * @author Tom N Harris <[email protected]>
|
---|
| 117 | * @author Andreas Gohr <[email protected]>
|
---|
| 118 | */
|
---|
| 119 | public function addPageWords($page, $text) {
|
---|
| 120 | if (!$this->lock())
|
---|
| 121 | return "locked";
|
---|
| 122 |
|
---|
| 123 | // load known documents
|
---|
| 124 | $pid = $this->addIndexKey('page', '', $page);
|
---|
| 125 | if ($pid === false) {
|
---|
| 126 | $this->unlock();
|
---|
| 127 | return false;
|
---|
| 128 | }
|
---|
| 129 |
|
---|
| 130 | $pagewords = array();
|
---|
| 131 | // get word usage in page
|
---|
| 132 | $words = $this->getPageWords($text);
|
---|
| 133 | if ($words === false) {
|
---|
| 134 | $this->unlock();
|
---|
| 135 | return false;
|
---|
| 136 | }
|
---|
| 137 |
|
---|
| 138 | if (!empty($words)) {
|
---|
| 139 | foreach (array_keys($words) as $wlen) {
|
---|
| 140 | $index = $this->getIndex('i', $wlen);
|
---|
| 141 | foreach ($words[$wlen] as $wid => $freq) {
|
---|
| 142 | $idx = ($wid<count($index)) ? $index[$wid] : '';
|
---|
| 143 | $index[$wid] = $this->updateTuple($idx, $pid, $freq);
|
---|
| 144 | $pagewords[] = "$wlen*$wid";
|
---|
| 145 | }
|
---|
| 146 | if (!$this->saveIndex('i', $wlen, $index)) {
|
---|
| 147 | $this->unlock();
|
---|
| 148 | return false;
|
---|
| 149 | }
|
---|
| 150 | }
|
---|
| 151 | }
|
---|
| 152 |
|
---|
| 153 | // Remove obsolete index entries
|
---|
| 154 | $pageword_idx = $this->getIndexKey('pageword', '', $pid);
|
---|
| 155 | if ($pageword_idx !== '') {
|
---|
| 156 | $oldwords = explode(':',$pageword_idx);
|
---|
| 157 | $delwords = array_diff($oldwords, $pagewords);
|
---|
| 158 | $upwords = array();
|
---|
| 159 | foreach ($delwords as $word) {
|
---|
| 160 | if ($word != '') {
|
---|
| 161 | list($wlen,$wid) = explode('*', $word);
|
---|
| 162 | $wid = (int)$wid;
|
---|
| 163 | $upwords[$wlen][] = $wid;
|
---|
| 164 | }
|
---|
| 165 | }
|
---|
| 166 | foreach ($upwords as $wlen => $widx) {
|
---|
| 167 | $index = $this->getIndex('i', $wlen);
|
---|
| 168 | foreach ($widx as $wid) {
|
---|
| 169 | $index[$wid] = $this->updateTuple($index[$wid], $pid, 0);
|
---|
| 170 | }
|
---|
| 171 | $this->saveIndex('i', $wlen, $index);
|
---|
| 172 | }
|
---|
| 173 | }
|
---|
| 174 | // Save the reverse index
|
---|
| 175 | $pageword_idx = join(':', $pagewords);
|
---|
| 176 | if (!$this->saveIndexKey('pageword', '', $pid, $pageword_idx)) {
|
---|
| 177 | $this->unlock();
|
---|
| 178 | return false;
|
---|
| 179 | }
|
---|
| 180 |
|
---|
| 181 | $this->unlock();
|
---|
| 182 | return true;
|
---|
| 183 | }
|
---|
| 184 |
|
---|
| 185 | /**
|
---|
| 186 | * Split the words in a page and add them to the index.
|
---|
| 187 | *
|
---|
| 188 | * @param string $text content of the page
|
---|
| 189 | * @return array list of word IDs and number of times used
|
---|
| 190 | * @author Andreas Gohr <[email protected]>
|
---|
| 191 | * @author Christopher Smith <[email protected]>
|
---|
| 192 | * @author Tom N Harris <[email protected]>
|
---|
| 193 | */
|
---|
| 194 | protected function getPageWords($text) {
|
---|
| 195 | global $conf;
|
---|
| 196 |
|
---|
| 197 | $tokens = $this->tokenizer($text);
|
---|
| 198 | $tokens = array_count_values($tokens); // count the frequency of each token
|
---|
| 199 |
|
---|
| 200 | $words = array();
|
---|
| 201 | foreach ($tokens as $w=>$c) {
|
---|
| 202 | $l = wordlen($w);
|
---|
| 203 | if (isset($words[$l])){
|
---|
| 204 | $words[$l][$w] = $c + (isset($words[$l][$w]) ? $words[$l][$w] : 0);
|
---|
| 205 | }else{
|
---|
| 206 | $words[$l] = array($w => $c);
|
---|
| 207 | }
|
---|
| 208 | }
|
---|
| 209 |
|
---|
| 210 | // arrive here with $words = array(wordlen => array(word => frequency))
|
---|
| 211 | $word_idx_modified = false;
|
---|
| 212 | $index = array(); //resulting index
|
---|
| 213 | foreach (array_keys($words) as $wlen) {
|
---|
| 214 | $word_idx = $this->getIndex('w', $wlen);
|
---|
| 215 | foreach ($words[$wlen] as $word => $freq) {
|
---|
| 216 | $wid = array_search($word, $word_idx);
|
---|
| 217 | if ($wid === false) {
|
---|
| 218 | $wid = count($word_idx);
|
---|
| 219 | $word_idx[] = $word;
|
---|
| 220 | $word_idx_modified = true;
|
---|
| 221 | }
|
---|
| 222 | if (!isset($index[$wlen]))
|
---|
| 223 | $index[$wlen] = array();
|
---|
| 224 | $index[$wlen][$wid] = $freq;
|
---|
| 225 | }
|
---|
| 226 | // save back the word index
|
---|
| 227 | if ($word_idx_modified && !$this->saveIndex('w', $wlen, $word_idx))
|
---|
| 228 | return false;
|
---|
| 229 | }
|
---|
| 230 |
|
---|
| 231 | return $index;
|
---|
| 232 | }
|
---|
| 233 |
|
---|
| 234 | /**
|
---|
| 235 | * Add/update keys to/of the metadata index.
|
---|
| 236 | *
|
---|
| 237 | * Adding new keys does not remove other keys for the page.
|
---|
| 238 | * An empty value will erase the key.
|
---|
| 239 | * The $key parameter can be an array to add multiple keys. $value will
|
---|
| 240 | * not be used if $key is an array.
|
---|
| 241 | *
|
---|
| 242 | * @param string $page a page name
|
---|
| 243 | * @param mixed $key a key string or array of key=>value pairs
|
---|
| 244 | * @param mixed $value the value or list of values
|
---|
| 245 | * @return boolean the function completed successfully
|
---|
| 246 | * @author Tom N Harris <[email protected]>
|
---|
| 247 | * @author Michael Hamann <[email protected]>
|
---|
| 248 | */
|
---|
| 249 | public function addMetaKeys($page, $key, $value=null) {
|
---|
| 250 | if (!is_array($key)) {
|
---|
| 251 | $key = array($key => $value);
|
---|
| 252 | } elseif (!is_null($value)) {
|
---|
| 253 | // $key is array, but $value is not null
|
---|
| 254 | trigger_error("array passed to addMetaKeys but value is not null", E_USER_WARNING);
|
---|
| 255 | }
|
---|
| 256 |
|
---|
| 257 | if (!$this->lock())
|
---|
| 258 | return "locked";
|
---|
| 259 |
|
---|
| 260 | // load known documents
|
---|
| 261 | $pid = $this->addIndexKey('page', '', $page);
|
---|
| 262 | if ($pid === false) {
|
---|
| 263 | $this->unlock();
|
---|
| 264 | return false;
|
---|
| 265 | }
|
---|
| 266 |
|
---|
| 267 | // Special handling for titles so the index file is simpler
|
---|
| 268 | if (array_key_exists('title', $key)) {
|
---|
| 269 | $value = $key['title'];
|
---|
| 270 | if (is_array($value))
|
---|
| 271 | $value = $value[0];
|
---|
| 272 | $this->saveIndexKey('title', '', $pid, $value);
|
---|
| 273 | unset($key['title']);
|
---|
| 274 | }
|
---|
| 275 |
|
---|
| 276 | foreach ($key as $name => $values) {
|
---|
| 277 | $metaname = idx_cleanName($name);
|
---|
| 278 | $this->addIndexKey('metadata', '', $metaname);
|
---|
| 279 | $metaidx = $this->getIndex($metaname.'_i', '');
|
---|
| 280 | $metawords = $this->getIndex($metaname.'_w', '');
|
---|
| 281 | $addwords = false;
|
---|
| 282 |
|
---|
| 283 | if (!is_array($values)) $values = array($values);
|
---|
| 284 |
|
---|
| 285 | $val_idx = $this->getIndexKey($metaname.'_p', '', $pid);
|
---|
| 286 | if ($val_idx != '') {
|
---|
| 287 | $val_idx = explode(':', $val_idx);
|
---|
| 288 | // -1 means remove, 0 keep, 1 add
|
---|
| 289 | $val_idx = array_combine($val_idx, array_fill(0, count($val_idx), -1));
|
---|
| 290 | } else {
|
---|
| 291 | $val_idx = array();
|
---|
| 292 | }
|
---|
| 293 |
|
---|
| 294 |
|
---|
| 295 | foreach ($values as $val) {
|
---|
| 296 | $val = (string)$val;
|
---|
| 297 | if ($val !== "") {
|
---|
| 298 | $id = array_search($val, $metawords);
|
---|
| 299 | if ($id === false) {
|
---|
| 300 | $id = count($metawords);
|
---|
| 301 | $metawords[$id] = $val;
|
---|
| 302 | $addwords = true;
|
---|
| 303 | }
|
---|
| 304 | // test if value is already in the index
|
---|
| 305 | if (isset($val_idx[$id]) && $val_idx[$id] <= 0)
|
---|
| 306 | $val_idx[$id] = 0;
|
---|
| 307 | else // else add it
|
---|
| 308 | $val_idx[$id] = 1;
|
---|
| 309 | }
|
---|
| 310 | }
|
---|
| 311 |
|
---|
| 312 | if ($addwords)
|
---|
| 313 | $this->saveIndex($metaname.'_w', '', $metawords);
|
---|
| 314 | $vals_changed = false;
|
---|
| 315 | foreach ($val_idx as $id => $action) {
|
---|
| 316 | if ($action == -1) {
|
---|
| 317 | $metaidx[$id] = $this->updateTuple($metaidx[$id], $pid, 0);
|
---|
| 318 | $vals_changed = true;
|
---|
| 319 | unset($val_idx[$id]);
|
---|
| 320 | } elseif ($action == 1) {
|
---|
| 321 | $metaidx[$id] = $this->updateTuple($metaidx[$id], $pid, 1);
|
---|
| 322 | $vals_changed = true;
|
---|
| 323 | }
|
---|
| 324 | }
|
---|
| 325 |
|
---|
| 326 | if ($vals_changed) {
|
---|
| 327 | $this->saveIndex($metaname.'_i', '', $metaidx);
|
---|
| 328 | $val_idx = implode(':', array_keys($val_idx));
|
---|
| 329 | $this->saveIndexKey($metaname.'_p', '', $pid, $val_idx);
|
---|
| 330 | }
|
---|
| 331 |
|
---|
| 332 | unset($metaidx);
|
---|
| 333 | unset($metawords);
|
---|
| 334 | }
|
---|
| 335 |
|
---|
| 336 | $this->unlock();
|
---|
| 337 | return true;
|
---|
| 338 | }
|
---|
| 339 |
|
---|
| 340 | /**
|
---|
| 341 | * Remove a page from the index
|
---|
| 342 | *
|
---|
| 343 | * Erases entries in all known indexes.
|
---|
| 344 | *
|
---|
| 345 | * @param string $page a page name
|
---|
| 346 | * @return boolean the function completed successfully
|
---|
| 347 | * @author Tom N Harris <[email protected]>
|
---|
| 348 | */
|
---|
| 349 | public function deletePage($page) {
|
---|
| 350 | if (!$this->lock())
|
---|
| 351 | return "locked";
|
---|
| 352 |
|
---|
| 353 | // load known documents
|
---|
| 354 | $pid = $this->getIndexKey('page', '', $page);
|
---|
| 355 | if ($pid === false) {
|
---|
| 356 | $this->unlock();
|
---|
| 357 | return false;
|
---|
| 358 | }
|
---|
| 359 |
|
---|
| 360 | // Remove obsolete index entries
|
---|
| 361 | $pageword_idx = $this->getIndexKey('pageword', '', $pid);
|
---|
| 362 | if ($pageword_idx !== '') {
|
---|
| 363 | $delwords = explode(':',$pageword_idx);
|
---|
| 364 | $upwords = array();
|
---|
| 365 | foreach ($delwords as $word) {
|
---|
| 366 | if ($word != '') {
|
---|
| 367 | list($wlen,$wid) = explode('*', $word);
|
---|
| 368 | $wid = (int)$wid;
|
---|
| 369 | $upwords[$wlen][] = $wid;
|
---|
| 370 | }
|
---|
| 371 | }
|
---|
| 372 | foreach ($upwords as $wlen => $widx) {
|
---|
| 373 | $index = $this->getIndex('i', $wlen);
|
---|
| 374 | foreach ($widx as $wid) {
|
---|
| 375 | $index[$wid] = $this->updateTuple($index[$wid], $pid, 0);
|
---|
| 376 | }
|
---|
| 377 | $this->saveIndex('i', $wlen, $index);
|
---|
| 378 | }
|
---|
| 379 | }
|
---|
| 380 | // Save the reverse index
|
---|
| 381 | if (!$this->saveIndexKey('pageword', '', $pid, "")) {
|
---|
| 382 | $this->unlock();
|
---|
| 383 | return false;
|
---|
| 384 | }
|
---|
| 385 |
|
---|
| 386 | $this->saveIndexKey('title', '', $pid, "");
|
---|
| 387 | $keyidx = $this->getIndex('metadata', '');
|
---|
| 388 | foreach ($keyidx as $metaname) {
|
---|
| 389 | $val_idx = explode(':', $this->getIndexKey($metaname.'_p', '', $pid));
|
---|
| 390 | $meta_idx = $this->getIndex($metaname.'_i', '');
|
---|
| 391 | foreach ($val_idx as $id) {
|
---|
| 392 | $meta_idx[$id] = $this->updateTuple($meta_idx[$id], $pid, 0);
|
---|
| 393 | }
|
---|
| 394 | $this->saveIndex($metaname.'_i', '', $meta_idx);
|
---|
| 395 | $this->saveIndexKey($metaname.'_p', '', $pid, '');
|
---|
| 396 | }
|
---|
| 397 |
|
---|
| 398 | $this->unlock();
|
---|
| 399 | return true;
|
---|
| 400 | }
|
---|
| 401 |
|
---|
| 402 | /**
|
---|
| 403 | * Split the text into words for fulltext search
|
---|
| 404 | *
|
---|
| 405 | * TODO: does this also need &$stopwords ?
|
---|
| 406 | *
|
---|
| 407 | * @triggers INDEXER_TEXT_PREPARE
|
---|
| 408 | * This event allows plugins to modify the text before it gets tokenized.
|
---|
| 409 | * Plugins intercepting this event should also intercept INDEX_VERSION_GET
|
---|
| 410 | *
|
---|
| 411 | * @param string $text plain text
|
---|
| 412 | * @param boolean $wc are wildcards allowed?
|
---|
| 413 | * @return array list of words in the text
|
---|
| 414 | * @author Tom N Harris <[email protected]>
|
---|
| 415 | * @author Andreas Gohr <[email protected]>
|
---|
| 416 | */
|
---|
| 417 | public function tokenizer($text, $wc=false) {
|
---|
| 418 | global $conf;
|
---|
| 419 | $words = array();
|
---|
| 420 | $wc = ($wc) ? '' : '\*';
|
---|
| 421 | $stopwords =& idx_get_stopwords();
|
---|
| 422 |
|
---|
| 423 | // prepare the text to be tokenized
|
---|
| 424 | $evt = new Doku_Event('INDEXER_TEXT_PREPARE', $text);
|
---|
| 425 | if ($evt->advise_before(true)) {
|
---|
| 426 | if (preg_match('/[^0-9A-Za-z ]/u', $text)) {
|
---|
| 427 | // handle asian chars as single words (may fail on older PHP version)
|
---|
| 428 | $asia = @preg_replace('/('.IDX_ASIAN.')/u', ' \1 ', $text);
|
---|
| 429 | if (!is_null($asia)) $text = $asia; // recover from regexp falure
|
---|
| 430 | }
|
---|
| 431 | }
|
---|
| 432 | $evt->advise_after();
|
---|
| 433 | unset($evt);
|
---|
| 434 |
|
---|
| 435 | $text = strtr($text,
|
---|
| 436 | array(
|
---|
| 437 | "\r" => ' ',
|
---|
| 438 | "\n" => ' ',
|
---|
| 439 | "\t" => ' ',
|
---|
| 440 | "\xC2\xAD" => '', //soft-hyphen
|
---|
| 441 | )
|
---|
| 442 | );
|
---|
| 443 | if (preg_match('/[^0-9A-Za-z ]/u', $text))
|
---|
| 444 | $text = utf8_stripspecials($text, ' ', '\._\-:'.$wc);
|
---|
| 445 |
|
---|
| 446 | $wordlist = explode(' ', $text);
|
---|
| 447 | foreach ($wordlist as $i => $word) {
|
---|
| 448 | $wordlist[$i] = (preg_match('/[^0-9A-Za-z]/u', $word)) ?
|
---|
| 449 | utf8_strtolower($word) : strtolower($word);
|
---|
| 450 | }
|
---|
| 451 |
|
---|
| 452 | foreach ($wordlist as $i => $word) {
|
---|
| 453 | if ((!is_numeric($word) && strlen($word) < IDX_MINWORDLENGTH)
|
---|
| 454 | || array_search($word, $stopwords) !== false)
|
---|
| 455 | unset($wordlist[$i]);
|
---|
| 456 | }
|
---|
| 457 | return array_values($wordlist);
|
---|
| 458 | }
|
---|
| 459 |
|
---|
| 460 | /**
|
---|
| 461 | * Find pages in the fulltext index containing the words,
|
---|
| 462 | *
|
---|
| 463 | * The search words must be pre-tokenized, meaning only letters and
|
---|
| 464 | * numbers with an optional wildcard
|
---|
| 465 | *
|
---|
| 466 | * The returned array will have the original tokens as key. The values
|
---|
| 467 | * in the returned list is an array with the page names as keys and the
|
---|
| 468 | * number of times that token appears on the page as value.
|
---|
| 469 | *
|
---|
| 470 | * @param arrayref $tokens list of words to search for
|
---|
| 471 | * @return array list of page names with usage counts
|
---|
| 472 | * @author Tom N Harris <[email protected]>
|
---|
| 473 | * @author Andreas Gohr <[email protected]>
|
---|
| 474 | */
|
---|
| 475 | public function lookup(&$tokens) {
|
---|
| 476 | $result = array();
|
---|
| 477 | $wids = $this->getIndexWords($tokens, $result);
|
---|
| 478 | if (empty($wids)) return array();
|
---|
| 479 | // load known words and documents
|
---|
| 480 | $page_idx = $this->getIndex('page', '');
|
---|
| 481 | $docs = array();
|
---|
| 482 | foreach (array_keys($wids) as $wlen) {
|
---|
| 483 | $wids[$wlen] = array_unique($wids[$wlen]);
|
---|
| 484 | $index = $this->getIndex('i', $wlen);
|
---|
| 485 | foreach($wids[$wlen] as $ixid) {
|
---|
| 486 | if ($ixid < count($index))
|
---|
| 487 | $docs["$wlen*$ixid"] = $this->parseTuples($page_idx, $index[$ixid]);
|
---|
| 488 | }
|
---|
| 489 | }
|
---|
| 490 | // merge found pages into final result array
|
---|
| 491 | $final = array();
|
---|
| 492 | foreach ($result as $word => $res) {
|
---|
| 493 | $final[$word] = array();
|
---|
| 494 | foreach ($res as $wid) {
|
---|
| 495 | // handle the case when ($ixid < count($index)) has been false
|
---|
| 496 | // and thus $docs[$wid] hasn't been set.
|
---|
| 497 | if (!isset($docs[$wid])) continue;
|
---|
| 498 | $hits = &$docs[$wid];
|
---|
| 499 | foreach ($hits as $hitkey => $hitcnt) {
|
---|
| 500 | // make sure the document still exists
|
---|
| 501 | if (!page_exists($hitkey, '', false)) continue;
|
---|
| 502 | if (!isset($final[$word][$hitkey]))
|
---|
| 503 | $final[$word][$hitkey] = $hitcnt;
|
---|
| 504 | else
|
---|
| 505 | $final[$word][$hitkey] += $hitcnt;
|
---|
| 506 | }
|
---|
| 507 | }
|
---|
| 508 | }
|
---|
| 509 | return $final;
|
---|
| 510 | }
|
---|
| 511 |
|
---|
| 512 | /**
|
---|
| 513 | * Find pages containing a metadata key.
|
---|
| 514 | *
|
---|
| 515 | * The metadata values are compared as case-sensitive strings. Pass a
|
---|
| 516 | * callback function that returns true or false to use a different
|
---|
| 517 | * comparison function. The function will be called with the $value being
|
---|
| 518 | * searched for as the first argument, and the word in the index as the
|
---|
| 519 | * second argument. The function preg_match can be used directly if the
|
---|
| 520 | * values are regexes.
|
---|
| 521 | *
|
---|
| 522 | * @param string $key name of the metadata key to look for
|
---|
| 523 | * @param string $value search term to look for, must be a string or array of strings
|
---|
| 524 | * @param callback $func comparison function
|
---|
| 525 | * @return array lists with page names, keys are query values if $value is array
|
---|
| 526 | * @author Tom N Harris <[email protected]>
|
---|
| 527 | * @author Michael Hamann <[email protected]>
|
---|
| 528 | */
|
---|
| 529 | public function lookupKey($key, &$value, $func=null) {
|
---|
| 530 | if (!is_array($value))
|
---|
| 531 | $value_array = array($value);
|
---|
| 532 | else
|
---|
| 533 | $value_array =& $value;
|
---|
| 534 |
|
---|
| 535 | // the matching ids for the provided value(s)
|
---|
| 536 | $value_ids = array();
|
---|
| 537 |
|
---|
| 538 | $metaname = idx_cleanName($key);
|
---|
| 539 |
|
---|
| 540 | // get all words in order to search the matching ids
|
---|
| 541 | if ($key == 'title') {
|
---|
| 542 | $words = $this->getIndex('title', '');
|
---|
| 543 | } else {
|
---|
| 544 | $words = $this->getIndex($metaname.'_w', '');
|
---|
| 545 | }
|
---|
| 546 |
|
---|
| 547 | if (!is_null($func)) {
|
---|
| 548 | foreach ($value_array as $val) {
|
---|
| 549 | foreach ($words as $i => $word) {
|
---|
| 550 | if (call_user_func_array($func, array($val, $word)))
|
---|
| 551 | $value_ids[$i][] = $val;
|
---|
| 552 | }
|
---|
| 553 | }
|
---|
| 554 | } else {
|
---|
| 555 | foreach ($value_array as $val) {
|
---|
| 556 | $xval = $val;
|
---|
| 557 | $caret = '^';
|
---|
| 558 | $dollar = '$';
|
---|
| 559 | // check for wildcards
|
---|
| 560 | if (substr($xval, 0, 1) == '*') {
|
---|
| 561 | $xval = substr($xval, 1);
|
---|
| 562 | $caret = '';
|
---|
| 563 | }
|
---|
| 564 | if (substr($xval, -1, 1) == '*') {
|
---|
| 565 | $xval = substr($xval, 0, -1);
|
---|
| 566 | $dollar = '';
|
---|
| 567 | }
|
---|
| 568 | if (!$caret || !$dollar) {
|
---|
| 569 | $re = $caret.preg_quote($xval, '/').$dollar;
|
---|
| 570 | foreach(array_keys(preg_grep('/'.$re.'/', $words)) as $i)
|
---|
| 571 | $value_ids[$i][] = $val;
|
---|
| 572 | } else {
|
---|
| 573 | if (($i = array_search($val, $words)) !== false)
|
---|
| 574 | $value_ids[$i][] = $val;
|
---|
| 575 | }
|
---|
| 576 | }
|
---|
| 577 | }
|
---|
| 578 |
|
---|
| 579 | unset($words); // free the used memory
|
---|
| 580 |
|
---|
| 581 | // initialize the result so it won't be null
|
---|
| 582 | $result = array();
|
---|
| 583 | foreach ($value_array as $val) {
|
---|
| 584 | $result[$val] = array();
|
---|
| 585 | }
|
---|
| 586 |
|
---|
| 587 | $page_idx = $this->getIndex('page', '');
|
---|
| 588 |
|
---|
| 589 | // Special handling for titles
|
---|
| 590 | if ($key == 'title') {
|
---|
| 591 | foreach ($value_ids as $pid => $val_list) {
|
---|
| 592 | $page = $page_idx[$pid];
|
---|
| 593 | foreach ($val_list as $val) {
|
---|
| 594 | $result[$val][] = $page;
|
---|
| 595 | }
|
---|
| 596 | }
|
---|
| 597 | } else {
|
---|
| 598 | // load all lines and pages so the used lines can be taken and matched with the pages
|
---|
| 599 | $lines = $this->getIndex($metaname.'_i', '');
|
---|
| 600 |
|
---|
| 601 | foreach ($value_ids as $value_id => $val_list) {
|
---|
| 602 | // parse the tuples of the form page_id*1:page2_id*1 and so on, return value
|
---|
| 603 | // is an array with page_id => 1, page2_id => 1 etc. so take the keys only
|
---|
| 604 | $pages = array_keys($this->parseTuples($page_idx, $lines[$value_id]));
|
---|
| 605 | foreach ($val_list as $val) {
|
---|
| 606 | $result[$val] = array_merge($result[$val], $pages);
|
---|
| 607 | }
|
---|
| 608 | }
|
---|
| 609 | }
|
---|
| 610 | if (!is_array($value)) $result = $result[$value];
|
---|
| 611 | return $result;
|
---|
| 612 | }
|
---|
| 613 |
|
---|
| 614 | /**
|
---|
| 615 | * Find the index ID of each search term.
|
---|
| 616 | *
|
---|
| 617 | * The query terms should only contain valid characters, with a '*' at
|
---|
| 618 | * either the beginning or end of the word (or both).
|
---|
| 619 | * The $result parameter can be used to merge the index locations with
|
---|
| 620 | * the appropriate query term.
|
---|
| 621 | *
|
---|
| 622 | * @param arrayref $words The query terms.
|
---|
| 623 | * @param arrayref $result Set to word => array("length*id" ...)
|
---|
| 624 | * @return array Set to length => array(id ...)
|
---|
| 625 | * @author Tom N Harris <[email protected]>
|
---|
| 626 | */
|
---|
| 627 | protected function getIndexWords(&$words, &$result) {
|
---|
| 628 | $tokens = array();
|
---|
| 629 | $tokenlength = array();
|
---|
| 630 | $tokenwild = array();
|
---|
| 631 | foreach ($words as $word) {
|
---|
| 632 | $result[$word] = array();
|
---|
| 633 | $caret = '^';
|
---|
| 634 | $dollar = '$';
|
---|
| 635 | $xword = $word;
|
---|
| 636 | $wlen = wordlen($word);
|
---|
| 637 |
|
---|
| 638 | // check for wildcards
|
---|
| 639 | if (substr($xword, 0, 1) == '*') {
|
---|
| 640 | $xword = substr($xword, 1);
|
---|
| 641 | $caret = '';
|
---|
| 642 | $wlen -= 1;
|
---|
| 643 | }
|
---|
| 644 | if (substr($xword, -1, 1) == '*') {
|
---|
| 645 | $xword = substr($xword, 0, -1);
|
---|
| 646 | $dollar = '';
|
---|
| 647 | $wlen -= 1;
|
---|
| 648 | }
|
---|
| 649 | if ($wlen < IDX_MINWORDLENGTH && $caret && $dollar && !is_numeric($xword))
|
---|
| 650 | continue;
|
---|
| 651 | if (!isset($tokens[$xword]))
|
---|
| 652 | $tokenlength[$wlen][] = $xword;
|
---|
| 653 | if (!$caret || !$dollar) {
|
---|
| 654 | $re = $caret.preg_quote($xword, '/').$dollar;
|
---|
| 655 | $tokens[$xword][] = array($word, '/'.$re.'/');
|
---|
| 656 | if (!isset($tokenwild[$xword]))
|
---|
| 657 | $tokenwild[$xword] = $wlen;
|
---|
| 658 | } else {
|
---|
| 659 | $tokens[$xword][] = array($word, null);
|
---|
| 660 | }
|
---|
| 661 | }
|
---|
| 662 | asort($tokenwild);
|
---|
| 663 | // $tokens = array( base word => array( [ query term , regexp ] ... ) ... )
|
---|
| 664 | // $tokenlength = array( base word length => base word ... )
|
---|
| 665 | // $tokenwild = array( base word => base word length ... )
|
---|
| 666 | $length_filter = empty($tokenwild) ? $tokenlength : min(array_keys($tokenlength));
|
---|
| 667 | $indexes_known = $this->indexLengths($length_filter);
|
---|
| 668 | if (!empty($tokenwild)) sort($indexes_known);
|
---|
| 669 | // get word IDs
|
---|
| 670 | $wids = array();
|
---|
| 671 | foreach ($indexes_known as $ixlen) {
|
---|
| 672 | $word_idx = $this->getIndex('w', $ixlen);
|
---|
| 673 | // handle exact search
|
---|
| 674 | if (isset($tokenlength[$ixlen])) {
|
---|
| 675 | foreach ($tokenlength[$ixlen] as $xword) {
|
---|
| 676 | $wid = array_search($xword, $word_idx);
|
---|
| 677 | if ($wid !== false) {
|
---|
| 678 | $wids[$ixlen][] = $wid;
|
---|
| 679 | foreach ($tokens[$xword] as $w)
|
---|
| 680 | $result[$w[0]][] = "$ixlen*$wid";
|
---|
| 681 | }
|
---|
| 682 | }
|
---|
| 683 | }
|
---|
| 684 | // handle wildcard search
|
---|
| 685 | foreach ($tokenwild as $xword => $wlen) {
|
---|
| 686 | if ($wlen >= $ixlen) break;
|
---|
| 687 | foreach ($tokens[$xword] as $w) {
|
---|
| 688 | if (is_null($w[1])) continue;
|
---|
| 689 | foreach(array_keys(preg_grep($w[1], $word_idx)) as $wid) {
|
---|
| 690 | $wids[$ixlen][] = $wid;
|
---|
| 691 | $result[$w[0]][] = "$ixlen*$wid";
|
---|
| 692 | }
|
---|
| 693 | }
|
---|
| 694 | }
|
---|
| 695 | }
|
---|
| 696 | return $wids;
|
---|
| 697 | }
|
---|
| 698 |
|
---|
| 699 | /**
|
---|
| 700 | * Return a list of all pages
|
---|
| 701 | * Warning: pages may not exist!
|
---|
| 702 | *
|
---|
| 703 | * @param string $key list only pages containing the metadata key (optional)
|
---|
| 704 | * @return array list of page names
|
---|
| 705 | * @author Tom N Harris <[email protected]>
|
---|
| 706 | */
|
---|
| 707 | public function getPages($key=null) {
|
---|
| 708 | $page_idx = $this->getIndex('page', '');
|
---|
| 709 | if (is_null($key)) return $page_idx;
|
---|
| 710 |
|
---|
| 711 | $metaname = idx_cleanName($key);
|
---|
| 712 |
|
---|
| 713 | // Special handling for titles
|
---|
| 714 | if ($key == 'title') {
|
---|
| 715 | $title_idx = $this->getIndex('title', '');
|
---|
| 716 | array_splice($page_idx, count($title_idx));
|
---|
| 717 | foreach ($title_idx as $i => $title)
|
---|
| 718 | if ($title === "") unset($page_idx[$i]);
|
---|
| 719 | return array_values($page_idx);
|
---|
| 720 | }
|
---|
| 721 |
|
---|
| 722 | $pages = array();
|
---|
| 723 | $lines = $this->getIndex($metaname.'_i', '');
|
---|
| 724 | foreach ($lines as $line) {
|
---|
| 725 | $pages = array_merge($pages, $this->parseTuples($page_idx, $line));
|
---|
| 726 | }
|
---|
| 727 | return array_keys($pages);
|
---|
| 728 | }
|
---|
| 729 |
|
---|
| 730 | /**
|
---|
| 731 | * Return a list of words sorted by number of times used
|
---|
| 732 | *
|
---|
| 733 | * @param int $min bottom frequency threshold
|
---|
| 734 | * @param int $max upper frequency limit. No limit if $max<$min
|
---|
| 735 | * @param int $length minimum length of words to count
|
---|
| 736 | * @param string $key metadata key to list. Uses the fulltext index if not given
|
---|
| 737 | * @return array list of words as the keys and frequency as values
|
---|
| 738 | * @author Tom N Harris <[email protected]>
|
---|
| 739 | */
|
---|
| 740 | public function histogram($min=1, $max=0, $minlen=3, $key=null) {
|
---|
| 741 | if ($min < 1)
|
---|
| 742 | $min = 1;
|
---|
| 743 | if ($max < $min)
|
---|
| 744 | $max = 0;
|
---|
| 745 |
|
---|
| 746 | $result = array();
|
---|
| 747 |
|
---|
| 748 | if ($key == 'title') {
|
---|
| 749 | $index = $this->getIndex('title', '');
|
---|
| 750 | $index = array_count_values($index);
|
---|
| 751 | foreach ($index as $val => $cnt) {
|
---|
| 752 | if ($cnt >= $min && (!$max || $cnt <= $max) && strlen($val) >= $minlen)
|
---|
| 753 | $result[$val] = $cnt;
|
---|
| 754 | }
|
---|
| 755 | }
|
---|
| 756 | elseif (!is_null($key)) {
|
---|
| 757 | $metaname = idx_cleanName($key);
|
---|
| 758 | $index = $this->getIndex($metaname.'_i', '');
|
---|
| 759 | $val_idx = array();
|
---|
| 760 | foreach ($index as $wid => $line) {
|
---|
| 761 | $freq = $this->countTuples($line);
|
---|
| 762 | if ($freq >= $min && (!$max || $freq <= $max) && strlen($val) >= $minlen)
|
---|
| 763 | $val_idx[$wid] = $freq;
|
---|
| 764 | }
|
---|
| 765 | if (!empty($val_idx)) {
|
---|
| 766 | $words = $this->getIndex($metaname.'_w', '');
|
---|
| 767 | foreach ($val_idx as $wid => $freq)
|
---|
| 768 | $result[$words[$wid]] = $freq;
|
---|
| 769 | }
|
---|
| 770 | }
|
---|
| 771 | else {
|
---|
| 772 | $lengths = idx_listIndexLengths();
|
---|
| 773 | foreach ($lengths as $length) {
|
---|
| 774 | if ($length < $minlen) continue;
|
---|
| 775 | $index = $this->getIndex('i', $length);
|
---|
| 776 | $words = null;
|
---|
| 777 | foreach ($index as $wid => $line) {
|
---|
| 778 | $freq = $this->countTuples($line);
|
---|
| 779 | if ($freq >= $min && (!$max || $freq <= $max)) {
|
---|
| 780 | if ($words === null)
|
---|
| 781 | $words = $this->getIndex('w', $length);
|
---|
| 782 | $result[$words[$wid]] = $freq;
|
---|
| 783 | }
|
---|
| 784 | }
|
---|
| 785 | }
|
---|
| 786 | }
|
---|
| 787 |
|
---|
| 788 | arsort($result);
|
---|
| 789 | return $result;
|
---|
| 790 | }
|
---|
| 791 |
|
---|
| 792 | /**
|
---|
| 793 | * Lock the indexer.
|
---|
| 794 | *
|
---|
| 795 | * @author Tom N Harris <[email protected]>
|
---|
| 796 | */
|
---|
| 797 | protected function lock() {
|
---|
| 798 | global $conf;
|
---|
| 799 | $status = true;
|
---|
| 800 | $run = 0;
|
---|
| 801 | $lock = $conf['lockdir'].'/_indexer.lock';
|
---|
| 802 | while (!@mkdir($lock, $conf['dmode'])) {
|
---|
| 803 | usleep(50);
|
---|
| 804 | if(is_dir($lock) && time()-@filemtime($lock) > 60*5){
|
---|
| 805 | // looks like a stale lock - remove it
|
---|
| 806 | if (!@rmdir($lock)) {
|
---|
| 807 | $status = "removing the stale lock failed";
|
---|
| 808 | return false;
|
---|
| 809 | } else {
|
---|
| 810 | $status = "stale lock removed";
|
---|
| 811 | }
|
---|
| 812 | }elseif($run++ == 1000){
|
---|
| 813 | // we waited 5 seconds for that lock
|
---|
| 814 | return false;
|
---|
| 815 | }
|
---|
| 816 | }
|
---|
| 817 | if ($conf['dperm'])
|
---|
| 818 | chmod($lock, $conf['dperm']);
|
---|
| 819 | return $status;
|
---|
| 820 | }
|
---|
| 821 |
|
---|
| 822 | /**
|
---|
| 823 | * Release the indexer lock.
|
---|
| 824 | *
|
---|
| 825 | * @author Tom N Harris <[email protected]>
|
---|
| 826 | */
|
---|
| 827 | protected function unlock() {
|
---|
| 828 | global $conf;
|
---|
| 829 | @rmdir($conf['lockdir'].'/_indexer.lock');
|
---|
| 830 | return true;
|
---|
| 831 | }
|
---|
| 832 |
|
---|
| 833 | /**
|
---|
| 834 | * Retrieve the entire index.
|
---|
| 835 | *
|
---|
| 836 | * The $suffix argument is for an index that is split into
|
---|
| 837 | * multiple parts. Different index files should use different
|
---|
| 838 | * base names.
|
---|
| 839 | *
|
---|
| 840 | * @param string $idx name of the index
|
---|
| 841 | * @param string $suffix subpart identifier
|
---|
| 842 | * @return array list of lines without CR or LF
|
---|
| 843 | * @author Tom N Harris <[email protected]>
|
---|
| 844 | */
|
---|
| 845 | protected function getIndex($idx, $suffix) {
|
---|
| 846 | global $conf;
|
---|
| 847 | $fn = $conf['indexdir'].'/'.$idx.$suffix.'.idx';
|
---|
| 848 | if (!@file_exists($fn)) return array();
|
---|
| 849 | return file($fn, FILE_IGNORE_NEW_LINES);
|
---|
| 850 | }
|
---|
| 851 |
|
---|
| 852 | /**
|
---|
| 853 | * Replace the contents of the index with an array.
|
---|
| 854 | *
|
---|
| 855 | * @param string $idx name of the index
|
---|
| 856 | * @param string $suffix subpart identifier
|
---|
| 857 | * @param arrayref $linex list of lines without LF
|
---|
| 858 | * @author Tom N Harris <[email protected]>
|
---|
| 859 | */
|
---|
| 860 | protected function saveIndex($idx, $suffix, &$lines) {
|
---|
| 861 | global $conf;
|
---|
| 862 | $fn = $conf['indexdir'].'/'.$idx.$suffix;
|
---|
| 863 | $fh = @fopen($fn.'.tmp', 'w');
|
---|
| 864 | if (!$fh) return false;
|
---|
| 865 | fwrite($fh, join("\n", $lines));
|
---|
| 866 | if (!empty($lines))
|
---|
| 867 | fwrite($fh, "\n");
|
---|
| 868 | fclose($fh);
|
---|
| 869 | if (isset($conf['fperm']))
|
---|
| 870 | chmod($fn.'.tmp', $conf['fperm']);
|
---|
| 871 | io_rename($fn.'.tmp', $fn.'.idx');
|
---|
| 872 | if ($suffix !== '')
|
---|
| 873 | $this->cacheIndexDir($idx, $suffix, empty($lines));
|
---|
| 874 | return true;
|
---|
| 875 | }
|
---|
| 876 |
|
---|
| 877 | /**
|
---|
| 878 | * Retrieve a line from the index.
|
---|
| 879 | *
|
---|
| 880 | * @param string $idx name of the index
|
---|
| 881 | * @param string $suffix subpart identifier
|
---|
| 882 | * @param int $id the line number
|
---|
| 883 | * @return string a line with trailing whitespace removed
|
---|
| 884 | * @author Tom N Harris <[email protected]>
|
---|
| 885 | */
|
---|
| 886 | protected function getIndexKey($idx, $suffix, $id) {
|
---|
| 887 | global $conf;
|
---|
| 888 | $fn = $conf['indexdir'].'/'.$idx.$suffix.'.idx';
|
---|
| 889 | if (!@file_exists($fn)) return '';
|
---|
| 890 | $fh = @fopen($fn, 'r');
|
---|
| 891 | if (!$fh) return '';
|
---|
| 892 | $ln = -1;
|
---|
| 893 | while (($line = fgets($fh)) !== false) {
|
---|
| 894 | if (++$ln == $id) break;
|
---|
| 895 | }
|
---|
| 896 | fclose($fh);
|
---|
| 897 | return rtrim((string)$line);
|
---|
| 898 | }
|
---|
| 899 |
|
---|
| 900 | /**
|
---|
| 901 | * Write a line into the index.
|
---|
| 902 | *
|
---|
| 903 | * @param string $idx name of the index
|
---|
| 904 | * @param string $suffix subpart identifier
|
---|
| 905 | * @param int $id the line number
|
---|
| 906 | * @param string $line line to write
|
---|
| 907 | * @author Tom N Harris <[email protected]>
|
---|
| 908 | */
|
---|
| 909 | protected function saveIndexKey($idx, $suffix, $id, $line) {
|
---|
| 910 | global $conf;
|
---|
| 911 | if (substr($line, -1) != "\n")
|
---|
| 912 | $line .= "\n";
|
---|
| 913 | $fn = $conf['indexdir'].'/'.$idx.$suffix;
|
---|
| 914 | $fh = @fopen($fn.'.tmp', 'w');
|
---|
| 915 | if (!$fh) return false;
|
---|
| 916 | $ih = @fopen($fn.'.idx', 'r');
|
---|
| 917 | if ($ih) {
|
---|
| 918 | $ln = -1;
|
---|
| 919 | while (($curline = fgets($ih)) !== false) {
|
---|
| 920 | fwrite($fh, (++$ln == $id) ? $line : $curline);
|
---|
| 921 | }
|
---|
| 922 | if ($id > $ln) {
|
---|
| 923 | while ($id > ++$ln)
|
---|
| 924 | fwrite($fh, "\n");
|
---|
| 925 | fwrite($fh, $line);
|
---|
| 926 | }
|
---|
| 927 | fclose($ih);
|
---|
| 928 | } else {
|
---|
| 929 | $ln = -1;
|
---|
| 930 | while ($id > ++$ln)
|
---|
| 931 | fwrite($fh, "\n");
|
---|
| 932 | fwrite($fh, $line);
|
---|
| 933 | }
|
---|
| 934 | fclose($fh);
|
---|
| 935 | if (isset($conf['fperm']))
|
---|
| 936 | chmod($fn.'.tmp', $conf['fperm']);
|
---|
| 937 | io_rename($fn.'.tmp', $fn.'.idx');
|
---|
| 938 | if ($suffix !== '')
|
---|
| 939 | $this->cacheIndexDir($idx, $suffix);
|
---|
| 940 | return true;
|
---|
| 941 | }
|
---|
| 942 |
|
---|
| 943 | /**
|
---|
| 944 | * Retrieve or insert a value in the index.
|
---|
| 945 | *
|
---|
| 946 | * @param string $idx name of the index
|
---|
| 947 | * @param string $suffix subpart identifier
|
---|
| 948 | * @param string $value line to find in the index
|
---|
| 949 | * @return int line number of the value in the index
|
---|
| 950 | * @author Tom N Harris <[email protected]>
|
---|
| 951 | */
|
---|
| 952 | protected function addIndexKey($idx, $suffix, $value) {
|
---|
| 953 | $index = $this->getIndex($idx, $suffix);
|
---|
| 954 | $id = array_search($value, $index);
|
---|
| 955 | if ($id === false) {
|
---|
| 956 | $id = count($index);
|
---|
| 957 | $index[$id] = $value;
|
---|
| 958 | if (!$this->saveIndex($idx, $suffix, $index)) {
|
---|
| 959 | trigger_error("Failed to write $idx index", E_USER_ERROR);
|
---|
| 960 | return false;
|
---|
| 961 | }
|
---|
| 962 | }
|
---|
| 963 | return $id;
|
---|
| 964 | }
|
---|
| 965 |
|
---|
| 966 | protected function cacheIndexDir($idx, $suffix, $delete=false) {
|
---|
| 967 | global $conf;
|
---|
| 968 | if ($idx == 'i')
|
---|
| 969 | $cachename = $conf['indexdir'].'/lengths';
|
---|
| 970 | else
|
---|
| 971 | $cachename = $conf['indexdir'].'/'.$idx.'lengths';
|
---|
| 972 | $lengths = @file($cachename.'.idx', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
|
---|
| 973 | if ($lengths === false) $lengths = array();
|
---|
| 974 | $old = array_search((string)$suffix, $lengths);
|
---|
| 975 | if (empty($lines)) {
|
---|
| 976 | if ($old === false) return;
|
---|
| 977 | unset($lengths[$old]);
|
---|
| 978 | } else {
|
---|
| 979 | if ($old !== false) return;
|
---|
| 980 | $lengths[] = $suffix;
|
---|
| 981 | sort($lengths);
|
---|
| 982 | }
|
---|
| 983 | $fh = @fopen($cachename.'.tmp', 'w');
|
---|
| 984 | if (!$fh) {
|
---|
| 985 | trigger_error("Failed to write index cache", E_USER_ERROR);
|
---|
| 986 | return;
|
---|
| 987 | }
|
---|
| 988 | @fwrite($fh, implode("\n", $lengths));
|
---|
| 989 | @fclose($fh);
|
---|
| 990 | if (isset($conf['fperm']))
|
---|
| 991 | chmod($cachename.'.tmp', $conf['fperm']);
|
---|
| 992 | io_rename($cachename.'.tmp', $cachename.'.idx');
|
---|
| 993 | }
|
---|
| 994 |
|
---|
| 995 | /**
|
---|
| 996 | * Get the list of lengths indexed in the wiki.
|
---|
| 997 | *
|
---|
| 998 | * Read the index directory or a cache file and returns
|
---|
| 999 | * a sorted array of lengths of the words used in the wiki.
|
---|
| 1000 | *
|
---|
| 1001 | * @author YoBoY <[email protected]>
|
---|
| 1002 | */
|
---|
| 1003 | protected function listIndexLengths() {
|
---|
| 1004 | global $conf;
|
---|
| 1005 | $cachename = $conf['indexdir'].'/lengths';
|
---|
| 1006 | clearstatcache();
|
---|
| 1007 | if (@file_exists($cachename.'.idx')) {
|
---|
| 1008 | $lengths = @file($cachename.'.idx', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
|
---|
| 1009 | if ($lengths !== false) {
|
---|
| 1010 | $idx = array();
|
---|
| 1011 | foreach ($lengths as $length)
|
---|
| 1012 | $idx[] = (int)$length;
|
---|
| 1013 | return $idx;
|
---|
| 1014 | }
|
---|
| 1015 | }
|
---|
| 1016 |
|
---|
| 1017 | $dir = @opendir($conf['indexdir']);
|
---|
| 1018 | if ($dir === false)
|
---|
| 1019 | return array();
|
---|
| 1020 | $lengths[] = array();
|
---|
| 1021 | while (($f = readdir($dir)) !== false) {
|
---|
| 1022 | if (substr($f, 0, 1) == 'i' && substr($f, -4) == '.idx') {
|
---|
| 1023 | $i = substr($f, 1, -4);
|
---|
| 1024 | if (is_numeric($i))
|
---|
| 1025 | $lengths[] = (int)$i;
|
---|
| 1026 | }
|
---|
| 1027 | }
|
---|
| 1028 | closedir($dir);
|
---|
| 1029 | sort($lengths);
|
---|
| 1030 | // save this in a file
|
---|
| 1031 | $fh = @fopen($cachename.'.tmp', 'w');
|
---|
| 1032 | if (!$fh) {
|
---|
| 1033 | trigger_error("Failed to write index cache", E_USER_ERROR);
|
---|
| 1034 | return;
|
---|
| 1035 | }
|
---|
| 1036 | @fwrite($fh, implode("\n", $lengths));
|
---|
| 1037 | @fclose($fh);
|
---|
| 1038 | if (isset($conf['fperm']))
|
---|
| 1039 | chmod($cachename.'.tmp', $conf['fperm']);
|
---|
| 1040 | io_rename($cachename.'.tmp', $cachename.'.idx');
|
---|
| 1041 |
|
---|
| 1042 | return $lengths;
|
---|
| 1043 | }
|
---|
| 1044 |
|
---|
| 1045 | /**
|
---|
| 1046 | * Get the word lengths that have been indexed.
|
---|
| 1047 | *
|
---|
| 1048 | * Reads the index directory and returns an array of lengths
|
---|
| 1049 | * that there are indices for.
|
---|
| 1050 | *
|
---|
| 1051 | * @author YoBoY <[email protected]>
|
---|
| 1052 | */
|
---|
| 1053 | protected function indexLengths($filter) {
|
---|
| 1054 | global $conf;
|
---|
| 1055 | $idx = array();
|
---|
| 1056 | if (is_array($filter)) {
|
---|
| 1057 | // testing if index files exist only
|
---|
| 1058 | $path = $conf['indexdir']."/i";
|
---|
| 1059 | foreach ($filter as $key => $value) {
|
---|
| 1060 | if (@file_exists($path.$key.'.idx'))
|
---|
| 1061 | $idx[] = $key;
|
---|
| 1062 | }
|
---|
| 1063 | } else {
|
---|
| 1064 | $lengths = idx_listIndexLengths();
|
---|
| 1065 | foreach ($lengths as $key => $length) {
|
---|
| 1066 | // keep all the values equal or superior
|
---|
| 1067 | if ((int)$length >= (int)$filter)
|
---|
| 1068 | $idx[] = $length;
|
---|
| 1069 | }
|
---|
| 1070 | }
|
---|
| 1071 | return $idx;
|
---|
| 1072 | }
|
---|
| 1073 |
|
---|
| 1074 | /**
|
---|
| 1075 | * Insert or replace a tuple in a line.
|
---|
| 1076 | *
|
---|
| 1077 | * @author Tom N Harris <[email protected]>
|
---|
| 1078 | */
|
---|
| 1079 | protected function updateTuple($line, $id, $count) {
|
---|
| 1080 | $newLine = $line;
|
---|
| 1081 | if ($newLine !== '')
|
---|
| 1082 | $newLine = preg_replace('/(^|:)'.preg_quote($id,'/').'\*\d*/', '', $newLine);
|
---|
| 1083 | $newLine = trim($newLine, ':');
|
---|
| 1084 | if ($count) {
|
---|
| 1085 | if (strlen($newLine) > 0)
|
---|
| 1086 | return "$id*$count:".$newLine;
|
---|
| 1087 | else
|
---|
| 1088 | return "$id*$count".$newLine;
|
---|
| 1089 | }
|
---|
| 1090 | return $newLine;
|
---|
| 1091 | }
|
---|
| 1092 |
|
---|
| 1093 | /**
|
---|
| 1094 | * Split a line into an array of tuples.
|
---|
| 1095 | *
|
---|
| 1096 | * @author Tom N Harris <[email protected]>
|
---|
| 1097 | * @author Andreas Gohr <[email protected]>
|
---|
| 1098 | */
|
---|
| 1099 | protected function parseTuples(&$keys, $line) {
|
---|
| 1100 | $result = array();
|
---|
| 1101 | if ($line == '') return $result;
|
---|
| 1102 | $parts = explode(':', $line);
|
---|
| 1103 | foreach ($parts as $tuple) {
|
---|
| 1104 | if ($tuple === '') continue;
|
---|
| 1105 | list($key, $cnt) = explode('*', $tuple);
|
---|
| 1106 | if (!$cnt) continue;
|
---|
| 1107 | $key = $keys[$key];
|
---|
| 1108 | if (!$key) continue;
|
---|
| 1109 | $result[$key] = $cnt;
|
---|
| 1110 | }
|
---|
| 1111 | return $result;
|
---|
| 1112 | }
|
---|
| 1113 |
|
---|
| 1114 | /**
|
---|
| 1115 | * Sum the counts in a list of tuples.
|
---|
| 1116 | *
|
---|
| 1117 | * @author Tom N Harris <[email protected]>
|
---|
| 1118 | */
|
---|
| 1119 | protected function countTuples($line) {
|
---|
| 1120 | $freq = 0;
|
---|
| 1121 | $parts = explode(':', $line);
|
---|
| 1122 | foreach ($parts as $tuple) {
|
---|
| 1123 | if ($tuple === '') continue;
|
---|
| 1124 | list($pid, $cnt) = explode('*', $tuple);
|
---|
| 1125 | $freq += (int)$cnt;
|
---|
| 1126 | }
|
---|
| 1127 | return $freq;
|
---|
| 1128 | }
|
---|
| 1129 | }
|
---|
| 1130 |
|
---|
| 1131 | /**
|
---|
| 1132 | * Create an instance of the indexer.
|
---|
| 1133 | *
|
---|
| 1134 | * @return object a Doku_Indexer
|
---|
| 1135 | * @author Tom N Harris <[email protected]>
|
---|
| 1136 | */
|
---|
| 1137 | function idx_get_indexer() {
|
---|
| 1138 | static $Indexer = null;
|
---|
| 1139 | if (is_null($Indexer)) {
|
---|
| 1140 | $Indexer = new Doku_Indexer();
|
---|
| 1141 | }
|
---|
| 1142 | return $Indexer;
|
---|
| 1143 | }
|
---|
| 1144 |
|
---|
| 1145 | /**
|
---|
| 1146 | * Returns words that will be ignored.
|
---|
| 1147 | *
|
---|
| 1148 | * @return array list of stop words
|
---|
| 1149 | * @author Tom N Harris <[email protected]>
|
---|
| 1150 | */
|
---|
| 1151 | function & idx_get_stopwords() {
|
---|
| 1152 | static $stopwords = null;
|
---|
| 1153 | if (is_null($stopwords)) {
|
---|
| 1154 | global $conf;
|
---|
| 1155 | $swfile = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt';
|
---|
| 1156 | if(@file_exists($swfile)){
|
---|
| 1157 | $stopwords = file($swfile, FILE_IGNORE_NEW_LINES);
|
---|
| 1158 | }else{
|
---|
| 1159 | $stopwords = array();
|
---|
| 1160 | }
|
---|
| 1161 | }
|
---|
| 1162 | return $stopwords;
|
---|
| 1163 | }
|
---|
| 1164 |
|
---|
| 1165 | /**
|
---|
| 1166 | * Adds/updates the search index for the given page
|
---|
| 1167 | *
|
---|
| 1168 | * Locking is handled internally.
|
---|
| 1169 | *
|
---|
| 1170 | * @param string $page name of the page to index
|
---|
| 1171 | * @param boolean $verbose print status messages
|
---|
| 1172 | * @param boolean $force force reindexing even when the index is up to date
|
---|
| 1173 | * @return boolean the function completed successfully
|
---|
| 1174 | * @author Tom N Harris <[email protected]>
|
---|
| 1175 | */
|
---|
| 1176 | function idx_addPage($page, $verbose=false, $force=false) {
|
---|
| 1177 | // check if indexing needed
|
---|
| 1178 | $idxtag = metaFN($page,'.indexed');
|
---|
| 1179 | if(!$force && @file_exists($idxtag)){
|
---|
| 1180 | if(trim(io_readFile($idxtag)) == idx_get_version()){
|
---|
| 1181 | $last = @filemtime($idxtag);
|
---|
| 1182 | if($last > @filemtime(wikiFN($page))){
|
---|
| 1183 | if ($verbose) print("Indexer: index for $page up to date".DOKU_LF);
|
---|
| 1184 | return false;
|
---|
| 1185 | }
|
---|
| 1186 | }
|
---|
| 1187 | }
|
---|
| 1188 |
|
---|
| 1189 | if (!page_exists($page)) {
|
---|
| 1190 | if (!@file_exists($idxtag)) {
|
---|
| 1191 | if ($verbose) print("Indexer: $page does not exist, ignoring".DOKU_LF);
|
---|
| 1192 | return false;
|
---|
| 1193 | }
|
---|
| 1194 | $Indexer = idx_get_indexer();
|
---|
| 1195 | $result = $Indexer->deletePage($page);
|
---|
| 1196 | if ($result === "locked") {
|
---|
| 1197 | if ($verbose) print("Indexer: locked".DOKU_LF);
|
---|
| 1198 | return false;
|
---|
| 1199 | }
|
---|
| 1200 | @unlink($idxtag);
|
---|
| 1201 | return $result;
|
---|
| 1202 | }
|
---|
| 1203 | $indexenabled = p_get_metadata($page, 'internal index', METADATA_RENDER_UNLIMITED);
|
---|
| 1204 | if ($indexenabled === false) {
|
---|
| 1205 | $result = false;
|
---|
| 1206 | if (@file_exists($idxtag)) {
|
---|
| 1207 | $Indexer = idx_get_indexer();
|
---|
| 1208 | $result = $Indexer->deletePage($page);
|
---|
| 1209 | if ($result === "locked") {
|
---|
| 1210 | if ($verbose) print("Indexer: locked".DOKU_LF);
|
---|
| 1211 | return false;
|
---|
| 1212 | }
|
---|
| 1213 | @unlink($idxtag);
|
---|
| 1214 | }
|
---|
| 1215 | if ($verbose) print("Indexer: index disabled for $page".DOKU_LF);
|
---|
| 1216 | return $result;
|
---|
| 1217 | }
|
---|
| 1218 |
|
---|
| 1219 | $body = '';
|
---|
| 1220 | $metadata = array();
|
---|
| 1221 | $metadata['title'] = p_get_metadata($page, 'title', METADATA_RENDER_UNLIMITED);
|
---|
| 1222 | if (($references = p_get_metadata($page, 'relation references', METADATA_RENDER_UNLIMITED)) !== null)
|
---|
| 1223 | $metadata['relation_references'] = array_keys($references);
|
---|
| 1224 | else
|
---|
| 1225 | $metadata['relation_references'] = array();
|
---|
| 1226 | $data = compact('page', 'body', 'metadata');
|
---|
| 1227 | $evt = new Doku_Event('INDEXER_PAGE_ADD', $data);
|
---|
| 1228 | if ($evt->advise_before()) $data['body'] = $data['body'] . " " . rawWiki($page);
|
---|
| 1229 | $evt->advise_after();
|
---|
| 1230 | unset($evt);
|
---|
| 1231 | extract($data);
|
---|
| 1232 |
|
---|
| 1233 | $Indexer = idx_get_indexer();
|
---|
| 1234 | $result = $Indexer->addPageWords($page, $body);
|
---|
| 1235 | if ($result === "locked") {
|
---|
| 1236 | if ($verbose) print("Indexer: locked".DOKU_LF);
|
---|
| 1237 | return false;
|
---|
| 1238 | }
|
---|
| 1239 |
|
---|
| 1240 | if ($result) {
|
---|
| 1241 | $result = $Indexer->addMetaKeys($page, $metadata);
|
---|
| 1242 | if ($result === "locked") {
|
---|
| 1243 | if ($verbose) print("Indexer: locked".DOKU_LF);
|
---|
| 1244 | return false;
|
---|
| 1245 | }
|
---|
| 1246 | }
|
---|
| 1247 |
|
---|
| 1248 | if ($result)
|
---|
| 1249 | io_saveFile(metaFN($page,'.indexed'), idx_get_version());
|
---|
| 1250 | if ($verbose) {
|
---|
| 1251 | print("Indexer: finished".DOKU_LF);
|
---|
| 1252 | return true;
|
---|
| 1253 | }
|
---|
| 1254 | return $result;
|
---|
| 1255 | }
|
---|
| 1256 |
|
---|
| 1257 | /**
|
---|
| 1258 | * Find tokens in the fulltext index
|
---|
| 1259 | *
|
---|
| 1260 | * Takes an array of words and will return a list of matching
|
---|
| 1261 | * pages for each one.
|
---|
| 1262 | *
|
---|
| 1263 | * Important: No ACL checking is done here! All results are
|
---|
| 1264 | * returned, regardless of permissions
|
---|
| 1265 | *
|
---|
| 1266 | * @param arrayref $words list of words to search for
|
---|
| 1267 | * @return array list of pages found, associated with the search terms
|
---|
| 1268 | */
|
---|
| 1269 | function idx_lookup(&$words) {
|
---|
| 1270 | $Indexer = idx_get_indexer();
|
---|
| 1271 | return $Indexer->lookup($words);
|
---|
| 1272 | }
|
---|
| 1273 |
|
---|
| 1274 | /**
|
---|
| 1275 | * Split a string into tokens
|
---|
| 1276 | *
|
---|
| 1277 | */
|
---|
| 1278 | function idx_tokenizer($string, $wc=false) {
|
---|
| 1279 | $Indexer = idx_get_indexer();
|
---|
| 1280 | return $Indexer->tokenizer($string, $wc);
|
---|
| 1281 | }
|
---|
| 1282 |
|
---|
| 1283 | /* For compatibility */
|
---|
| 1284 |
|
---|
| 1285 | /**
|
---|
| 1286 | * Read the list of words in an index (if it exists).
|
---|
| 1287 | *
|
---|
| 1288 | * @author Tom N Harris <[email protected]>
|
---|
| 1289 | */
|
---|
| 1290 | function idx_getIndex($idx, $suffix) {
|
---|
| 1291 | global $conf;
|
---|
| 1292 | $fn = $conf['indexdir'].'/'.$idx.$suffix.'.idx';
|
---|
| 1293 | if (!@file_exists($fn)) return array();
|
---|
| 1294 | return file($fn);
|
---|
| 1295 | }
|
---|
| 1296 |
|
---|
| 1297 | /**
|
---|
| 1298 | * Get the list of lengths indexed in the wiki.
|
---|
| 1299 | *
|
---|
| 1300 | * Read the index directory or a cache file and returns
|
---|
| 1301 | * a sorted array of lengths of the words used in the wiki.
|
---|
| 1302 | *
|
---|
| 1303 | * @author YoBoY <[email protected]>
|
---|
| 1304 | */
|
---|
| 1305 | function idx_listIndexLengths() {
|
---|
| 1306 | global $conf;
|
---|
| 1307 | // testing what we have to do, create a cache file or not.
|
---|
| 1308 | if ($conf['readdircache'] == 0) {
|
---|
| 1309 | $docache = false;
|
---|
| 1310 | } else {
|
---|
| 1311 | clearstatcache();
|
---|
| 1312 | if (@file_exists($conf['indexdir'].'/lengths.idx')
|
---|
| 1313 | && (time() < @filemtime($conf['indexdir'].'/lengths.idx') + $conf['readdircache'])) {
|
---|
| 1314 | if (($lengths = @file($conf['indexdir'].'/lengths.idx', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES)) !== false) {
|
---|
| 1315 | $idx = array();
|
---|
| 1316 | foreach ($lengths as $length) {
|
---|
| 1317 | $idx[] = (int)$length;
|
---|
| 1318 | }
|
---|
| 1319 | return $idx;
|
---|
| 1320 | }
|
---|
| 1321 | }
|
---|
| 1322 | $docache = true;
|
---|
| 1323 | }
|
---|
| 1324 |
|
---|
| 1325 | if ($conf['readdircache'] == 0 || $docache) {
|
---|
| 1326 | $dir = @opendir($conf['indexdir']);
|
---|
| 1327 | if ($dir === false)
|
---|
| 1328 | return array();
|
---|
| 1329 | $idx = array();
|
---|
| 1330 | while (($f = readdir($dir)) !== false) {
|
---|
| 1331 | if (substr($f, 0, 1) == 'i' && substr($f, -4) == '.idx') {
|
---|
| 1332 | $i = substr($f, 1, -4);
|
---|
| 1333 | if (is_numeric($i))
|
---|
| 1334 | $idx[] = (int)$i;
|
---|
| 1335 | }
|
---|
| 1336 | }
|
---|
| 1337 | closedir($dir);
|
---|
| 1338 | sort($idx);
|
---|
| 1339 | // save this in a file
|
---|
| 1340 | if ($docache) {
|
---|
| 1341 | $handle = @fopen($conf['indexdir'].'/lengths.idx', 'w');
|
---|
| 1342 | @fwrite($handle, implode("\n", $idx));
|
---|
| 1343 | @fclose($handle);
|
---|
| 1344 | }
|
---|
| 1345 | return $idx;
|
---|
| 1346 | }
|
---|
| 1347 |
|
---|
| 1348 | return array();
|
---|
| 1349 | }
|
---|
| 1350 |
|
---|
| 1351 | /**
|
---|
| 1352 | * Get the word lengths that have been indexed.
|
---|
| 1353 | *
|
---|
| 1354 | * Reads the index directory and returns an array of lengths
|
---|
| 1355 | * that there are indices for.
|
---|
| 1356 | *
|
---|
| 1357 | * @author YoBoY <[email protected]>
|
---|
| 1358 | */
|
---|
| 1359 | function idx_indexLengths($filter) {
|
---|
| 1360 | global $conf;
|
---|
| 1361 | $idx = array();
|
---|
| 1362 | if (is_array($filter)) {
|
---|
| 1363 | // testing if index files exist only
|
---|
| 1364 | $path = $conf['indexdir']."/i";
|
---|
| 1365 | foreach ($filter as $key => $value) {
|
---|
| 1366 | if (@file_exists($path.$key.'.idx'))
|
---|
| 1367 | $idx[] = $key;
|
---|
| 1368 | }
|
---|
| 1369 | } else {
|
---|
| 1370 | $lengths = idx_listIndexLengths();
|
---|
| 1371 | foreach ($lengths as $key => $length) {
|
---|
| 1372 | // keep all the values equal or superior
|
---|
| 1373 | if ((int)$length >= (int)$filter)
|
---|
| 1374 | $idx[] = $length;
|
---|
| 1375 | }
|
---|
| 1376 | }
|
---|
| 1377 | return $idx;
|
---|
| 1378 | }
|
---|
| 1379 |
|
---|
| 1380 | /**
|
---|
| 1381 | * Clean a name of a key for use as a file name.
|
---|
| 1382 | *
|
---|
| 1383 | * Romanizes non-latin characters, then strips away anything that's
|
---|
| 1384 | * not a letter, number, or underscore.
|
---|
| 1385 | *
|
---|
| 1386 | * @author Tom N Harris <[email protected]>
|
---|
| 1387 | */
|
---|
| 1388 | function idx_cleanName($name) {
|
---|
| 1389 | $name = utf8_romanize(trim((string)$name));
|
---|
| 1390 | $name = preg_replace('#[ \./\\:-]+#', '_', $name);
|
---|
| 1391 | $name = preg_replace('/[^A-Za-z0-9_]/', '', $name);
|
---|
| 1392 | return strtolower($name);
|
---|
| 1393 | }
|
---|
| 1394 |
|
---|
| 1395 | //Setup VIM: ex: et ts=4 :
|
---|