Changeset 7582 for trunk/indexers/mg
- Timestamp:
- 2004-06-10T15:02:05+12:00 (20 years ago)
- Location:
- trunk/indexers/mg/src/text
- Files:
-
- 2 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/indexers/mg/src/text/ivf.pass1.c
r3745 r7582 45 45 /* 46 46 $Log$ 47 Revision 1.2 2004/06/10 03:02:05 kjdon 48 fixed the bug that was causing it not to be able to create a second index using jni - basically had to reset all the static variables at the start of each pass. the tricky thing to find was the static variables in occur_to_lexical in ivf.pass2 49 47 50 Revision 1.1 2003/02/20 21:18:23 mdewsnip 48 51 Addition of MG package for search and retrieval … … 123 126 hash_rec; 124 127 125 126 127 128 129 130 131 132 128 static unsigned long words_read = 0, words_diff = 0, bytes_diff = 0; 133 129 static unsigned long outputbytes = 0; … … 142 138 143 139 static hash_rec **HashTable; 144 static unsigned long HashSize ;145 static unsigned long HashUsed ;140 static unsigned long HashSize=0; 141 static unsigned long HashUsed=0; 146 142 static u_char *Pool; 147 static int PoolLeft ;143 static int PoolLeft=0; 148 144 149 145 static hash_rec *hr_pool; 150 static int hr_PoolLeft ;146 static int hr_PoolLeft=0; 151 147 152 148 static hash_rec **first_occr; 153 static int max_first_occr ;149 static int max_first_occr=0; 154 150 155 151 static mg_ullong L1_bits = 0; … … 161 157 static unsigned long callnum = 0, lcallnum = 0, wordnum = 0, lwordnum = 0; 162 158 static unsigned long ptrcnt = 0; 163 static unsigned long checknum ;159 static unsigned long checknum=0; 164 160 static long max_mem = 0; 165 161 … … 173 169 } 174 170 175 176 171 void 172 ResetStaticI1Vars () 173 { 174 words_read = 0; 175 words_diff = 0; 176 bytes_diff = 0; 177 outputbytes = 0; 178 inputbytes = 0; 179 MaxMemInUse = 0; 180 MemInUse = 0; 181 ChunksWritten = 0; 182 183 HashSize=0; 184 HashUsed=0; 185 PoolLeft=0; 186 187 hr_PoolLeft=0; 188 189 max_first_occr=0; 190 191 L1_bits = 0; 192 L2_bits = 0; 193 L3_bits = 0; 194 L1_ohead = 0; 195 L2_ohead = 0; 196 L3_ohead = 0; 197 callnum = 0; 198 lcallnum = 0; 199 wordnum = 0; 200 lwordnum = 0; 201 ptrcnt = 0; 202 checknum=0; 203 max_mem = 0; 204 205 } 177 206 int 178 207 init_ivf_1 (char *file_name) 179 208 { 209 ResetStaticI1Vars(); 180 210 if (!(ic = create_file (file_name, INVF_CHUNK_SUFFIX, "wb", MAGIC_CHUNK, 181 211 MG_MESSAGE))) /* [RPAP - Feb 97: WIN32 Port] */ … … 784 814 } 785 815 786 787 788 789 816 int 790 817 done_ivf_1 (char *FileName) … … 822 849 msg_prefix = temp_str; 823 850 851 /* free the memory */ 852 Xfree(HashTable); 853 if (Pool) { 854 Pool -= (POOL_SIZE-PoolLeft); 855 Xfree(Pool); 856 } 857 if (hr_pool) { 858 hr_pool -= (HASH_POOL_SIZE-hr_PoolLeft); 859 Xfree(hr_pool); 860 } 861 if (first_occr) { 862 Xfree(first_occr); 863 } 864 824 865 return (COMPALLOK); 825 866 } /* done_encode */ -
trunk/indexers/mg/src/text/ivf.pass2.c
r3745 r7582 24 24 /* 25 25 $Log$ 26 Revision 1.2 2004/06/10 03:02:05 kjdon 27 fixed the bug that was causing it not to be able to create a second index using jni - basically had to reset all the static variables at the start of each pass. the tricky thing to find was the static variables in occur_to_lexical in ivf.pass2 28 26 29 Revision 1.1 2003/02/20 21:18:23 mdewsnip 27 30 Addition of MG package for search and retrieval … … 208 211 } 209 212 210 213 void 214 ResetStaticI2Vars() 215 { 216 docs_left = 0; 217 next_docs_left = 0; 218 N = 0; 219 MemBufSize=0; 220 BufToUse=0; 221 memset(&idh, 0, sizeof(idh)); 222 wl_size = 0; 223 224 dict_size = 0; 225 no_of_ptrs = 0; 226 chunks_read = 0; 227 Disk_pos = 0; 228 callnum = 0; 229 wordnum = 0; 230 231 totalIbytes = 0; 232 totalDbytes = 0; 233 totalHbytes = 0; 234 235 MemInUse = 0; 236 MaxMemInUse = 0; 237 max_buffer_len = 0; 238 239 } 211 240 212 241 … … 329 358 330 359 unsigned long 331 occur_to_lexical (long occ )360 occur_to_lexical (long occ, int clear_state) 332 361 { 333 362 static long pos = -1; 334 363 static random_bitio_state rbs; 335 364 static int val = 0; 365 if (clear_state) { 366 pos = -1; 367 val = 0; 368 return 0; 369 } 336 370 if (pos == -1) 337 371 { … … 372 406 init_ivf_2 (char *file_name) 373 407 { 408 ResetStaticI2Vars(); 409 occur_to_lexical(0, 1); /* clear the static vars in here*/ 374 410 u_char prev[MAXSTEMLEN + 1]; 375 411 int i; … … 417 453 } 418 454 totalDbytes += sizeof (word_rec) * idh.dict_size; 419 420 455 /* separate storage for the log(b) values, one byte each */ 421 456 if (!(lg_bs = Xmalloc (sizeof (u_char) * idh.dict_size))) … … 529 564 totalIbytes = (totalIbits + 7ul) >> 3ul; 530 565 #endif 531 532 566 return (COMPALLOK); 533 567 … … 550 584 { 551 585 MemBufSize = sizeof (unsigned long) * dict_size; 552 if (max_buffer_len > MemBufSize) 586 if (max_buffer_len > MemBufSize) 553 587 MemBufSize = max_buffer_len; 554 588 if (!(MemoryBuffer = Xmalloc (MemBufSize))) … … 556 590 ChangeMemInUse (MemBufSize); 557 591 } 558 559 592 counts = (unsigned long *) MemoryBuffer; 560 bzero ((char *) counts, sizeof (unsigned long) * dict_size); 561 562 docs_left = next_docs_left; 593 /* bzero ((char *) counts, sizeof (unsigned long) * dict_size); */ 594 bzero ((char *) counts, MemBufSize); 595 docs_left = next_docs_left; 563 596 if (!docs_left) 564 597 FatalError (1, "The number of docs in the current chunk is 0"); … … 570 603 local_N = docs_left; 571 604 572 573 574 605 for (wr = WordRecs, i = 0; i < dict_size; i++, wr++) 575 wr->ptr = 0; 606 wr->ptr = 0; 576 607 577 608 bzero ((char *) lg_bs, dict_size); … … 580 611 { 581 612 unsigned long word_num, wcnt, fcnt, p; 582 word_num = occur_to_lexical (i );583 613 word_num = occur_to_lexical (i,0); 614 584 615 wr = &WordRecs[word_num]; 585 616 586 617 wcnt = BIO_Stdio_Gamma_Decode (&sbs, NULL) - 1; 587 618 if (wcnt >= 2) … … 590 621 fcnt = wcnt; 591 622 623 592 624 p = fcnt; 593 594 625 if (wcnt) 595 626 { … … 603 634 } 604 635 605 } 606 636 637 } 607 638 crbs_pos = BIO_Random_Tell (&crbs); 608 609 639 totalIbits = 0; 610 640 last_total = 0; … … 781 811 Xfree (rbsi); 782 812 ChangeMemInUse (-chunks_read * sizeof (random_bitio_state)); 783 chunks_read = 0; 813 /* chunks_read = 0; */ 784 814 Xfree (chunk_ptrs); 785 815 ChangeMemInUse (-chunks_read * sizeof (unsigned long)); 816 chunks_read = 0; 786 817 Disk_pos = 0; 787 818 BIO_Random_Seek (0, &crbs); … … 1007 1038 1008 1039 free_perf_hash (phd); 1009 1010 free (MemoryBuffer); 1040 phd = NULL; 1041 1042 Xfree (MemoryBuffer); 1043 MemoryBuffer = NULL; 1011 1044 ChangeMemInUse (-MemBufSize); 1012 1045 … … 1129 1162 #endif 1130 1163 1164 Xfree(chunk_data); 1165 chunk_data = NULL; 1131 1166 Xfree (WordRecs); 1167 WordRecs = NULL; 1132 1168 Xfree (lg_bs); 1133 1169 lg_bs = NULL; 1170 Xfree (idf); 1171 idf = NULL; 1172 Xfree (word_list); 1173 word_list = NULL; 1134 1174 /* Free the memory allocated for the BIO_Random */ 1135 occur_to_lexical (-1 );1175 occur_to_lexical (-1,1); 1136 1176 1137 1177 BIO_Random_Done (&crbs);
Note:
See TracChangeset
for help on using the changeset viewer.