Changeset 7582 for trunk/indexers

Timestamp:

2004-06-10T15:02:05+12:00 (20 years ago)

Author:

kjdon

Message:

fixed the bug that was causing it not to be able to create a second index using jni - basically had to reset all the static variables at the start of each pass. the tricky thing to find was the static variables in occur_to_lexical in ivf.pass2

Location:

trunk/indexers/mg/src/text

Files:

• ivf.pass1.c (modified) (7 diffs)
• ivf.pass2.c (modified) (15 diffs)

trunk/indexers/mg/src/text/ivf.pass1.c

@@ -45 +45 @@
 /*
    $Log$
+   Revision 1.2  2004/06/10 03:02:05  kjdon
+   fixed the bug that was causing it not to be able to create a second index using jni - basically had to reset all the static variables at the start of each pass. the tricky thing to find was the static variables in occur_to_lexical in ivf.pass2
+
    Revision 1.1  2003/02/20 21:18:23  mdewsnip
    Addition of MG package for search and retrieval
@@ -123 +126 @@
 hash_rec;
 
-
-
-
-
-
-
-
 static unsigned long words_read = 0, words_diff = 0, bytes_diff = 0;
 static unsigned long outputbytes = 0;
@@ -142 +138 @@
 
 static hash_rec **HashTable;
-static unsigned long HashSize;
-static unsigned long HashUsed;
+static unsigned long HashSize=0;
+static unsigned long HashUsed=0;
 static u_char *Pool;
-static int PoolLeft;
+static int PoolLeft=0;
 
 static hash_rec *hr_pool;
-static int hr_PoolLeft;
+static int hr_PoolLeft=0;
 
 static hash_rec **first_occr;
-static int max_first_occr;
+static int max_first_occr=0;
 
 static mg_ullong L1_bits = 0;
@@ -161 +157 @@
 static unsigned long callnum = 0, lcallnum = 0, wordnum = 0, lwordnum = 0;
 static unsigned long ptrcnt = 0;
-static unsigned long checknum;
+static unsigned long checknum=0;
 static long max_mem = 0;
 
@@ -173 +169 @@
 }
 
-
-
+void
+ResetStaticI1Vars () 
+{
+  words_read = 0;
+  words_diff = 0; 
+  bytes_diff = 0;
+  outputbytes = 0;
+  inputbytes = 0;
+  MaxMemInUse = 0;
+  MemInUse = 0;
+  ChunksWritten = 0;
+  
+  HashSize=0;
+  HashUsed=0;
+  PoolLeft=0;
+  
+  hr_PoolLeft=0;
+  
+  max_first_occr=0;
+  
+  L1_bits = 0;
+  L2_bits = 0;
+  L3_bits = 0;
+  L1_ohead = 0;
+  L2_ohead = 0;
+  L3_ohead = 0;
+  callnum = 0;
+  lcallnum = 0;
+  wordnum = 0;
+  lwordnum = 0;
+  ptrcnt = 0;
+  checknum=0;
+  max_mem = 0;
+  
+}
 int 
 init_ivf_1 (char *file_name)
 {
+  ResetStaticI1Vars();
   if (!(ic = create_file (file_name, INVF_CHUNK_SUFFIX, "wb", MAGIC_CHUNK,
               MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
@@ -784 +814 @@
 }
 
-
-
-
 int 
 done_ivf_1 (char *FileName)
@@ -822 +849 @@
   msg_prefix = temp_str;
 
+  /* free the memory */
+  Xfree(HashTable);
+  if (Pool) {
+    Pool -= (POOL_SIZE-PoolLeft);
+    Xfree(Pool);
+  }
+  if (hr_pool) {
+    hr_pool -= (HASH_POOL_SIZE-hr_PoolLeft);
+    Xfree(hr_pool);
+  }
+  if (first_occr) {
+    Xfree(first_occr);
+  }
+
   return (COMPALLOK);
 }               /* done_encode */

trunk/indexers/mg/src/text/ivf.pass2.c

@@ -24 +24 @@
 /*
    $Log$
+   Revision 1.2  2004/06/10 03:02:05  kjdon
+   fixed the bug that was causing it not to be able to create a second index using jni - basically had to reset all the static variables at the start of each pass. the tricky thing to find was the static variables in occur_to_lexical in ivf.pass2
+
    Revision 1.1  2003/02/20 21:18:23  mdewsnip
    Addition of MG package for search and retrieval
@@ -208 +211 @@
 }
 
-
+void 
+ResetStaticI2Vars() 
+{
+  docs_left = 0;
+  next_docs_left = 0;
+  N = 0;
+  MemBufSize=0;
+  BufToUse=0;
+  memset(&idh, 0, sizeof(idh));
+  wl_size = 0;
+  
+  dict_size = 0;
+  no_of_ptrs = 0;
+  chunks_read = 0;
+  Disk_pos = 0;
+  callnum = 0;
+  wordnum = 0;
+  
+  totalIbytes = 0;
+  totalDbytes = 0;
+  totalHbytes = 0;
+  
+  MemInUse = 0;
+  MaxMemInUse = 0;
+  max_buffer_len = 0;
+
+}
 
 
@@ -329 +358 @@
 
 unsigned long 
-occur_to_lexical (long occ)
+occur_to_lexical (long occ, int clear_state)
 {
   static long pos = -1;
   static random_bitio_state rbs;
   static int val = 0;
+  if (clear_state) {
+    pos = -1;
+    val = 0;
+    return 0;
+  }
   if (pos == -1)
     {
@@ -372 +406 @@
 init_ivf_2 (char *file_name)
 {
+  ResetStaticI2Vars();
+  occur_to_lexical(0, 1); /* clear the static vars in here*/
   u_char prev[MAXSTEMLEN + 1];
   int i;
@@ -417 +453 @@
     }
   totalDbytes += sizeof (word_rec) * idh.dict_size;
-
   /* separate storage for the log(b) values, one byte each */
   if (!(lg_bs = Xmalloc (sizeof (u_char) * idh.dict_size)))
@@ -529 +564 @@
   totalIbytes = (totalIbits + 7ul) >> 3ul;
 #endif
-
   return (COMPALLOK);
 
@@ -550 +584 @@
     {
       MemBufSize = sizeof (unsigned long) * dict_size;
-      if (max_buffer_len > MemBufSize)
+      if (max_buffer_len > MemBufSize) 
     MemBufSize = max_buffer_len;
       if (!(MemoryBuffer = Xmalloc (MemBufSize)))
@@ -556 +590 @@
       ChangeMemInUse (MemBufSize);
     }
-
   counts = (unsigned long *) MemoryBuffer;
-  bzero ((char *) counts, sizeof (unsigned long) * dict_size);
-
-  docs_left = next_docs_left;
+/*   bzero ((char *) counts, sizeof (unsigned long) * dict_size); */
+  bzero ((char *) counts, MemBufSize);
+  docs_left = next_docs_left; 
   if (!docs_left)
     FatalError (1, "The number of docs in the current chunk is 0");
@@ -570 +603 @@
   local_N = docs_left;
 
-
-
   for (wr = WordRecs, i = 0; i < dict_size; i++, wr++)
-    wr->ptr = 0;
+    wr->ptr = 0; 
 
   bzero ((char *) lg_bs, dict_size);
@@ -580 +611 @@
     {
       unsigned long word_num, wcnt, fcnt, p;
-      word_num = occur_to_lexical (i);
-
+      word_num = occur_to_lexical (i,0);
+      
       wr = &WordRecs[word_num];
-
+      
       wcnt = BIO_Stdio_Gamma_Decode (&sbs, NULL) - 1;
       if (wcnt >= 2)
@@ -590 +621 @@
     fcnt = wcnt;
 
+
       p = fcnt;
-
       if (wcnt)
     {
@@ -603 +634 @@
     }
 
-    }
-
+
+  }  
   crbs_pos = BIO_Random_Tell (&crbs);
-
   totalIbits = 0;
   last_total = 0;
@@ -781 +811 @@
   Xfree (rbsi);
   ChangeMemInUse (-chunks_read * sizeof (random_bitio_state));
-  chunks_read = 0;
+/*   chunks_read = 0; */
   Xfree (chunk_ptrs);
   ChangeMemInUse (-chunks_read * sizeof (unsigned long));
+  chunks_read = 0;
   Disk_pos = 0;
   BIO_Random_Seek (0, &crbs);
@@ -1007 +1038 @@
 
   free_perf_hash (phd);
-
-  free (MemoryBuffer);
+  phd = NULL;
+
+  Xfree (MemoryBuffer);
+  MemoryBuffer = NULL;
   ChangeMemInUse (-MemBufSize);
 
@@ -1129 +1162 @@
 #endif
 
+  Xfree(chunk_data);
+  chunk_data = NULL;
   Xfree (WordRecs);
+  WordRecs = NULL;
   Xfree (lg_bs);
-
+  lg_bs = NULL;
+  Xfree (idf);
+  idf = NULL;
+  Xfree (word_list);
+  word_list = NULL;
   /* Free the memory allocated for the BIO_Random */
-  occur_to_lexical (-1);
+  occur_to_lexical (-1,1);
 
   BIO_Random_Done (&crbs);