Changeset 29629 for main/trunk/greenstone2/common-src

Timestamp:

2014-12-15T09:57:01+13:00 (9 years ago)

Author:

kjdon

Message:

updating AndFragsToQueryResult to use actual num docs in weight generation

File:

• main/trunk/greenstone2/common-src/indexers/mgpp/text/Terms.cpp (modified) (9 diffs)

main/trunk/greenstone2/common-src/indexers/mgpp/text/Terms.cpp

@@ -582 +582 @@
   
   // log (N / ft)
-  float wordLog =
-    log((double)indexData.levels.levelInfo[indexData.curLevel].numEntries/
-    (double)termData.matchDocs);
+  //float wordLog =
+  //    log((double)indexData.levels.levelInfo[indexData.curLevel].numEntries/
+  //    (double)termData.matchDocs);
 
   // Wqt = fqt * log (N / ft)
   // note: terms are allowed to have a weight of zero so
   // they can be excluded from the ranking
-  float Wqt = termWeight * wordLog;
+  //float Wqt = termWeight * wordLog;
 
   // Wdt = fdt * log (N / ft)
@@ -605 +605 @@
   mg_u_long resultOutI = 0;
   
-  mg_u_long actual_num_term_match_docs = 0;
-  
+  mg_u_long actual_num_match_docs = 0;
+  vector<mg_u_long> docFreqsArray;
   while (termDataI < termDataSize) {
     if (indexData.levelConverter.FragToLevel (termData.fragNums[termDataI],
@@ -612 +612 @@
       if (levelDocNum != lastLevelDocNum) {
     if (lastLevelDocNum > 0) {
-      ++actual_num_term_match_docs;
-
-      Wdt = termDocFreq * wordLog;
+      ++actual_num_match_docs;
+
+      //Wdt = termDocFreq * wordLog;
       
       // find this document number
@@ -624 +624 @@
       if (resultI < resultSize && result.docs[resultI] == lastLevelDocNum) {
         result.docs[resultOutI] = lastLevelDocNum;
-        if (needRanks)
-          result.ranks[resultOutI] = result.ranks[resultI] + Wqt * Wdt;
+        if (needRanks) {
+          // store the doc freq so we can calulate the rank for the new term
+          // once we know the num docs
+          docFreqsArray.push_back(termDocFreq);
+          // just store the old rank for now, and we'll add on the new bit at the end
+          result.ranks[resultOutI] = result.ranks[resultI]; // + Wqt * Wdt;
+        }
         ++resultI;
         ++resultOutI;
@@ -635 +640 @@
       }
 
-      if (needRanks)
+      if (needRanks) {
     termDocFreq += termData.fragFreqs[termDataI];
-     overallwordfreq += termData.fragFreqs[termDataI]; 
+      }
+      overallwordfreq += termData.fragFreqs[termDataI]; 
     }
     
@@ -644 +650 @@
 
   if (lastLevelDocNum > 0) {
-    ++actual_num_term_match_docs;
+    ++actual_num_match_docs;
     // add the last document information
-    Wdt = termDocFreq * wordLog;
+    //Wdt = termDocFreq * wordLog;
 
     // find this document number
@@ -656 +662 @@
     if (resultI < resultSize && result.docs[resultI] == lastLevelDocNum) {
       result.docs[resultOutI] = lastLevelDocNum;
-      if (needRanks)
-    result.ranks[resultOutI] = result.ranks[resultI] + Wqt * Wdt;
+      if (needRanks) {
+    // store the doc freq so we can calulate the rank for the new term
+    // once we know the num docs
+    docFreqsArray.push_back(termDocFreq);
+    // just store the old rank for now, and we'll add on the new bit at the end
+    result.ranks[resultOutI] = result.ranks[resultI]; // + Wqt * Wdt;
+      }
       ++resultI;
       ++resultOutI;
@@ -669 +680 @@
   else
     result.ranks.erase (result.ranks.begin(), result.ranks.end());
+  
+  // Calcalate correct ranks
+  float wordLog = log((double)indexData.levels.levelInfo[indexData.curLevel].numEntries / (double)actual_num_match_docs);
+  float Wqt = termWeight * wordLog;
+  float factor = wordLog * Wqt;
+
+  mg_u_long docFreqI = 0;
+  mg_u_long docFreqSize = docFreqsArray.size();
+  
+  while (docFreqI < docFreqSize) {
+    result.ranks[docFreqI] = result.ranks[docFreqI] + docFreqsArray[docFreqI]*factor;
+    ++docFreqI;
+  }
   
   // add the term frequency information
@@ -678 +702 @@
     termFreqData.equivTerms = equivTerms;
     //termFreqData.matchDocs = termData.matchDocs;
-    termFreqData.matchDocs = actual_num_term_match_docs;
+    termFreqData.matchDocs = actual_num_match_docs;
     termFreqData.termFreq = overallwordfreq;
     result.termFreqs.push_back (termFreqData);