Changeset 1300

Timestamp:

2000-07-24T14:46:11+12:00 (24 years ago)

Author:

kjm18

Message:

added full text browsing functionality

Location:

trunk/gsdl/src/mgpp/text

Files:

• MGQuery.cpp (modified) (2 diffs)
• MGQuery.h (modified) (4 diffs)
• Terms.cpp (modified) (2 diffs)
• Terms.h (modified) (2 diffs)
• invf.cpp (modified) (2 diffs)
• invf.h (modified) (2 diffs)

trunk/gsdl/src/mgpp/text/MGQuery.cpp

@@ -572 +572 @@
 
 
+void BrowseQueryNode::Clear () {
+  UCArrayClear(term);
+}
+
+void BrowseQueryNode::Calculate (IndexData &indexData, BrowseQueryResult &result) const {
+
+  unsigned long number=0;
+  FindNearestWordNumber(indexData, term, number);
+  if (number + startPosition > 0 ) {
+    number = number+startPosition;
+  }
+  else {
+    number = 1;
+  }
+
+  GetTermList (indexData, number, numTerms, result.termFreqs);
+
+}
+
+
+
+void BrowseQueryNode::Free () {
+  Clear();
+}
+
+
+void BrowseQueryNode::Print (ostream &s, int indent) const {
+  PrintIndentText(s, "BROWSEQUERYNODE\n", indent);
+  PrintIndent (s, indent+2);
+  s << "TERM:"<<term<<"\n";
+  PrintIndent (s, indent+2);
+  s << "Start position: "<< startPosition<<", Num terms: "<< numTerms<<"\n";
+
+
+}
+
+
+
 void MGQuery (IndexData &indexData,
           const QueryInfo &queryInfo,
@@ -693 +731 @@
 
 
-
+// new function for full text browsing, 
+void MGBrowseQuery (IndexData &indexData, UCArray &level,
+          const BrowseQueryNode &node,
+          BrowseQueryResult &result) {
+
+  indexData.LoadLevel(level);
+  node.Calculate(indexData, result); 
+
+}
+
+
+
+
+
+
+

trunk/gsdl/src/mgpp/text/MGQuery.h

@@ -119 +119 @@
 then 'the' has to be at position between -2 and -1 relative to 'cat'.
 "the cat" could also be searched for by 'cat' with no range limits, then 'the' with range 0 to 1.
+range values are relative to the gaps between words:
+   x   y   z   X   a   b   c
+    -3   -2  -1  0   1   2   3
+
  */
 class TermNode {
@@ -157 +161 @@
 };
 
+class BrowseQueryNode :public QueryNode {
+ public:
+  UCArray term;
+  signed long startPosition;
+  unsigned long numTerms;
+
+  void Clear();
+  BrowseQueryNode () { Clear(); }
+  //  ~BrowseQueryNode ();
+
+  void Calculate (IndexData &indexData, BrowseQueryResult &result) const;
+  void Free ();
+  void Print (ostream &s, int indent=0) const;
+
+
+
+
+};
 
 void MGQuery (IndexData &indexData,
@@ -163 +185 @@
           QueryResult &result);
 
-// this function for retriving results with both section doc nums
+// this function for retrieving results with both section doc nums
 // and Document docnums
 void MGQuery (IndexData &indexData,
@@ -171 +193 @@
 
 
+// new function for full text browsing, 
+void MGBrowseQuery (IndexData &indexData, UCArray &level,
+          const BrowseQueryNode &node,
+          BrowseQueryResult &result);
+
 #endif
+
+
+
+
+
+
+
+
+
+

trunk/gsdl/src/mgpp/text/Terms.cpp

@@ -143 +143 @@
       (r1.termFreqs == r2.termFreqs));
 }
+
+//-------------------------------------------------------
+// new BrowseQueryResult stuff
+void BrowseQueryResult::Clear () {
+  termFreqs.erase (termFreqs.begin(), termFreqs.end());
+}
+
+BrowseQueryResult::BrowseQueryResult () {
+  Clear ();
+}
+
+
+
+ostream &operator<< (ostream &s, const BrowseQueryResult &r) {
+  s << "terms: ";
+  unsigned long i;
+  for (i=0; i<r.termFreqs.size(); i++)
+    s << r.termFreqs[i] << ", ";
+    s << "\n\n";
+  return s;
+}
+
+
+bool operator== (const BrowseQueryResult &r1, const BrowseQueryResult &r2) {
+  return ((r1.termFreqs == r2.termFreqs));
+     
+}
+
+
+
 
 //--------------------------------------
@@ -668 +698 @@
     result.ranks.erase (result.ranks.begin(), result.ranks.end());
 }
+
+
+
+//--------------------------------------------------------------
+// functions to support full text browse
+
+void FindNearestWordNumber (IndexData &indexData,
+                const UCArray &term,
+                unsigned long &number) {
+
+    // find the word number for this term
+    unsigned long wordElNum = 0;
+    unsigned long numLevels = indexData.bdh.num_levels;
+    word_block_dict_el wordDictEl;
+    wordDictEl.SetNumLevels (numLevels);
+    if (NearestSearchWordBlockDictEl (indexData.dictFile, indexData.biWords,
+                      indexData.bdh.entries_per_wblk,
+                      indexData.bdh.word_dict_size,
+                      numLevels, term, wordDictEl, wordElNum))
+      number = wordElNum;
+
+}
+
+void GetTermList(IndexData &indexData,
+         unsigned long startTerm,
+         unsigned long numTerms,
+         TermFreqArray &terms) {
+
+  word_block_dict_el_array wordBlocks; // = new word_block_dict_el_array();
+  TermFreqData termdata;
+
+  terms.erase(terms.begin(), terms.end());
+
+  SearchWordBlockDictElNumRange (indexData.dictFile, indexData.biWords,
+                 indexData.bdh.entries_per_wblk,
+                 indexData.bdh.word_dict_size,
+                 indexData.bdh.num_levels, startTerm,
+                 numTerms, wordBlocks);
+
+  word_block_dict_el_array::iterator here = wordBlocks.begin();
+  word_block_dict_el_array::iterator end = wordBlocks.end();
+
+  while (here != end) {
+    termdata.Clear();
+    termdata.term = (*here).el;
+    termdata.termFreq = (*here).freq;
+    terms.push_back(termdata);
+    here++;
+  }
+
+}
+
+void GetTermList(IndexData &indexData,
+         unsigned long startTerm,
+         unsigned long numTerms,
+         UCArrayVector &terms) {
+
+  
+  
+  SearchWordBlockDictElNumRange (indexData.dictFile, indexData.biWords,
+                 indexData.bdh.entries_per_wblk,
+                 indexData.bdh.word_dict_size,
+                 indexData.bdh.num_levels, startTerm,
+                 numTerms, terms);
+
+}
+
+
+

trunk/gsdl/src/mgpp/text/Terms.h

@@ -163 +163 @@
                 QueryResult &result);
 
-
+//-----------------------------------------------------------------
 // new QueryResult class to handle retrieval of doc and level nums.
 // Use this class with extended version of MGQuery
@@ -180 +180 @@
 bool operator== (const ExtQueryResult &r1, const ExtQueryResult &r2);
 
+//------------------------------------------------------------
+// new functions to handle full text browse
+
+class BrowseQueryResult {
+ public:
+  TermFreqArray termFreqs;
+  void Clear();
+  BrowseQueryResult ();
+
+};
+
+
+ostream &operator<< (ostream &s, const BrowseQueryResult &r);
+bool operator== (const BrowseQueryResult &r1, const BrowseQueryResult &r2);
+
+void FindNearestWordNumber (IndexData &indexData,
+                const UCArray &term,
+                unsigned long &number);
+
+void GetTermList(IndexData &indexData,
+         unsigned long startTerm,
+         unsigned long numTerms,
+         TermFreqArray &terms);
+
+void GetTermList (IndexData &indexData,
+         unsigned long startTerm,
+         unsigned long numTerms,
+         UCArrayVector &terms);
+
 #endif
 
+
+
+
+

trunk/gsdl/src/mgpp/text/invf.cpp

@@ -513 +513 @@
 
 
+// use the block dictionary functions for tag entries, and word block dict
+// functions for word entries.
+
 
 bool SearchBlockDictElNum (FILE *dictFile,
@@ -689 +692 @@
   return false;
 }
+
+//----------------------------------------------------------------
+// functions for full text browse
+
+bool NearestSearchWordBlockDictEl (FILE *dictFile,
+                   const block_idx &bIdx,
+                   unsigned long entriesPerBlock,
+                   unsigned long dictSize,
+                   unsigned long numLevels,
+                   const UCArray &el,
+                   word_block_dict_el &dictEl,
+                   unsigned long &elNum) {
+
+  UCArrayClear (dictEl.el);
+
+  // find the block that contains the element
+  unsigned long blockIdxNum;
+  if (!SearchEl (bIdx, entriesPerBlock, el,
+         blockIdxNum, elNum))
+    return false;
+
+  unsigned long blockEndElNum = elNum + entriesPerBlock;
+  if (blockEndElNum > dictSize) blockEndElNum = dictSize;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  while (elNum < blockEndElNum) {
+    dictEl.Read (dictFile, numLevels);
+    int res = DictCompare (el, dictEl.el); // look for the first word that is
+                                     // greater or equal to the el
+    if (res <= 0) {
+      return true; // found one
+   } 
+    
+    elNum++;
+  }
+  // it must be the last term
+  return true;
+
+
+}
+
+
+bool SearchWordBlockDictElNumRange (FILE *dictFile,
+                    const block_idx &bIdx,
+                    unsigned long entriesPerBlock,
+                    unsigned long dictSize,
+                    unsigned long numLevels,
+                    unsigned long elNum,
+                    unsigned long numWords,
+                    UCArrayVector &terms) {
+
+  word_block_dict_el dictEl;  
+  dictEl.SetNumLevels (numLevels);
+  UCArrayClear(dictEl.el);
+
+  terms.erase(terms.begin(), terms.end());
+
+  if (elNum >= dictSize) return false;
+  
+  // find the block that contains the element
+  unsigned long blockIdxNum, curElNum;
+  if (!SearchElNum (bIdx, entriesPerBlock, elNum,
+            blockIdxNum, curElNum))
+    return false;
+
+  unsigned long lastElNum = elNum + numWords - 1;
+  if (lastElNum > dictSize) lastElNum = dictSize;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  
+  // get the first term
+  do {
+    dictEl.Read (dictFile, numLevels);
+  } while (curElNum++ < elNum);
+
+  terms.push_back(dictEl.el);
+  while (curElNum <= lastElNum ) {
+    dictEl.Read(dictFile, numLevels);
+    terms.push_back(dictEl.el);
+    curElNum++;
+  }
+  
+
+  return true;
+}
+
+// NOte: before each addition of dictEl to the array, the level freqs array
+// is deleted, as this was causing problems - generating a seg fault, I think if
+// the vector had to be reallocated or something.
+// setNumLevels has to be called each time before a read, now, to set up the level
+//freqs array. this is necessary.
+bool SearchWordBlockDictElNumRange (FILE *dictFile,
+                    const block_idx &bIdx,
+                    unsigned long entriesPerBlock,
+                    unsigned long dictSize,
+                    unsigned long numLevels,
+                    unsigned long elNum,
+                    unsigned long numWords,
+                    word_block_dict_el_array &terms) {
+
+  word_block_dict_el dictEl;
+  dictEl.SetNumLevels (numLevels);
+  UCArrayClear(dictEl.el);
+
+  block_dict_el elem;
+  terms.erase(terms.begin(), terms.end());
+
+  if (elNum >= dictSize) return false;
+  
+  // find the block that contains the element
+  unsigned long blockIdxNum, curElNum;
+  if (!SearchElNum (bIdx, entriesPerBlock, elNum,
+            blockIdxNum, curElNum))
+    return false;
+
+  unsigned long lastElNum = elNum + numWords - 1;
+  if (lastElNum > dictSize) lastElNum = dictSize;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  // get the first term
+  do {
+    dictEl.Read (dictFile, numLevels);
+  } while (curElNum++ < elNum);
+
+  dictEl.levelFreqs = NULL;
+  terms.push_back(dictEl);
+
+  while (curElNum <= lastElNum ) { 
+    dictEl.SetNumLevels(numLevels);
+    dictEl.Read(dictFile, numLevels);
+    dictEl.levelFreqs = NULL;
+    terms.push_back(dictEl);
+    curElNum++;
+  }
+
+  return true;
+}
+
+
+
+
+
+
+
+
+

trunk/gsdl/src/mgpp/text/invf.h

@@ -158 +158 @@
 };
 
-
+typedef vector<word_block_dict_el> word_block_dict_el_array;
 
 struct block_idx_info {
@@ -302 +302 @@
                 unsigned long &elNum);
 
-
+//----------------------------------------------------------
+
+// new functions for full text browse
+
+bool NearestSearchWordBlockDictEl (FILE *dictFile,
+                const block_idx &bIdx,
+                unsigned long entriesPerBlock,
+                unsigned long dictSize,
+                unsigned long numLevels,
+                const UCArray &el,
+                word_block_dict_el &dictEl,
+                unsigned long &elNum);
+
+// returns a list of word_block_dict_el, with no levelfreqs
+bool SearchWordBlockDictElNumRange (FILE *dictFile,
+                    const block_idx &bIdx,
+                    unsigned long entriesPerBlock,
+                    unsigned long dictSize,
+                    unsigned long numLevels,
+                    unsigned long elNum,
+                    unsigned long numWords,
+                    word_block_dict_el_array &terms);
+                    
+// just returns a list of terms
+bool SearchWordBlockDictElNumRange (FILE *dictFile,
+                    const block_idx &bIdx,
+                    unsigned long entriesPerBlock,
+                    unsigned long dictSize,
+                    unsigned long numLevels,
+                    unsigned long elNum,
+                    unsigned long numWords,
+                    UCArrayVector &terms);
+