Changeset 31244 for other-projects/hathitrust/wcsa/extracted-features-solr

Timestamp:

2016-12-18T16:57:05+13:00 (7 years ago)

Author:

davidb

Message:

Tidy up

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java (modified) (4 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java

@@ -16 +16 @@
 import org.apache.commons.compress.compressors.CompressorException;
 import org.json.JSONObject;
+import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.icu.segmentation.ICUTokenizer;
@@ -40 +41 @@
                     tokenizer.setReader(reader);
                         
-                    //TokenStream tokenStream = analyzer.tokenStream(fieldName, reader);
-                    //OffsetAttribute offsetAttribute = tokenizer.addAttribute(OffsetAttribute.class);
                     CharTermAttribute charTermAttribute = tokenizer.addAttribute(CharTermAttribute.class);
 
@@ -48 +47 @@
                         
                         while (tokenizer.incrementToken()) {
-                            //int startOffset = offsetAttribute.startOffset();
-                            //int endOffset = offsetAttribute.endOffset();
                             String term = charTermAttribute.toString();
                             tokens.add(term);
                         }
                         
+                        tokenizer.end();
                         tokenizer.close();
                     } 
@@ -107 +105 @@
             }
     */
-            
-            
             
            return sb.toString();