source: trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/extractor/IndexExtractor.java@ 13242

package org.greenstone.gsdl3.gs3build.extractor;

import java.io.FileReader;

import java.net.URL;

import java.util.List;
import java.util.ArrayList;
import java.util.Iterator;
import org.greenstone.gsdl3.gs3build.util.GS2TextFileHandler;

import org.xml.sax.XMLReader;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.Attributes;
import org.xml.sax.helpers.XMLReaderFactory;
import org.xml.sax.helpers.DefaultHandler;

import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
import org.greenstone.gsdl3.gs3build.doctypes.IndexDocument;
import org.greenstone.gsdl3.gs3build.doctypes.DocumentLoader;
import org.greenstone.gsdl3.gs3build.doctypes.DocumentList;

import org.greenstone.gsdl3.gs3build.metadata.MetadataLabel;

public class IndexExtractor implements ExtractorInterface
{
  class IndexHandlerException extends Exception
  { public IndexHandlerException(String value)
    { super(value);
    }
  }

  /**
   *  An inner class to handle GML files
   */
  class IndexHandler extends GS2TextFileHandler
  { List    labels;
    URL     base;

    IndexHandler(String content, URL url, DocumentList documentList) throws IndexHandlerException
    { 
      super(content);

      this.labels  = new ArrayList();
      this.base    = url;

      String parentDir;
      int leaf = this.base.toString().lastIndexOf('/');
      if (leaf >= 0) {
    parentDir = this.base.toString().substring(0, leaf+1);
      }
      else {
    parentDir = this.base.toString();
      }

      // get the first line
      this.getLine();

      if (!this.hasMore())
      { throw new IndexHandlerException("No title line");
      }
      
      // get the first totem - it should be "key:"
      String entry = this.getEntry(true);
      
      // now get all the labels
      while (this.hasMore())
      { String label = this.getEntry(true);
        if (label == null || label.length() == 0) {
      continue;
    }
    
    this.labels.add(label);
    System.out.println("Adding label: " + label);
      }

      while (this.hasMoreLines()) {
    this.getLine();

    // Get the file pattern itself
    String filePattern = this.getEntry(true);
    if (filePattern == null || filePattern.length() == 0) {
      continue;
    }

    // get a list of documents that match the file pattern
    List documentIds = documentList.findDocumentIdsUsingFile(filePattern);
    if (documentIds != null) {
      Iterator iterator = documentIds.iterator();
      while (iterator.hasNext()) {
        System.out.println("Matches file " + iterator.next().toString());
      }
    }

    // if no files match this data, then skip this row
    // TODO: raise a quality error message
    if (documentIds == null || documentIds.size() == 0) {
      continue;
    }

    // cache up the documents that match for speed improvements...
    List documents = new ArrayList();
    Iterator idIterator = documentIds.iterator();
    while (idIterator.hasNext()) {
      String docIdString = idIterator.next().toString();
      System.out.println(docIdString);
      DocumentID docId   = new DocumentID(docIdString);
      DocumentInterface document = documentList.getDocument(docId);
      if (document != null) {
        documents.add(document);
      }
    }

    // Next, split the row into the separate metadata items
    int entryNo = 0;
    while (this.hasMore()) {
      String item = this.getEntry(true);
      if (item == null || item.length() == 0) {
        entryNo ++;
        continue;
      }

      String label = null;
      if (item.startsWith("<")) {
        int labelEnd = item.indexOf('>');
        if (labelEnd >= 0) {
          label = item.substring(1, labelEnd);

          item = item.substring(labelEnd+1, item.length());

          // eliminate any weird whitespace
          item.trim();

          // cope with a solo 'item' label with no following string
          if (item.length() == 0) {
        entryNo ++;
        continue;
          }
        }
        // starts with a bracketed label
      }
      else if (entryNo < this.labels.size()) {
        label = (String) this.labels.get(entryNo);
      }

      // Actually post the metadata -
      // it may be good to have cached all the documents that we're going to change
      // in order to minimise rewrites...
      if (label != null) {
        Iterator docIterator = documents.iterator();
        while (docIterator.hasNext()) {
          DocumentInterface document = (DocumentInterface) docIterator.next();

          // Post to document
          // TODO: tailor this to posting documents to *sections* as required...
          document.addDocumentMetadata(new MetadataLabel(label), item);
          System.out.println("Posting " + label + "=" + item + " to " + parentDir + filePattern);
        }
      }
      entryNo ++;
    }
    
    // write out the modified documents
    // TODO: nicer/more generalised interface for this and related activity in
    //       extractor manager (actually, enricher manager);
    Iterator docIterator = documents.iterator();
    while (docIterator.hasNext()) {
      DocumentInterface document = (DocumentInterface) docIterator.next();

      System.out.println("Writing modified document " + document.getID());
      documentList.storeChangedDocument(document);
    }
      }
    }

  }

  private DocumentList documentList;

  /**
   *  Construct of extractor
   */
  public IndexExtractor()
  { // Intentionally left blank
  }

  /**
   *  This extractor doesn't need to do any preparation/completion work,
   *  so this member function is empty.
   */
  public void configure(String outputDir)
  { // Intentionally left blank
  }

  public void configure(DocumentList list)
  { this.documentList = list;
  }

  /**
   *  This extractor doesn't need to do any preparation/completion work,
   *  so this member function is empty.
   */
  public void startPass(int passNo)
  { // Intentionally left blank
  }

  /**
   *  Process the document - for a GML document, this results in the
   *  decoration of other files, for other documents, it does nothing.
   */
  public void extractDocument(DocumentID docID, DocumentInterface document)
  { if (document.getDocumentType().equals(IndexDocument.INDEX_DOCUMENT_TYPE))
    { // Extract the content from the index file
      
      // get the file
      String documentText = 
    DocumentLoader.getAsString(document.getDocumentFiles().getFile(0).getURL());
    
      if (documentText == null) {
    System.err.println("IndexExtractor: Unable to load any content for " + document.getDocumentFiles().getFile(0).toString());
    return;
      }

      try {
    IndexHandler handler = new IndexHandler(documentText, document.getDocumentFiles().getFile(0).getURL(), this.documentList);
      }
      catch (IndexHandlerException ex) {
      }
      
      // for each document post it to the corresponding document
    }
  }

  protected static void postMetadata(String file, String value, String label)
  {
  }

  /**
   *  This extractor doesn't need to do any preparation/completion work,
   *  so this member function is empty.
   */
  public void endPass(int passNo)
  { // Intentionally left blank
  }

  /**
   *  This extractor is a simple, single-pass extractor
   *
   *  @see: org.greenstone.gsdl3.gs3build.extractor.ExtractorInterface:getNumberOfPasses
   */
  public int getNumberOfPasses()
  { return 1;
  }
}