Changeset 8422

Timestamp:

2004-10-31T16:30:46+13:00 (19 years ago)

Author:

schweer

Message:

George's bugfixes for adding documents to an existing collection; some changes for detecting deleted documents

Location:

trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build

Files:

• CollectionManager.java (modified) (11 diffs)
• doctypes/DocumentList.java (modified) (2 diffs)
• doctypes/TextRecogniser.java (modified) (1 diff)
• indexers/IndexerManager.java (modified) (2 diffs)
• util/GS3SQLConnection.java (modified) (5 diffs)

trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/CollectionManager.java

@@ -19 +19 @@
 import java.net.URLEncoder;
 
+import java.sql.*;
+
 import javax.xml.parsers.*;
 
@@ -31 +33 @@
 import org.xml.sax.SAXParseException;
 
+import org.greenstone.gsdl3.gs3build.database.*;
 import org.greenstone.gsdl3.gs3build.collection.*;
 import org.greenstone.gsdl3.gs3build.classifier.*;
@@ -124 +127 @@
     return null;
       }
-      System.out.println(this.type.toLowerCase());
+      System.out.println("Creating a classifier of type " + this.type.toLowerCase());
       
       if (this.type.toLowerCase().equals("hierarchy")) {
-    System.out.println(this.file);
+    System.out.println("  hierarchy file is " + this.file);
     classifier = new HierarchyClassifier(this.className, this.file, this.fields, this.sort);
       }
@@ -197 +200 @@
 
     this.buildDocNo = 1;
+
+    try {
+      GS3SQLSelect select = new GS3SQLSelect("build");
+      select.addField("*");
+      this.database.execute(select.toString());
+      ResultSet results = this.database.getResultSet();
+      if (results != null && 
+      results.first()) {
+    System.out.println("Reading all keys");
+    do {
+      String key = results.getString("buildKey");
+      String value = results.getString("buildValue");
+
+      if (key.equals("NextSeqNo")) {
+        this.buildDocNo = Integer.parseInt(value);
+      }
+      else if (key.equals("lastBuildDate")) {
+        int year, month, day;
+        year = Integer.parseInt(value.substring(0, 4));
+        month = Integer.parseInt(value.substring(4, 6)) - 1; // -1 because Gregorian Calendar perversely treats January as 0, etc.
+        day = Integer.parseInt(value.substring(6, 8));
+        
+        this.lastBuildDate = new GregorianCalendar(year, month, day);
+      }
+    } while (results.next());
+      }
+    }
+    catch (SQLException ex)
+      { System.out.println(ex);
+      }
   }
 
@@ -249 +282 @@
   public void configureCollection()
   { File collectionConfig = new File(GSFile.collectionConfigFile(this.collectionHome));
-    
+
     // get the File and read it in
     try
@@ -360 +393 @@
 
   public String getEtcDirectory()
-  
   { return GSFile.collectionEtcDir(this.collectionHome);
   }
@@ -370 +402 @@
   public String getBuildDirectory()
   { return GSFile.collectionBuildDir(this.collectionHome);
-  }
-  
-  public GregorianCalendar getLastBuildDate(){
-    return lastBuildDate;
   }
 
@@ -400 +428 @@
       { this.buildDocNo = 1;
       }
+      else 
+      { System.out.println("Continuing build sequence from " + this.buildDocNo);
+      }
     }
     this.lastBuildDate = today;
@@ -406 +437 @@
   public void endBuild()
   {
-      // here we write out the build config file
-      // create the xml for the buildConfig
-      XMLConverter converter = new XMLConverter();
-      Document doc = converter.newDOM();
-      Element build_config = doc.createElement(GSXML.COLLECTION_BUILD_ELEM);
-      Element meta_list = doc.createElement(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
-      build_config.appendChild(meta_list);
-      Element service_list = doc.createElement(GSXML.SERVICE_CLASS_ELEM+GSXML.LIST_MODIFIER);
-      build_config.appendChild(service_list);
-
-      // the document structure and metadata retrieval will use GS3REtrieve service, so add it in here
-      Element base_retrieve_service = doc.createElement(GSXML.SERVICE_CLASS_ELEM);
-      base_retrieve_service.setAttribute(GSXML.NAME_ATT, "GS3Retrieve");
-      service_list.appendChild(base_retrieve_service);
-      // ask the indexers to add stuff into the service rack list
-      this.buildManager.getIndexerManager().addServiceDescriptions(service_list);
-      this.buildManager.getClassifierManager().addServiceDescriptions(service_list);
-      // get the String
-      String build_config_string = converter.getString(build_config);
-      // write it to the file
-      try {
-      File build_config_file = new File(GSFile.collectionBuildConfigFileBuilding(this.collectionHome));
-      BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(build_config_file), "UTF-8"));
-      writer.write(build_config_string, 0, build_config_string.length());
-      writer.close();
-      } catch (Exception e) {
-      System.err.println("CollectionManager.endBuild() Error while trying to output the buildConfig.xml file.");
-      System.err.println(e.getMessage());
-      }
+    // here we write out the build config file
+    // create the xml for the buildConfig
+    XMLConverter converter = new XMLConverter();
+    Document doc = converter.newDOM();
+    Element build_config = doc.createElement(GSXML.COLLECTION_BUILD_ELEM);
+    Element meta_list = doc.createElement(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
+    build_config.appendChild(meta_list);
+    Element service_list = doc.createElement(GSXML.SERVICE_CLASS_ELEM+GSXML.LIST_MODIFIER);
+    build_config.appendChild(service_list);
+    
+    // the document structure and metadata retrieval will use GS3REtrieve service, so add it in here
+    Element base_retrieve_service = doc.createElement(GSXML.SERVICE_CLASS_ELEM);
+    base_retrieve_service.setAttribute(GSXML.NAME_ATT, "GS3Retrieve");
+    service_list.appendChild(base_retrieve_service);
+    // ask the indexers to add stuff into the service rack list
+    this.buildManager.getIndexerManager().addServiceDescriptions(service_list);
+    this.buildManager.getClassifierManager().addServiceDescriptions(service_list);
+    // get the String
+    String build_config_string = converter.getString(build_config);
+    // write it to the file
+    try {
+      File build_config_file = new File(GSFile.collectionBuildConfigFileBuilding(this.collectionHome));
+      BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(build_config_file), "UTF-8"));
+      writer.write(build_config_string, 0, build_config_string.length());
+      writer.close();
+    } catch (Exception e) {
+      System.err.println("CollectionManager.endBuild() Error while trying to output the buildConfig.xml file.");
+      System.err.println(e.getMessage());
+    }
+    
+    // Update build date information
+    GS3SQLDelete remove = new GS3SQLDelete("build");
+    //    GS3SQLWhere where = new GS3SQLWhere(new GS3SQLWhereItem("buildKey", "=", "NextSeqNo"));
+    //    rem
+    this.database.execute(remove.toString());
+    
+    GS3SQLInsert insert = new GS3SQLInsert("build");
+    insert.addValue("buildKey", "NextSeqNo");
+    insert.addValue("buildValue", Integer.toString(this.buildDocNo));
+    this.database.execute(insert.toString());
+    
+    insert = new GS3SQLInsert("build");
+    insert.addValue("buildKey", "lastBuildDate");
+    insert.addValue("buildValue", getDateString(this.lastBuildDate));
+    this.database.execute(insert.toString());
+
+    // Do tail of build output
     Date startDate = this.lastBuildDate.getTime();
     Date date = new Date();
@@ -449 +498 @@
   }
 
-  public String getNextDocumentID()
-  { StringBuffer ID = new StringBuffer();
-
+  private static String getDateString(GregorianCalendar date)
+  { StringBuffer dateString = new StringBuffer();
+  
     int value;
-    ID.append(lastBuildDate.get(Calendar.YEAR));
+    dateString.append(date.get(Calendar.YEAR));
 
     // the use of month is a little odd, hence the following
@@ -460 +509 @@
     // it into January = 1, etc., and the padding is altered
     // correspondingly.
-    value = lastBuildDate.get(Calendar.MONTH);
+    value = date.get(Calendar.MONTH);
     if (value < 9)
-    { ID.append("0");
-    }
-    ID.append(value + 1);
-    value = lastBuildDate.get(Calendar.DAY_OF_MONTH);
+    { dateString.append("0");
+    }
+    dateString.append(value + 1);
+    value = date.get(Calendar.DAY_OF_MONTH);
     if (value < 10)
-      ID.append("0");
-    ID.append(value);
-
-
-    value = this.buildDocNo;
+      dateString.append("0");
+    dateString.append(value);
+
+    return dateString.toString();
+  }
+
+  public String getNextDocumentID()
+  { StringBuffer ID = new StringBuffer(getDateString(this.lastBuildDate));
+
+    int value = this.buildDocNo;
     this.buildDocNo ++;
     

trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentList.java

@@ -321 +321 @@
     // set the document identifier, if not already set
     if (document.getID() == null) {
+      System.out.println("Posting new docuument ID");
       DocumentID id = this.idFactory.getNewDocumentID(document);
       document.setID(id);
@@ -327 +328 @@
     // add to the database as well, if it is modified...
     if (document.isChanged()) {
+      System.out.println("Document was changed");
       document.getSQLWriter().writeDocument(document, this.connection);
     }

trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/TextRecogniser.java

@@ -38 +38 @@
       { this.listRepository.addDocument(new TextDocument(url));
     // TODO: spawn knowledge of children too...
-        System.out.println(">>> Posting text document " + fileName);
+        System.out.println("Posting text document " + fileName);
     return true;
       }

trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/indexers/IndexerManager.java

@@ -47 +47 @@
     { for (int p = 0; p < this.indexers[i].getNumberOfPasses(); p ++)
       { 
-      if (!this.indexers[i].startPass(p)) {
-          continue;
-      }
+    if (!this.indexers[i].startPass(p)) {
+      continue;
+    }
+
     Iterator iterator = this.documentList.iterator();
 
@@ -62 +63 @@
         // note any changes made to this document...
         if (document.isChanged()) {
+          //System.out.println("Writing document "+document.getID());
           this.documentList.storeChangedDocument(document);
-          //          System.out.println("Writing document "+document.getID());
         }
         /**

trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/GS3SQLConnection.java

@@ -9 +9 @@
 import java.sql.Statement;
 import java.sql.ResultSet;
-
 
 import org.greenstone.gsdl3.gs3build.database.*;
@@ -193 +192 @@
       this.connection = GS3SQLConnectionFactory.reconnect(collectionName);
 
+      statement = this.connection.createStatement();
+
+      // create build history table
+      GS3SQLCreateTable buildTable = new GS3SQLCreateTable("build");
+      buildTable.addAutoPrimaryKey("buildRef");
+      buildTable.addProperty("buildKey", 64);
+      buildTable.addProperty("buildValue");
+      statement.execute(buildTable.toString());
+
       // create document table
       GS3SQLCreateTable docTable = new GS3SQLCreateTable("document");
@@ -202 +210 @@
       docTable.addProperty("IndexedDate", GS3SQLField.DATETIME_TYPE);
       docTable.addProperty("ModifiedDate", GS3SQLField.DATETIME_TYPE);
-      statement = this.connection.createStatement();
       statement.execute(docTable.toString());
       
-      //   create structure table
+      // create structure table
       GS3SQLCreateTable structureMap = new GS3SQLCreateTable("structure");
       structureMap.addAutoPrimaryKey("StructureRef");
@@ -215 +222 @@
       statement.execute(structureMap.toString());
     
-      //   create section table
+      // create section table
       GS3SQLCreateTable sectionMap = new GS3SQLCreateTable("divisions");
       sectionMap.addAutoPrimaryKey("DivisionRef");
@@ -266 +273 @@
       namespaces.addProperty("Creator", 128);
       statement.execute(namespaces.toString());
-      
-      //   create metadata values table
+
+      // create metadata grouping table
+      GS3SQLCreateTable metagroups = new GS3SQLCreateTable("mdgroups");
+      metagroups.addProperty("NamespaceRef", GS3SQLField.INTEGER_TYPE);
+      metagroups.addAutoPrimaryKey("MdGroupRef");
+      metagroups.addProperty("MdGroupID", 32);
+      statement.execute(metagroups.toString());
+
+      // create metadata qualifiers table
+      GS3SQLCreateTable metaqualifiers = new GS3SQLCreateTable("mdqualifiers");
+      metaqualifiers.addProperty("MdGroupRef", GS3SQLField.INTEGER_TYPE);
+      metaqualifiers.addProperty("QualifierName", 64);
+      metaqualifiers.addProperty("QualifierValue", 64);
+      statement.execute(metaqualifiers.toString());
+      
+      // create metadata values table
       GS3SQLCreateTable metadata = new GS3SQLCreateTable("mdvalues");
-      // todo: some unique id for the namespaces item
-      metadata.addProperty("NamespaceRef", GS3SQLField.INTEGER_TYPE);
-      metadata.addProperty("Label", 256);
-      metadata.addProperty("Value");
+      metadata.addProperty("NamespaceRef", GS3SQLField.INTEGER_TYPE); // the parent namespace {one candidate primary key}
+      metadata.addProperty("GroupRef", GS3SQLField.INTEGER_TYPE);     // the parent group     {a sub-key or independent primary key}
+      metadata.addProperty("Label", 256);                             // the key of the metadata item
+      metadata.addProperty("Value");                                  // the value of the metadata item
       statement.execute(metadata.toString());