Changeset 26214

Timestamp:

2012-09-19T12:06:28+12:00 (12 years ago)

Author:

jmt12

Message:

New hash based generation for associated files directory - so docno is no longer essential

Location:

gs2-extensions/video-and-audio/trunk/src/opt/Terrier

Files:

• ImageDocument.java (modified) (3 diffs)
• VideoDocument.java (modified) (3 diffs)

gs2-extensions/video-and-audio/trunk/src/opt/Terrier/ImageDocument.java

@@ -20 +20 @@
 import java.io.StringReader;
 import java.io.Reader;
+import java.nio.charset.Charset;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
 import java.util.Collections;
 import java.util.Arrays;
@@ -79 +82 @@
     this.properties.put("abstract", "This is an image so here is some dummy text to prevent indexer failing.");
     // B. Properties derived from filename
-    String title = this.properties.get("filename").substring(properties.get("filename").lastIndexOf(System.getProperty("file.separator")) + 1);
+    // - A simple title for the document
+    String filepath = this.properties.get("filename");
+    String title = filepath.substring(filepath.lastIndexOf(System.getProperty("file.separator")) + 1);
     this.properties.put("title", title);
-    String ext = this.properties.get("filename").substring(properties.get("filename").lastIndexOf(".") + 1);
+    String ext = filepath.substring(filepath.lastIndexOf(".") + 1);
+    // - The name of the copy of the original document
     String target_filename = "doc." + ext;
     this.properties.put("source","doc." + ext);
-    String assoc_filename = "D" + this.properties.get("docno");
-    if (assoc_filename.equals("Dnull"))
-    {
-      System.err.println("Error! Bogus assoc dir: " + this.properties.get("docno"));
+    // - A unique associated directory. This gets a little tricky as we need
+    //   to create the directory at the same time if an effort to promote
+    //   synchronous behaviour
+    String unique_id = this.generateHash(filepath);
+    //   - we start with the first 4 characters
+    int offset = 0;
+    String assoc_filename = "D" + unique_id.substring(offset, offset + 4);
+    //   - we add ".dir" as a suffix to the directory that actually contains
+    //     files (so the non-suffixed version contains nested directories)
+    Path assoc_path = Paths.get(ApplicationSetup.TERRIER_SHARE, "images", "assoc", assoc_filename + ".dir");
+    //   - then we continue adding blocks of 4 characters until we get a
+    //     directory that doesn't already exist
+    while (assoc_path.toFile().exists() && offset < unique_id.length())
+    {
+      offset += 4;
+      assoc_filename += System.getProperty("file.separator") + unique_id.substring(offset, offset + 4);
+      assoc_path = Paths.get(ApplicationSetup.TERRIER_SHARE, "images", "assoc", assoc_filename + ".dir");
+    }
+    //   - still not unique? but run out of unique_id... time to complain
+    if (assoc_path.toFile().exists())
+    {
+      logger.error("ImageDoument - can't determine unique assocfilepath");
       System.exit(0);
     }
-
+    //   - create the directories quick... hopefully before someone else does
+    assoc_path.toFile().mkdirs(); // bet there is a nice nio way to do this
     this.properties.put("assocfile", assoc_filename);
 
     // Copy (symlink) the file into place in the shared directory
     Path source_path = Paths.get(properties.get("filename"));
-    Path assoc_path = Paths.get(ApplicationSetup.TERRIER_SHARE, "images", "assoc", assoc_filename);
-    assoc_path.toFile().mkdirs(); // bet there is a nice nio way to do this
     Path target_path = assoc_path.resolve(target_filename);
     if (target_path.toFile().exists())
@@ -264 +287 @@
   }
   /** getReader() **/
+
+  /**
+   */
+  private String generateHash(String string)
+  {
+    StringBuffer sb = new StringBuffer();
+    try
+    {
+      final MessageDigest message_digest = MessageDigest.getInstance("MD5");
+      message_digest.reset();
+      message_digest.update(string.getBytes(Charset.forName("UTF8")));
+      final byte[] result_bytes = message_digest.digest();
+      for (int i = 0; i < result_bytes.length; ++i)
+      {
+        sb.append(Integer.toHexString((result_bytes[i] & 0xFF) | 0x100).substring(1,3));
+      }
+    }
+    catch (NoSuchAlgorithmException e)
+    {
+      System.err.println("Exception: " + e);
+      System.exit(0);
+    }
+    return sb.toString();
+  }
+  /** generateHash(String) **/
 }
 

gs2-extensions/video-and-audio/trunk/src/opt/Terrier/VideoDocument.java

@@ -24 +24 @@
 import java.io.StringReader;
 import java.io.Reader;
-import java.lang.Thread;
+import java.nio.charset.Charset;
 import java.nio.file.Files;
-import java.nio.file.FileVisitResult;
-import static java.nio.file.FileVisitResult.*;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
 import java.util.Collections;
 import java.util.Arrays;
@@ -88 +88 @@
     this.properties.put("abstract", "This is a video so here is some dummy text to prevent indexer failing.");
     // B. Properties derived from filename
-    String title = this.properties.get("filename").substring(properties.get("filename").lastIndexOf(System.getProperty("file.separator")) + 1);
+    String filepath = this.properties.get("filename");
+    String title = filepath.substring(filepath.lastIndexOf(System.getProperty("file.separator")) + 1);
     this.properties.put("title", title);
-    String ext = this.properties.get("filename").substring(properties.get("filename").lastIndexOf(".") + 1);
+    String ext = filepath.substring(filepath.lastIndexOf(".") + 1);
     String target_filename = "doc." + ext;
     this.properties.put("source","doc." + ext);
-    String assoc_filename = "D" + properties.get("docno");
+    // - A unique associated directory. This gets a little tricky as we need
+    //   to create the directory at the same time if an effort to promote
+    //   synchronous behaviour
+    String unique_id = this.generateHash(filepath);
+    //   - we start with the first 4 characters
+    int offset = 0;
+    String assoc_filename = "D" + unique_id.substring(offset, offset + 4);
+    //   - we add ".dir" as a suffix to the directory that actually contains
+    //     files (so the non-suffixed version contains nested directories)
+    Path assoc_path = Paths.get(ApplicationSetup.TERRIER_SHARE, "images", "assoc", assoc_filename + ".dir");
+    //   - then we continue adding blocks of 4 characters until we get a
+    //     directory that doesn't already exist
+    while (assoc_path.toFile().exists() && offset < unique_id.length())
+    {
+      offset += 4;
+      assoc_filename += System.getProperty("file.separator") + unique_id.substring(offset, offset + 4);
+      assoc_path = Paths.get(ApplicationSetup.TERRIER_SHARE, "images", "assoc", assoc_filename + ".dir");
+    }
+    //   - still not unique? but run out of unique_id... time to complain
+    if (assoc_path.toFile().exists())
+    {
+      logger.error("ImageDoument - can't determine unique assocfilepath");
+      System.exit(0);
+    }
+    //   - create the directories quick... hopefully before someone else does
+    assoc_path.toFile().mkdirs(); // bet there is a nice nio way to do this
     this.properties.put("assocfile", assoc_filename);
 
     // Copy (symlink) the file into place in the shared directory
     Path raw_video_path = Paths.get(properties.get("filename"));
-    Path assoc_path = Paths.get(ApplicationSetup.TERRIER_SHARE, "images", "assoc", assoc_filename);
-    // - if the assoc path already exists, we need to recursively delete it and
-    //   its contents
-    if (Files.exists(assoc_path))
-    {
-      logger.info("VideoDocument - removing existing (old) associated files");
-      try
-      {
-        Files.walkFileTree(assoc_path, new SimpleFileVisitor<Path>()
-        {
-          @Override
-          public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
-            throws IOException
-          {
-            ///ystem.out.println("Deleting file: " + file);
-            Files.delete(file);
-            return CONTINUE;
-          }
-          @Override
-          public FileVisitResult postVisitDirectory(Path dir, IOException exc)
-            throws IOException
-          {
-            ///ystem.out.println("Deleting dir: " + dir);
-            if (exc == null)
-            {
-              Files.delete(dir);
-              return CONTINUE;
-            }
-            else
-            {
-              throw exc;
-            }
-          }
-        });
-      }
-      catch (Exception e)
-      {
-        logger.error("Exception while recursively deleting assoc folder:", e);
-      }
-    }
-    assoc_path.toFile().mkdirs(); // bet there is a nice nio way to do this
     Path target_path = assoc_path.resolve(target_filename);
     logger.info("VideoDocument - symlinking original video into assoc directory");
@@ -389 +374 @@
   }
   /** getReader() **/
+
+  /**
+   */
+  private String generateHash(String string)
+  {
+    StringBuffer sb = new StringBuffer();
+    try
+    {
+      final MessageDigest message_digest = MessageDigest.getInstance("MD5");
+      message_digest.reset();
+      message_digest.update(string.getBytes(Charset.forName("UTF8")));
+      final byte[] result_bytes = message_digest.digest();
+      for (int i = 0; i < result_bytes.length; ++i)
+      {
+        sb.append(Integer.toHexString((result_bytes[i] & 0xFF) | 0x100).substring(1,3));
+      }
+    }
+    catch (NoSuchAlgorithmException e)
+    {
+      System.err.println("Exception: " + e);
+      System.exit(0);
+    }
+    return sb.toString();
+  }
+  /** generateHash(String) **/
 }