Changeset 26214 for gs2-extensions
- Timestamp:
- 2012-09-19T12:06:28+12:00 (12 years ago)
- Location:
- gs2-extensions/video-and-audio/trunk/src/opt/Terrier
- Files:
-
- 2 edited
Legend:
- Unmodified
- Added
- Removed
-
gs2-extensions/video-and-audio/trunk/src/opt/Terrier/ImageDocument.java
r26208 r26214 20 20 import java.io.StringReader; 21 21 import java.io.Reader; 22 import java.nio.charset.Charset; 22 23 import java.nio.file.Files; 23 24 import java.nio.file.Path; 24 25 import java.nio.file.Paths; 26 import java.security.MessageDigest; 27 import java.security.NoSuchAlgorithmException; 25 28 import java.util.Collections; 26 29 import java.util.Arrays; … … 79 82 this.properties.put("abstract", "This is an image so here is some dummy text to prevent indexer failing."); 80 83 // B. Properties derived from filename 81 String title = this.properties.get("filename").substring(properties.get("filename").lastIndexOf(System.getProperty("file.separator")) + 1); 84 // - A simple title for the document 85 String filepath = this.properties.get("filename"); 86 String title = filepath.substring(filepath.lastIndexOf(System.getProperty("file.separator")) + 1); 82 87 this.properties.put("title", title); 83 String ext = this.properties.get("filename").substring(properties.get("filename").lastIndexOf(".") + 1); 88 String ext = filepath.substring(filepath.lastIndexOf(".") + 1); 89 // - The name of the copy of the original document 84 90 String target_filename = "doc." + ext; 85 91 this.properties.put("source","doc." + ext); 86 String assoc_filename = "D" + this.properties.get("docno"); 87 if (assoc_filename.equals("Dnull")) 88 { 89 System.err.println("Error! Bogus assoc dir: " + this.properties.get("docno")); 92 // - A unique associated directory. This gets a little tricky as we need 93 // to create the directory at the same time if an effort to promote 94 // synchronous behaviour 95 String unique_id = this.generateHash(filepath); 96 // - we start with the first 4 characters 97 int offset = 0; 98 String assoc_filename = "D" + unique_id.substring(offset, offset + 4); 99 // - we add ".dir" as a suffix to the directory that actually contains 100 // files (so the non-suffixed version contains nested directories) 101 Path assoc_path = Paths.get(ApplicationSetup.TERRIER_SHARE, "images", "assoc", assoc_filename + ".dir"); 102 // - then we continue adding blocks of 4 characters until we get a 103 // directory that doesn't already exist 104 while (assoc_path.toFile().exists() && offset < unique_id.length()) 105 { 106 offset += 4; 107 assoc_filename += System.getProperty("file.separator") + unique_id.substring(offset, offset + 4); 108 assoc_path = Paths.get(ApplicationSetup.TERRIER_SHARE, "images", "assoc", assoc_filename + ".dir"); 109 } 110 // - still not unique? but run out of unique_id... time to complain 111 if (assoc_path.toFile().exists()) 112 { 113 logger.error("ImageDoument - can't determine unique assocfilepath"); 90 114 System.exit(0); 91 115 } 92 116 // - create the directories quick... hopefully before someone else does 117 assoc_path.toFile().mkdirs(); // bet there is a nice nio way to do this 93 118 this.properties.put("assocfile", assoc_filename); 94 119 95 120 // Copy (symlink) the file into place in the shared directory 96 121 Path source_path = Paths.get(properties.get("filename")); 97 Path assoc_path = Paths.get(ApplicationSetup.TERRIER_SHARE, "images", "assoc", assoc_filename);98 assoc_path.toFile().mkdirs(); // bet there is a nice nio way to do this99 122 Path target_path = assoc_path.resolve(target_filename); 100 123 if (target_path.toFile().exists()) … … 264 287 } 265 288 /** getReader() **/ 289 290 /** 291 */ 292 private String generateHash(String string) 293 { 294 StringBuffer sb = new StringBuffer(); 295 try 296 { 297 final MessageDigest message_digest = MessageDigest.getInstance("MD5"); 298 message_digest.reset(); 299 message_digest.update(string.getBytes(Charset.forName("UTF8"))); 300 final byte[] result_bytes = message_digest.digest(); 301 for (int i = 0; i < result_bytes.length; ++i) 302 { 303 sb.append(Integer.toHexString((result_bytes[i] & 0xFF) | 0x100).substring(1,3)); 304 } 305 } 306 catch (NoSuchAlgorithmException e) 307 { 308 System.err.println("Exception: " + e); 309 System.exit(0); 310 } 311 return sb.toString(); 312 } 313 /** generateHash(String) **/ 266 314 } 267 315 -
gs2-extensions/video-and-audio/trunk/src/opt/Terrier/VideoDocument.java
r26190 r26214 24 24 import java.io.StringReader; 25 25 import java.io.Reader; 26 import java. lang.Thread;26 import java.nio.charset.Charset; 27 27 import java.nio.file.Files; 28 import java.nio.file.FileVisitResult;29 import static java.nio.file.FileVisitResult.*;30 28 import java.nio.file.Path; 31 29 import java.nio.file.Paths; 32 30 import java.nio.file.SimpleFileVisitor; 33 31 import java.nio.file.attribute.BasicFileAttributes; 32 import java.security.MessageDigest; 33 import java.security.NoSuchAlgorithmException; 34 34 import java.util.Collections; 35 35 import java.util.Arrays; … … 88 88 this.properties.put("abstract", "This is a video so here is some dummy text to prevent indexer failing."); 89 89 // B. Properties derived from filename 90 String title = this.properties.get("filename").substring(properties.get("filename").lastIndexOf(System.getProperty("file.separator")) + 1); 90 String filepath = this.properties.get("filename"); 91 String title = filepath.substring(filepath.lastIndexOf(System.getProperty("file.separator")) + 1); 91 92 this.properties.put("title", title); 92 String ext = this.properties.get("filename").substring(properties.get("filename").lastIndexOf(".") + 1);93 String ext = filepath.substring(filepath.lastIndexOf(".") + 1); 93 94 String target_filename = "doc." + ext; 94 95 this.properties.put("source","doc." + ext); 95 String assoc_filename = "D" + properties.get("docno"); 96 // - A unique associated directory. This gets a little tricky as we need 97 // to create the directory at the same time if an effort to promote 98 // synchronous behaviour 99 String unique_id = this.generateHash(filepath); 100 // - we start with the first 4 characters 101 int offset = 0; 102 String assoc_filename = "D" + unique_id.substring(offset, offset + 4); 103 // - we add ".dir" as a suffix to the directory that actually contains 104 // files (so the non-suffixed version contains nested directories) 105 Path assoc_path = Paths.get(ApplicationSetup.TERRIER_SHARE, "images", "assoc", assoc_filename + ".dir"); 106 // - then we continue adding blocks of 4 characters until we get a 107 // directory that doesn't already exist 108 while (assoc_path.toFile().exists() && offset < unique_id.length()) 109 { 110 offset += 4; 111 assoc_filename += System.getProperty("file.separator") + unique_id.substring(offset, offset + 4); 112 assoc_path = Paths.get(ApplicationSetup.TERRIER_SHARE, "images", "assoc", assoc_filename + ".dir"); 113 } 114 // - still not unique? but run out of unique_id... time to complain 115 if (assoc_path.toFile().exists()) 116 { 117 logger.error("ImageDoument - can't determine unique assocfilepath"); 118 System.exit(0); 119 } 120 // - create the directories quick... hopefully before someone else does 121 assoc_path.toFile().mkdirs(); // bet there is a nice nio way to do this 96 122 this.properties.put("assocfile", assoc_filename); 97 123 98 124 // Copy (symlink) the file into place in the shared directory 99 125 Path raw_video_path = Paths.get(properties.get("filename")); 100 Path assoc_path = Paths.get(ApplicationSetup.TERRIER_SHARE, "images", "assoc", assoc_filename);101 // - if the assoc path already exists, we need to recursively delete it and102 // its contents103 if (Files.exists(assoc_path))104 {105 logger.info("VideoDocument - removing existing (old) associated files");106 try107 {108 Files.walkFileTree(assoc_path, new SimpleFileVisitor<Path>()109 {110 @Override111 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)112 throws IOException113 {114 ///ystem.out.println("Deleting file: " + file);115 Files.delete(file);116 return CONTINUE;117 }118 @Override119 public FileVisitResult postVisitDirectory(Path dir, IOException exc)120 throws IOException121 {122 ///ystem.out.println("Deleting dir: " + dir);123 if (exc == null)124 {125 Files.delete(dir);126 return CONTINUE;127 }128 else129 {130 throw exc;131 }132 }133 });134 }135 catch (Exception e)136 {137 logger.error("Exception while recursively deleting assoc folder:", e);138 }139 }140 assoc_path.toFile().mkdirs(); // bet there is a nice nio way to do this141 126 Path target_path = assoc_path.resolve(target_filename); 142 127 logger.info("VideoDocument - symlinking original video into assoc directory"); … … 389 374 } 390 375 /** getReader() **/ 376 377 /** 378 */ 379 private String generateHash(String string) 380 { 381 StringBuffer sb = new StringBuffer(); 382 try 383 { 384 final MessageDigest message_digest = MessageDigest.getInstance("MD5"); 385 message_digest.reset(); 386 message_digest.update(string.getBytes(Charset.forName("UTF8"))); 387 final byte[] result_bytes = message_digest.digest(); 388 for (int i = 0; i < result_bytes.length; ++i) 389 { 390 sb.append(Integer.toHexString((result_bytes[i] & 0xFF) | 0x100).substring(1,3)); 391 } 392 } 393 catch (NoSuchAlgorithmException e) 394 { 395 System.err.println("Exception: " + e); 396 System.exit(0); 397 } 398 return sb.toString(); 399 } 400 /** generateHash(String) **/ 391 401 }
Note:
See TracChangeset
for help on using the changeset viewer.