Changeset 26237

Timestamp:

2012-09-24T12:11:36+12:00 (12 years ago)

Author:

jmt12

Message:

Allows the specification of a maxfiles - limiting the number of files written to manifests and hence processed by Terrier

File:

• gs2-extensions/parallel-building/trunk/src/opt/Terrier/FileIndexer.java (modified) (8 diffs)

gs2-extensions/parallel-building/trunk/src/opt/Terrier/FileIndexer.java

@@ -14 +14 @@
 import java.util.ArrayDeque;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.HashSet;
@@ -406 +408 @@
     String index_prefix = ApplicationSetup.getProperty("terrier.index.prefix", "data");
     int batch_size = 0; // No limit
+    int max_files = 0; // No limit
     String divider = "===============================================================================";
 
@@ -412 +415 @@
     if (args.length < 1)
     {
-      System.out.println("Usage: <FileIndexer> -prepare -path <path> [-batchsize <int>]");
+      System.out.println("Usage: <FileIndexer> -prepare -path <path> [-batchsize <int>] [-maxfiles <int>]");
       System.out.println("       <FileIndexer> -index -path <path> [-prefix <str>]");
       System.out.println("       <FileIndexer> -merge");
@@ -439 +442 @@
         batch_size = Integer.parseInt(value);
       }
+      else if (key.equals("-maxfiles"))
+      {
+        max_files = Integer.parseInt(value);
+      }
       else if (key.equals("-path"))
       {
@@ -476 +483 @@
         System.out.println("Batch Size: " + batch_size);
       }
+      if (max_files != 0)
+      {
+        System.out.println("Max Files: " + max_files);
+      }
       System.out.println(divider);
       //    - we're going to perform a breadth-first recursive search for files
@@ -483 +494 @@
       search_paths.add(a_path);
       int file_count = 0;
+      int total_file_count = 0; // Used in conjunction with max_files
       int manifest_count = 0;
+      boolean have_max_files = false;
       PrintWriter manifest_writer = null;
-      while (!search_paths.isEmpty())
+      while (!search_paths.isEmpty() && !have_max_files)
       {
         Path search_path = search_paths.remove();
-        File files[] = search_path.toFile().listFiles();
-        for (int i = 0; i < files.length; i++)
+        File files_raw[] = search_path.toFile().listFiles();
+        // randomize file order
+        List<File> files = Arrays.asList(files_raw);
+        Collections.shuffle(files);
+        // iterate through files filling manifests
+        for (int i = 0; i < files.size() && !have_max_files; i++)
         {
-          Path file_path = files[i].toPath();
+          Path file_path = files.get(i).toPath();
           String file_name = file_path.getFileName().toString();
           // skip . and ..
@@ -524 +541 @@
               manifest_writer.println(file_path.toString());
               file_count++;
-              // if we have a file limit, and we've found enough files...
+              total_file_count++;
+              // if we have a batch size, and we've found enough files...
               if (manifest_writer != null && batch_size > 0 && file_count >= batch_size)
               {
@@ -531 +549 @@
                 manifest_count++;
                 file_count = 0;
+              }
+              // if we have a max files limit and we've done enough files, then
+              // break out of for and while loops
+              if (max_files > 0 && file_count >= max_files)
+              {
+                have_max_files = true;
               }
             }