Changeset 13406

Timestamp:

2006-12-01T16:01:40+13:00 (17 years ago)

Author:

mdewsnip

Message:

(Remote building) Hugely simplified the zip file filtering in the process of fixing a nasty bug with zipping up collection archives.

Location:

trunk/gli/src/org/greenstone/gatherer/remote

Files:

• ZipCollectionArchives.java (modified) (1 diff)
• ZipTools.java (modified) (3 diffs)

trunk/gli/src/org/greenstone/gatherer/remote/ZipCollectionArchives.java

@@ -57 +57 @@
 
 
-    public int filterFileContent(BufferedInputStream bis, byte[] data, int bytes_to_read)
+    public boolean shouldIncludeFileContent(String relative_file_path)
     {
+        // Only content for doc.xml files is included
+        return (relative_file_path.equals("doc.xml") || relative_file_path.endsWith(File.separator + "doc.xml"));
+    }
+
+
+    public void filterFileContent(String relative_file_path, BufferedInputStream bis, ZipOutputStream zos)
+    {
+        // Reset the status in case there were errors in previous doc.xml files
+        section_depth = 0;
+        in_section_content = false;
+
+        // Filter out the <Content>...</Content> of the doc.xml files
         try {
-        // Read the next chunk of bytes from the file
-        byte[] bytes_read_this_time = new byte[bytes_to_read];
-        int num_bytes_read_this_time = bis.read(bytes_read_this_time, 0, bytes_to_read);
-        if (num_bytes_read_this_time == -1) {
-            // We must be at the end of the file
-            return -1;
-        }
-
-        // Add this chunk to the bytes remaining from last time
-        byte[] bytes_unprocessed = new byte[bytes_remaining.length + num_bytes_read_this_time];
-        for (int i = 0; i < bytes_remaining.length; i++) {
-            bytes_unprocessed[i] = bytes_remaining[i];
-        }
-        for (int i = 0; i < num_bytes_read_this_time; i++) {
-            bytes_unprocessed[bytes_remaining.length + i] = bytes_read_this_time[i];
-        }
-
-        // Split the chunk into lines and filter them
-        int num_bytes_processed = 0;
-        int num_bytes_unfiltered = 0;
-        int last_line_start = 0;
-        for (int i = 0; (i < bytes_unprocessed.length && num_bytes_unfiltered < bytes_to_read); i++) {
-            // Found the end of a line
-            if (bytes_unprocessed[i] == '\n') {
-            int line_length = i - last_line_start + 1;
-            String line = new String(bytes_unprocessed, last_line_start, line_length);
-
-            // If this line isn't filtered, write it to the data array
-            if (!filterFileLine(line)) {
-                for (int j = 0; (j < line_length && num_bytes_unfiltered < bytes_to_read); j++) {
-                data[num_bytes_unfiltered] = bytes_unprocessed[last_line_start + j];
-                num_bytes_unfiltered++;
-                num_bytes_processed++;
-                }
-            }
-            // Otherwise ignore it
-            else {
-                num_bytes_processed = i + 1;
-            }
-
-            last_line_start = i + 1;
+        BufferedReader reader = new BufferedReader(new InputStreamReader(bis, "UTF-8"));
+        String line = null;
+        while ((line = reader.readLine()) != null) {
+            // If this line isn't filtered, write it out to zos
+            if (!filterFileLine(line)) {
+            line += "\n";
+            byte[] bytes = line.getBytes("UTF-8");
+            zos.write(bytes, 0, bytes.length);
             }
         }
-
-        // Create a new byte array containing any bytes remaining (these will be processed next time)
-        int num_bytes_remaining = bytes_unprocessed.length - num_bytes_processed;
-        bytes_remaining = new byte[num_bytes_remaining];
-        for (int i = 0; i < num_bytes_remaining; i++) {
-            bytes_remaining[i] = bytes_unprocessed[num_bytes_processed + i];
-        }
-
-        return num_bytes_unfiltered;
         }
         catch (Exception exception) {
         exception.printStackTrace();
         }
-
-        return -1;
     }
 

trunk/gli/src/org/greenstone/gatherer/remote/ZipTools.java

@@ -87 +87 @@
             if (zip_filter.shouldIncludeFileContent(relative_file_path)) {
             BufferedInputStream bis = new BufferedInputStream(new FileInputStream(file));
-            byte[] data = new byte[1024];
-            int bytes_read;
-            while ((bytes_read = zip_filter.filterFileContent(bis, data, 1024)) > -1) {
-                zos.write(data, 0, bytes_read);
-            }
+            zip_filter.filterFileContent(relative_file_path, bis, zos);
             bis.close();
             }
@@ -109 +105 @@
     public boolean shouldIncludeFileContent(String relative_file_path);
 
-    public int filterFileContent(BufferedInputStream bis, byte[] data, int bytes_to_read);
+    public void filterFileContent(String relative_file_path, BufferedInputStream bis, ZipOutputStream zos);
     }
 
@@ -130 +126 @@
 
 
-    public int filterFileContent(BufferedInputStream bis, byte[] data, int bytes_to_read)
+    public void filterFileContent(String relative_file_path, BufferedInputStream bis, ZipOutputStream zos)
     {
+        // No filtering: just read the file and write it directly out to zip
         try {
-        return bis.read(data, 0, bytes_to_read);
+        byte[] data = new byte[1024];
+        int bytes_read;
+        while ((bytes_read = bis.read(data, 0, 1024)) > -1) {
+            zos.write(data, 0, bytes_read);
+        }
         }
         catch (Exception exception) {
+        // We can't use DebugStream here
         exception.printStackTrace();
         }
-
-        return -1;
     }
     }