source: main/trunk/model-sites-dev/von-sparql/collect/nz-natlib-cat/pre-import/hathiTab_Split/src/split.java@ 28791

import java.io.*;
import com.google.common.hash.*;
import com.google.common.io.Files;

public class split {

    public static void main(String[] args) throws Exception
    {
        long StartTime = System.currentTimeMillis();

        //Default values for arguments
        int NRecords = 250;
        String OutputPath = "./out/hathi_out";
        String InputPath = "./hathi_tab.txt";

        //Incorrect number of arguments supplied
        if(args.length!=2)
        {
            System.err.println("USAGE: ./RUN_HATHI [records_per_file] [output_path]");
            return;
        }
        
        try{
            NRecords = Integer.parseInt(args[0]);
            OutputPath = args[1];
        }
        catch(Exception e)
        {
            System.err.println("Invalid Arguments");
            System.err.println("USAGE: ./RUN [records_per_file] [output_path]");
            return;
        }   

            String headerpath = "hathiTab_Split" + File.separator + "etc" + File.separator + "header.txt";  
        BufferedReader reader;

         try{
            reader = new BufferedReader(new FileReader(headerpath));
        }
        catch(Exception e){
            System.err.println(headerpath + " does not exist");
            return;
        }
        
        String header = reader.readLine();
        
        reader.close();
        try{
            reader = new BufferedReader(new FileReader(InputPath));
        }
        catch(Exception e){
            System.err.println(InputPath + " does not exist");
            return;
        }

        String TempFilename = OutputPath + "/temp.txt"; 
        File f;
        int RecordCount = 0;
        
        String record = "";
        while(record!=null)
        {       
            f = new File(TempFilename);
            FileWriter fwriter = new FileWriter(f);
            BufferedWriter writer = new BufferedWriter(fwriter);
            
            //Write header to top of file
            writer.write(header);
            writer.newLine();

            //Write segment of records to file
            for(int i=0; (i<NRecords && (record=reader.readLine())!=null); i++)
            {
                writer.write(record);
                writer.newLine();
                RecordCount++;
            }
            writer.close();
            fwriter.close();
            
            //Print update
            if(RecordCount%(NRecords * 50)==0)
                System.err.print("\rProcessed " + RecordCount + " records");
            
            //Calculate MD5
            HashCode hc = Files.hash(f, Hashing.md5());
            String s = hc.toString();
            
            //Ensure target folder exists, then rename file to hash string
            File target = new File(OutputPath+"/"+s.substring(0, 2)+"/"+s.substring(2)+".tab");
            File parent = target.getParentFile();
            if(!parent.exists() && !parent.mkdirs()){
                throw new IllegalStateException("Couldn't create dir " + parent);
            }
            f.renameTo(target);
        }       
        reader.close();
        System.err.print("\rProcessed " + RecordCount + " records");
        System.err.println("\nTime taken: " + (System.currentTimeMillis()-StartTime) + "ms");       
    }
}