source: main/trunk/model-sites-dev/von-sparql/collect/nz-natlib-cat/pre-import/marcXML_Split/src/split.java@ 28734

import java.io.*;
import com.google.common.hash.*;
import com.google.common.io.Files;
import org.marc4j.*;
import org.marc4j.marc.Record;

public class split {

    public static void main(String[] args) throws Exception
    {
        long StartTime = System.currentTimeMillis();

        //Default values for arguments
        int NRecords = 250;
        String OutputPath = "./out/";
        String InputPath = "./NZDataFull.xml";

        //Incorrect number of arguments supplied
        if(args.length!=4)
        {
            System.err.println("USAGE: java split [-n records_per_file] [-o output_path]");
            return;
        }
        //read arguments    
        for(int i=0; i < args.length; i+=2)
        {
            if(args[i].equals("-n"))
                NRecords = Integer.parseInt(args[i+1]);
            else if(args[i].equals("-o"))
                OutputPath = args[i+1];
        }
        
        InputStream in;
        try{
            in = new FileInputStream(InputPath);
        }
        catch(Exception e){
            System.err.println("./NZDataFull.xml does not exist");
            return;
        }

        MarcXmlReader reader = new MarcXmlReader(in);
        String TempFilename = OutputPath + "/temp.xml"; 
        File f;
        int RecordCount = 0;
        
        while(reader.hasNext())
        {       
            f = new File(TempFilename);
            MarcWriter writer = new MarcXmlWriter(new FileOutputStream(f),true);
            Record record;

            //Write segment of records to file
            for(int i=0; (i<NRecords && reader.hasNext()); i++)
            {
                record = reader.next();
                writer.write(record);
                RecordCount++;
            }
            writer.close();
            
            //Print update
            if(RecordCount%(NRecords * 50)==0)
                System.err.print("\rProcessed " + RecordCount + " records");
            
            //Calculate MD5
            HashCode hc = Files.hash(f, Hashing.md5());
            String s = hc.toString();
            
            //Ensure target folder exists, then rename file to hash string
            File target = new File(OutputPath+"/"+s.substring(0, 2)+"/"+s.substring(2)+".xml");
            File parent = target.getParentFile();
            if(!parent.exists() && !parent.mkdirs()){
                throw new IllegalStateException("Couldn't create dir " + parent);
            }
            f.renameTo(target);
        }       
        System.err.print("\rProcessed " + RecordCount + " records");
        System.err.println("\nTime taken: " + (System.currentTimeMillis()-StartTime) + "ms");       
    }
}