source: main/trunk/model-sites-dev/von-sparql/collect/nz-natlib-cat/pre-import/marcXML_Split/src/split.java@ 28794

import java.io.*;
import com.google.common.hash.*;
import com.google.common.io.Files;
import org.marc4j.*;
import org.marc4j.marc.Record;

public class split {

    public static void main(String[] args) throws Exception
    {
        long StartTime = System.currentTimeMillis();

        //Default values for arguments
        int NRecords = 250;
        String OutputPath = "./out/nz_out";
        String InputPath = "./NZDataFull.xml";
        //String InputPath = "./NZ-small-macron-test.xml";

        //Incorrect number of arguments supplied
        if(args.length!=2)
        {
            System.err.println("USAGE: ./RUN [records_per_file] [output_path]");
            return;
        }
        
        try{
            NRecords = Integer.parseInt(args[0]);
            OutputPath = args[1];
        }
        catch(Exception e)
        {
            System.err.println("Invalid Arguments");
            System.err.println("USAGE: ./RUN [records_per_file] [output_path]");
            return;
        }   

        InputStream in;
        try{
            in = new FileInputStream(InputPath);
        }
        catch(Exception e){
            System.err.println("./NZDataFull.xml does not exist");
            return;
        }

        MarcXmlReader reader = new MarcXmlReader(in);
        String TempFilename = OutputPath + "/temp.xml"; 
        File f;
        int RecordCount = 0;
                int ParseErrorCount = 0;
        
        while(reader.hasNext())
        {   
            f = new File(TempFilename);
            MarcWriter writer = new MarcXmlWriter(new FileOutputStream(f),"UTF8",true);
            Record record;

            //Write segment of records to file
            for(int i=0; (i<NRecords && reader.hasNext()); i++)
            {
                try {
                    record = reader.next();
                }
                catch (Exception e) {
                    System.err.println("Warning: Failed to parse record number: " + RecordCount);
                    ParseErrorCount++;
                    continue;
                }
                                writer.write(record);
                                RecordCount++;
            }
            writer.close();
            
            //Print update
            if(RecordCount%(NRecords * 50)==0)
                System.err.print("\rProcessed " + RecordCount + " records");
            
            //Calculate MD5
            HashCode hc = Files.hash(f, Hashing.md5());
            String s = hc.toString();
            
            //Ensure target folder exists, then rename file to hash string
            File target = new File(OutputPath+"/"+s.substring(0, 2)+"/"+s.substring(2)+".xml");
            File parent = target.getParentFile();
            if(!parent.exists() && !parent.mkdirs()){
                throw new IllegalStateException("Couldn't create dir " + parent);
            }
            f.renameTo(target);
        }       
        System.err.print("\rProcessed " + RecordCount + " records");
        System.err.println("\nTime taken: " + (System.currentTimeMillis()-StartTime) + "ms");       
    }
}