Index: /main/trunk/model-sites-dev/von-sparql/collect/nz-natlib-cat/pre-import/UTF8_Fix/.classpath
===================================================================
--- /main/trunk/model-sites-dev/von-sparql/collect/nz-natlib-cat/pre-import/UTF8_Fix/.classpath	(revision 28790)
+++ /main/trunk/model-sites-dev/von-sparql/collect/nz-natlib-cat/pre-import/UTF8_Fix/.classpath	(revision 28790)
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+	<classpathentry kind="src" path="src"/>
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
+	<classpathentry kind="lib" path="/home/sjs49/workspace/marcXML_Split/marc4j-2.6.0.jar" sourcepath="/home/sjs49/workspace/marcXML_Split/marc4j-master/src/org/marc4j"/>
+	<classpathentry kind="lib" path="/home/sjs49/workspace/marcXML_Split/guava-15.0.jar"/>
+	<classpathentry kind="output" path="bin"/>
+</classpath>
Index: /main/trunk/model-sites-dev/von-sparql/collect/nz-natlib-cat/pre-import/UTF8_Fix/.project
===================================================================
--- /main/trunk/model-sites-dev/von-sparql/collect/nz-natlib-cat/pre-import/UTF8_Fix/.project	(revision 28790)
+++ /main/trunk/model-sites-dev/von-sparql/collect/nz-natlib-cat/pre-import/UTF8_Fix/.project	(revision 28790)
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>marcXML_Split</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.eclipse.jdt.core.javabuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.eclipse.jdt.core.javanature</nature>
+	</natures>
+</projectDescription>
Index: /main/trunk/model-sites-dev/von-sparql/collect/nz-natlib-cat/pre-import/UTF8_Fix/Makefile
===================================================================
--- /main/trunk/model-sites-dev/von-sparql/collect/nz-natlib-cat/pre-import/UTF8_Fix/Makefile	(revision 28790)
+++ /main/trunk/model-sites-dev/von-sparql/collect/nz-natlib-cat/pre-import/UTF8_Fix/Makefile	(revision 28790)
@@ -0,0 +1,2 @@
+all: src/ByteFixer.java src/UTF8Fix.java
+	javac -d ./bin src/ByteFixer.java src/UTF8Fix.java
Index: /main/trunk/model-sites-dev/von-sparql/collect/nz-natlib-cat/pre-import/UTF8_Fix/src/ByteFixer.java
===================================================================
--- /main/trunk/model-sites-dev/von-sparql/collect/nz-natlib-cat/pre-import/UTF8_Fix/src/ByteFixer.java	(revision 28790)
+++ /main/trunk/model-sites-dev/von-sparql/collect/nz-natlib-cat/pre-import/UTF8_Fix/src/ByteFixer.java	(revision 28790)
@@ -0,0 +1,130 @@
+import java.io.*;
+import java.util.ArrayList;
+
+
+public class ByteFixer {
+	private File fin ;
+	private ArrayList<Byte> Queue;
+	private byte[] OverrunByte;
+	private DataInputStream reader;
+	public ByteFixer(String FileName)	{
+		Queue = new ArrayList<Byte>();
+		fin = new File(FileName);
+		try{
+			reader = new DataInputStream(new FileInputStream(fin));
+		} catch(FileNotFoundException e){
+			System.err.println("Input file not found");
+			return;
+		}
+	}
+
+	public void close(){
+		try {
+			reader.close();
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+	}
+	public void fixCharacter() throws UnsupportedEncodingException{
+		
+		boolean done_ansel_quick_fix = false;
+		
+		
+		if (Queue.size() == 1) {
+			byte first_char = Queue.get(0);
+			if (first_char == 0xB0) {
+				done_ansel_quick_fix = true;
+				Queue.set(0,(byte)'`');
+			}
+			else if (first_char == 0xFE) {
+				done_ansel_quick_fix = true;
+				Queue.set(0,(byte)'\'');
+			}
+		}
+
+		if (done_ansel_quick_fix) {
+			return;
+		}
+		
+		//Copy Queue to byte array
+		byte[] Character = new byte[Queue.size()];
+		for(int i=0; i < Queue.size(); i++)
+			Character[i]=Queue.get(i);
+		
+		//Convert to ISO8859 
+		String str = new String(Character,"ISO-8859-1");
+		
+		byte[] UTF8 = str.getBytes("UTF-8");
+		
+		System.err.println("FIXED TO: " + new String(UTF8, "UTF-8"));
+		
+		Queue = new ArrayList<Byte>();
+		for(int i=0; i < UTF8.length; i++)
+			Queue.add(UTF8[i]);
+		return;
+	}
+	
+	public byte[] getNextCharacter(){
+		
+		int head;
+		
+		if(OverrunByte!=null){
+			byte[] Queue_Bytes = new byte[1];
+			Queue_Bytes[0] = OverrunByte[0];
+			OverrunByte = null;
+			return Queue_Bytes;
+		}
+		
+		try{
+			 head = reader.readUnsignedByte();
+			 if((head & 0x80)==0){
+				 //Simple ASCII case, 1-byte sequence
+				 Queue.add((byte)head);
+			 }
+			 else{
+				 //If >1-byte sequence
+				 
+				 Queue.add((byte)head);
+				 
+				 if ((head & 0xC0) != 0xC0) {
+					 // Isn't the start of a valid UTF-8 sequence
+					 fixCharacter();
+				 }
+				 else {
+					 head = head << 1;
+					 while((head & 0x80)!=0){
+						 int next = reader.readUnsignedByte();
+						 if((next & 0xC0)==0x80)
+							 Queue.add((byte)next);
+						 else{
+							 //If sequence end
+							 String s = Integer.toBinaryString(next);
+							 System.err.println("Encountered non-UTF8 character " + s);
+							 //Convert to UTF8
+							 fixCharacter();
+							 //Save overrun byte
+							 OverrunByte = new byte[1];
+							 OverrunByte[0] = (byte)next;
+							 break;
+						 }
+						 head = head << 1;
+					 }
+				 }
+			 }
+		}catch(EOFException e){
+			System.out.println("End of file reached");
+			return null;
+		}catch(Exception e){
+			e.printStackTrace();
+		}
+		//Copy Queue to byte array
+		byte[] Queue_Bytes = new byte[Queue.size()];
+		for(int i=0; i < Queue.size(); i++){
+			Queue_Bytes[i] = Queue.get(i);
+		}
+		//Reset queue to be empty
+		Queue = new ArrayList<Byte>();
+		return Queue_Bytes;
+	}
+	
+}
Index: /main/trunk/model-sites-dev/von-sparql/collect/nz-natlib-cat/pre-import/UTF8_Fix/src/UTF8Fix.java
===================================================================
--- /main/trunk/model-sites-dev/von-sparql/collect/nz-natlib-cat/pre-import/UTF8_Fix/src/UTF8Fix.java	(revision 28790)
+++ /main/trunk/model-sites-dev/von-sparql/collect/nz-natlib-cat/pre-import/UTF8_Fix/src/UTF8Fix.java	(revision 28790)
@@ -0,0 +1,56 @@
+import java.io.*;
+
+
+public class UTF8Fix{
+	
+	public static void main(String[] args){
+		
+		if(args.length < 2 || args.length > 2){
+			System.err.println("USAGE: java UTF8Fix [input] [output]");
+			return;
+		}
+		ByteFixer byte_fixer = new ByteFixer(args[0]);
+		File fout = new File(args[1]);
+		BufferedOutputStream writer = null;
+		
+		try {
+			writer = new BufferedOutputStream(new FileOutputStream(fout));
+		} catch (FileNotFoundException e) {
+			System.err.println("Problem opening writer, file not found");
+			System.err.println("USAGE: java UTF8Fix [input] [output]");
+			return;
+		}
+				
+		byte[] character;
+		int count = 0;
+		String dots = "";
+		while(true){
+			character = byte_fixer.getNextCharacter();
+			if(character==null)
+				break;
+			
+			try {
+				writer.write(character);
+				count++;
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+			
+			if(count%1000000==0){
+				dots+= ".";
+				if(dots.length() > 20){
+					dots = "";
+					System.out.println();
+				}
+				System.out.print("\r" + dots);
+				System.out.flush();
+			}
+		}
+		try {
+			byte_fixer.close();
+			writer.close();
+		} catch (IOException e) {
+			System.err.println("Unable to close reader/writer");
+		}		
+	}
+}
