1 | import java.io.*;
|
---|
2 | import java.util.ArrayList;
|
---|
3 |
|
---|
4 |
|
---|
5 | public class ByteFixer {
|
---|
6 | private File fin ;
|
---|
7 | private ArrayList<Byte> Queue;
|
---|
8 | private byte[] OverrunByte;
|
---|
9 | private DataInputStream reader;
|
---|
10 | public ByteFixer(String FileName) {
|
---|
11 | Queue = new ArrayList<Byte>();
|
---|
12 | fin = new File(FileName);
|
---|
13 | try{
|
---|
14 | reader = new DataInputStream(new FileInputStream(fin));
|
---|
15 | } catch(FileNotFoundException e){
|
---|
16 | System.err.println("Input file not found");
|
---|
17 | return;
|
---|
18 | }
|
---|
19 | }
|
---|
20 |
|
---|
21 | public void close(){
|
---|
22 | try {
|
---|
23 | reader.close();
|
---|
24 | } catch (IOException e) {
|
---|
25 | e.printStackTrace();
|
---|
26 | }
|
---|
27 | }
|
---|
28 | public void fixCharacter() throws UnsupportedEncodingException{
|
---|
29 |
|
---|
30 | boolean done_ansel_quick_fix = false;
|
---|
31 |
|
---|
32 |
|
---|
33 | if (Queue.size() == 1) {
|
---|
34 | byte first_char = Queue.get(0);
|
---|
35 | if (first_char == 0xB0) {
|
---|
36 | done_ansel_quick_fix = true;
|
---|
37 | Queue.set(0,(byte)'`');
|
---|
38 | }
|
---|
39 | else if (first_char == 0xFE) {
|
---|
40 | done_ansel_quick_fix = true;
|
---|
41 | Queue.set(0,(byte)'\'');
|
---|
42 | }
|
---|
43 | }
|
---|
44 |
|
---|
45 | if (done_ansel_quick_fix) {
|
---|
46 | return;
|
---|
47 | }
|
---|
48 |
|
---|
49 | //Copy Queue to byte array
|
---|
50 | byte[] Character = new byte[Queue.size()];
|
---|
51 | for(int i=0; i < Queue.size(); i++)
|
---|
52 | Character[i]=Queue.get(i);
|
---|
53 |
|
---|
54 | //Convert to ISO8859
|
---|
55 | String str = new String(Character,"ISO-8859-1");
|
---|
56 |
|
---|
57 | byte[] UTF8 = str.getBytes("UTF-8");
|
---|
58 |
|
---|
59 | System.err.println("FIXED TO: " + new String(UTF8, "UTF-8"));
|
---|
60 |
|
---|
61 | Queue = new ArrayList<Byte>();
|
---|
62 | for(int i=0; i < UTF8.length; i++)
|
---|
63 | Queue.add(UTF8[i]);
|
---|
64 | return;
|
---|
65 | }
|
---|
66 |
|
---|
67 | public byte[] getNextCharacter(){
|
---|
68 |
|
---|
69 | int head;
|
---|
70 |
|
---|
71 | if(OverrunByte!=null){
|
---|
72 | byte[] Queue_Bytes = new byte[1];
|
---|
73 | Queue_Bytes[0] = OverrunByte[0];
|
---|
74 | OverrunByte = null;
|
---|
75 | return Queue_Bytes;
|
---|
76 | }
|
---|
77 |
|
---|
78 | try{
|
---|
79 | head = reader.readUnsignedByte();
|
---|
80 | if((head & 0x80)==0){
|
---|
81 | //Simple ASCII case, 1-byte sequence
|
---|
82 | Queue.add((byte)head);
|
---|
83 | }
|
---|
84 | else{
|
---|
85 | //If >1-byte sequence
|
---|
86 |
|
---|
87 | Queue.add((byte)head);
|
---|
88 |
|
---|
89 | if ((head & 0xC0) != 0xC0) {
|
---|
90 | // Isn't the start of a valid UTF-8 sequence
|
---|
91 | fixCharacter();
|
---|
92 | }
|
---|
93 | else {
|
---|
94 | head = head << 1;
|
---|
95 | while((head & 0x80)!=0){
|
---|
96 | int next = reader.readUnsignedByte();
|
---|
97 | if((next & 0xC0)==0x80)
|
---|
98 | Queue.add((byte)next);
|
---|
99 | else{
|
---|
100 | //If sequence end
|
---|
101 | String s = Integer.toBinaryString(next);
|
---|
102 | System.err.println("Encountered non-UTF8 character " + s);
|
---|
103 | //Convert to UTF8
|
---|
104 | fixCharacter();
|
---|
105 | //Save overrun byte
|
---|
106 | OverrunByte = new byte[1];
|
---|
107 | OverrunByte[0] = (byte)next;
|
---|
108 | break;
|
---|
109 | }
|
---|
110 | head = head << 1;
|
---|
111 | }
|
---|
112 | }
|
---|
113 | }
|
---|
114 | }catch(EOFException e){
|
---|
115 | System.out.println("End of file reached");
|
---|
116 | return null;
|
---|
117 | }catch(Exception e){
|
---|
118 | e.printStackTrace();
|
---|
119 | }
|
---|
120 | //Copy Queue to byte array
|
---|
121 | byte[] Queue_Bytes = new byte[Queue.size()];
|
---|
122 | for(int i=0; i < Queue.size(); i++){
|
---|
123 | Queue_Bytes[i] = Queue.get(i);
|
---|
124 | }
|
---|
125 | //Reset queue to be empty
|
---|
126 | Queue = new ArrayList<Byte>();
|
---|
127 | return Queue_Bytes;
|
---|
128 | }
|
---|
129 |
|
---|
130 | }
|
---|