source: other-projects/rsyntax-textarea/src/java/org/fife/ui/rsyntaxtextarea/modes/PlainTextTokenMaker.java@ 25584

Last change on this file since 25584 was 25584, checked in by davidb, 12 years ago

Initial cut an a text edit area for GLI that supports color syntax highlighting
File size: 18.7 KB
Line 
1/* The following code was generated by JFlex 1.4.1 on 12/9/11 6:40 PM */
2
3/*
4 * 11/07/2008
5 *
6 * PlainTextTokenMaker.flex - Scanner for plain text files.
7 *
8 * This library is distributed under a modified BSD license. See the included
9 * RSyntaxTextArea.License.txt file for details.
10 */
11package org.fife.ui.rsyntaxtextarea.modes;
12
13import java.io.*;
14import javax.swing.text.Segment;
15
16import org.fife.ui.rsyntaxtextarea.*;
17
18
19/**
20 * Scanner for plain text files.
21 *
22 * This implementation was created using
23 * <a href="http://www.jflex.de/">JFlex</a> 1.4.1; however, the generated file
24 * was modified for performance. Memory allocation needs to be almost
25 * completely removed to be competitive with the handwritten lexers (subclasses
26 * of <code>AbstractTokenMaker</code>), so this class has been modified so that
27 * Strings are never allocated (via yytext()), and the scanner never has to
28 * worry about refilling its buffer (needlessly copying chars around).
29 * We can achieve this because RText always scans exactly 1 line of tokens at a
30 * time, and hands the scanner this line as an array of characters (a Segment
31 * really). Since tokens contain pointers to char arrays instead of Strings
32 * holding their contents, there is no need for allocating new memory for
33 * Strings.<p>
34 *
35 * The actual algorithm generated for scanning has, of course, not been
36 * modified.<p>
37 *
38 * If you wish to regenerate this file yourself, keep in mind the following:
39 * <ul>
40 * <li>The generated <code>PlainTextTokenMaker.java</code> file will contain
41 * two definitions of both <code>zzRefill</code> and <code>yyreset</code>.
42 * You should hand-delete the second of each definition (the ones
43 * generated by the lexer), as these generated methods modify the input
44 * buffer, which we'll never have to do.</li>
45 * <li>You should also change the declaration/definition of zzBuffer to NOT
46 * be initialized. This is a needless memory allocation for us since we
47 * will be pointing the array somewhere else anyway.</li>
48 * <li>You should NOT call <code>yylex()</code> on the generated scanner
49 * directly; rather, you should use <code>getTokenList</code> as you would
50 * with any other <code>TokenMaker</code> instance.</li>
51 * </ul>
52 *
53 * @author Robert Futrell
54 * @version 0.5
55 *
56 */
57
58public class PlainTextTokenMaker extends AbstractJFlexTokenMaker {
59
60 /** This character denotes the end of file */
61 public static final int YYEOF = -1;
62
63 /** lexical states */
64 public static final int YYINITIAL = 0;
65
66 /**
67 * Translates characters to character classes
68 */
69 private static final String ZZ_CMAP_PACKED =
70 "\11\0\1\3\1\2\25\0\1\3\1\4\1\0\1\4\1\6\1\4"+
71 "\7\4\1\4\1\21\1\5\12\1\1\17\1\4\1\0\1\4\1\0"+
72 "\2\4\32\1\1\4\1\0\1\4\1\0\1\4\1\0\4\1\1\16"+
73 "\1\13\1\1\1\7\1\14\2\1\1\15\3\1\1\11\2\1\1\12"+
74 "\1\10\2\1\1\20\3\1\3\0\1\4\uff81\0";
75
76 /**
77 * Translates characters to character classes
78 */
79 private static final char [] ZZ_CMAP = zzUnpackCMap(ZZ_CMAP_PACKED);
80
81 /**
82 * Translates DFA states to action switch labels.
83 */
84 private static final int [] ZZ_ACTION = zzUnpackAction();
85
86 private static final String ZZ_ACTION_PACKED_0 =
87 "\1\0\2\1\1\2\1\3\14\1\1\0\1\4\2\0";
88
89 private static int [] zzUnpackAction() {
90 int [] result = new int[21];
91 int offset = 0;
92 offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result);
93 return result;
94 }
95
96 private static int zzUnpackAction(String packed, int offset, int [] result) {
97 int i = 0; /* index in packed string */
98 int j = offset; /* index in unpacked array */
99 int l = packed.length();
100 while (i < l) {
101 int count = packed.charAt(i++);
102 int value = packed.charAt(i++);
103 do result[j++] = value; while (--count > 0);
104 }
105 return j;
106 }
107
108
109 /**
110 * Translates a state to a row index in the transition table
111 */
112 private static final int [] ZZ_ROWMAP = zzUnpackRowMap();
113
114 private static final String ZZ_ROWMAP_PACKED_0 =
115 "\0\0\0\22\0\44\0\22\0\66\0\110\0\132\0\154"+
116 "\0\176\0\220\0\242\0\264\0\306\0\330\0\352\0\374"+
117 "\0\u010e\0\u0120\0\u0132\0\u0144\0\u0132";
118
119 private static int [] zzUnpackRowMap() {
120 int [] result = new int[21];
121 int offset = 0;
122 offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result);
123 return result;
124 }
125
126 private static int zzUnpackRowMap(String packed, int offset, int [] result) {
127 int i = 0; /* index in packed string */
128 int j = offset; /* index in unpacked array */
129 int l = packed.length();
130 while (i < l) {
131 int high = packed.charAt(i++) << 16;
132 result[j++] = high | packed.charAt(i++);
133 }
134 return j;
135 }
136
137 /**
138 * The transition table of the DFA
139 */
140 private static final int [] ZZ_TRANS = zzUnpackTrans();
141
142 private static final String ZZ_TRANS_PACKED_0 =
143 "\1\2\1\3\1\4\1\5\3\2\1\6\3\3\1\7"+
144 "\3\3\1\2\1\10\1\2\23\0\1\3\5\0\10\3"+
145 "\1\0\1\3\4\0\1\5\17\0\1\3\5\0\1\3"+
146 "\1\11\6\3\1\0\1\3\2\0\1\3\5\0\1\3"+
147 "\1\12\3\3\1\13\2\3\1\0\1\3\2\0\1\3"+
148 "\5\0\10\3\1\0\1\14\2\0\1\3\5\0\1\3"+
149 "\1\15\6\3\1\0\1\3\2\0\1\3\5\0\2\3"+
150 "\1\16\5\3\1\0\1\3\2\0\1\3\5\0\6\3"+
151 "\1\17\1\3\1\0\1\3\2\0\1\3\5\0\10\3"+
152 "\1\0\1\20\2\0\1\3\5\0\2\3\1\21\5\3"+
153 "\1\0\1\3\2\0\1\3\5\0\10\3\1\22\1\3"+
154 "\2\0\1\3\5\0\7\3\1\16\1\0\1\3\2\0"+
155 "\1\3\5\0\10\3\1\0\1\3\1\23\1\0\1\3"+
156 "\5\0\3\3\1\16\4\3\1\22\1\3\6\0\1\24"+
157 "\15\0\1\23\2\0\1\25\12\23\1\25\1\23\1\25"+
158 "\5\0\1\23\14\0";
159
160 private static int [] zzUnpackTrans() {
161 int [] result = new int[342];
162 int offset = 0;
163 offset = zzUnpackTrans(ZZ_TRANS_PACKED_0, offset, result);
164 return result;
165 }
166
167 private static int zzUnpackTrans(String packed, int offset, int [] result) {
168 int i = 0; /* index in packed string */
169 int j = offset; /* index in unpacked array */
170 int l = packed.length();
171 while (i < l) {
172 int count = packed.charAt(i++);
173 int value = packed.charAt(i++);
174 value--;
175 do result[j++] = value; while (--count > 0);
176 }
177 return j;
178 }
179
180
181 /* error codes */
182 private static final int ZZ_UNKNOWN_ERROR = 0;
183 private static final int ZZ_NO_MATCH = 1;
184 private static final int ZZ_PUSHBACK_2BIG = 2;
185
186 /* error messages for the codes above */
187 private static final String ZZ_ERROR_MSG[] = {
188 "Unkown internal scanner error",
189 "Error: could not match input",
190 "Error: pushback value was too large"
191 };
192
193 /**
194 * ZZ_ATTRIBUTE[aState] contains the attributes of state <code>aState</code>
195 */
196 private static final int [] ZZ_ATTRIBUTE = zzUnpackAttribute();
197
198 private static final String ZZ_ATTRIBUTE_PACKED_0 =
199 "\1\0\1\11\1\1\1\11\15\1\1\0\1\1\2\0";
200
201 private static int [] zzUnpackAttribute() {
202 int [] result = new int[21];
203 int offset = 0;
204 offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result);
205 return result;
206 }
207
208 private static int zzUnpackAttribute(String packed, int offset, int [] result) {
209 int i = 0; /* index in packed string */
210 int j = offset; /* index in unpacked array */
211 int l = packed.length();
212 while (i < l) {
213 int count = packed.charAt(i++);
214 int value = packed.charAt(i++);
215 do result[j++] = value; while (--count > 0);
216 }
217 return j;
218 }
219
220 /** the input device */
221 private java.io.Reader zzReader;
222
223 /** the current state of the DFA */
224 private int zzState;
225
226 /** the current lexical state */
227 private int zzLexicalState = YYINITIAL;
228
229 /** this buffer contains the current text to be matched and is
230 the source of the yytext() string */
231 private char zzBuffer[];
232
233 /** the textposition at the last accepting state */
234 private int zzMarkedPos;
235
236 /** the current text position in the buffer */
237 private int zzCurrentPos;
238
239 /** startRead marks the beginning of the yytext() string in the buffer */
240 private int zzStartRead;
241
242 /** endRead marks the last character in the buffer, that has been read
243 from input */
244 private int zzEndRead;
245
246 /** zzAtEOF == true <=> the scanner is at the EOF */
247 private boolean zzAtEOF;
248
249 /* user code: */
250
251
252 /**
253 * Constructor. This must be here because JFlex does not generate a
254 * no-parameter constructor.
255 */
256 public PlainTextTokenMaker() {
257 }
258
259
260 /**
261 * Adds the token specified to the current linked list of tokens.
262 *
263 * @param tokenType The token's type.
264 * @param link Whether this token is a hyperlink.
265 */
266 private void addToken(int tokenType, boolean link) {
267 int so = zzStartRead + offsetShift;
268 super.addToken(zzBuffer, zzStartRead,zzMarkedPos-1, tokenType, so, link);
269 zzStartRead = zzMarkedPos;
270 }
271
272
273 /**
274 * Always returns <code>Token.NULL</code>, as there are no multiline
275 * tokens in properties files.
276 *
277 * @param text The line of tokens to examine.
278 * @param initialTokenType The token type to start with (i.e., the value
279 * of <code>getLastTokenTypeOnLine</code> for the line before
280 * <code>text</code>).
281 * @return <code>Token.NULL</code>.
282 */
283 public int getLastTokenTypeOnLine(Segment text, int initialTokenType) {
284 return Token.NULL;
285 }
286
287
288 /**
289 * Returns the text to place at the beginning and end of a
290 * line to "comment" it in a this programming language.
291 *
292 * @return <code>null</code>, as there are no comments in plain text.
293 */
294 public String[] getLineCommentStartAndEnd() {
295 return null;
296 }
297
298
299 /**
300 * Always returns <tt>false</tt>, as you never want "mark occurrences"
301 * working in plain text files.
302 *
303 * @param type The token type.
304 * @return Whether tokens of this type should have "mark occurrences"
305 * enabled.
306 */
307 public boolean getMarkOccurrencesOfTokenType(int type) {
308 return false;
309 }
310
311
312 /**
313 * Returns the first token in the linked list of tokens generated
314 * from <code>text</code>. This method must be implemented by
315 * subclasses so they can correctly implement syntax highlighting.
316 *
317 * @param text The text from which to get tokens.
318 * @param initialTokenType The token type we should start with.
319 * @param startOffset The offset into the document at which
320 * <code>text</code> starts.
321 * @return The first <code>Token</code> in a linked list representing
322 * the syntax highlighted text.
323 */
324 public Token getTokenList(Segment text, int initialTokenType, int startOffset) {
325
326 resetTokenList();
327 this.offsetShift = -text.offset + startOffset;
328
329 // Start off in the proper state.
330 s = text;
331 try {
332 yyreset(zzReader);
333 yybegin(YYINITIAL);
334 return yylex();
335 } catch (IOException ioe) {
336 ioe.printStackTrace();
337 return new DefaultToken();
338 }
339
340 }
341
342
343 /**
344 * Refills the input buffer.
345 *
346 * @return <code>true</code> if EOF was reached, otherwise
347 * <code>false</code>.
348 * @exception IOException if any I/O-Error occurs.
349 */
350 private boolean zzRefill() {
351 return zzCurrentPos>=s.offset+s.count;
352 }
353
354
355 /**
356 * Resets the scanner to read from a new input stream.
357 * Does not close the old reader.
358 *
359 * All internal variables are reset, the old input stream
360 * <b>cannot</b> be reused (internal buffer is discarded and lost).
361 * Lexical state is set to <tt>YY_INITIAL</tt>.
362 *
363 * @param reader the new input stream
364 */
365 public final void yyreset(java.io.Reader reader) {
366 // 's' has been updated.
367 zzBuffer = s.array;
368 /*
369 * We replaced the line below with the two below it because zzRefill
370 * no longer "refills" the buffer (since the way we do it, it's always
371 * "full" the first time through, since it points to the segment's
372 * array). So, we assign zzEndRead here.
373 */
374 //zzStartRead = zzEndRead = s.offset;
375 zzStartRead = s.offset;
376 zzEndRead = zzStartRead + s.count - 1;
377 zzCurrentPos = zzMarkedPos = s.offset;
378 zzLexicalState = YYINITIAL;
379 zzReader = reader;
380 zzAtEOF = false;
381 }
382
383
384
385
386 /**
387 * Creates a new scanner
388 * There is also a java.io.InputStream version of this constructor.
389 *
390 * @param in the java.io.Reader to read input from.
391 */
392 public PlainTextTokenMaker(java.io.Reader in) {
393 this.zzReader = in;
394 }
395
396 /**
397 * Creates a new scanner.
398 * There is also java.io.Reader version of this constructor.
399 *
400 * @param in the java.io.Inputstream to read input from.
401 */
402 public PlainTextTokenMaker(java.io.InputStream in) {
403 this(new java.io.InputStreamReader(in));
404 }
405
406 /**
407 * Unpacks the compressed character translation table.
408 *
409 * @param packed the packed character translation table
410 * @return the unpacked character translation table
411 */
412 private static char [] zzUnpackCMap(String packed) {
413 char [] map = new char[0x10000];
414 int i = 0; /* index in packed string */
415 int j = 0; /* index in unpacked array */
416 while (i < 94) {
417 int count = packed.charAt(i++);
418 char value = packed.charAt(i++);
419 do map[j++] = value; while (--count > 0);
420 }
421 return map;
422 }
423
424
425 /**
426 * Closes the input stream.
427 */
428 public final void yyclose() throws java.io.IOException {
429 zzAtEOF = true; /* indicate end of file */
430 zzEndRead = zzStartRead; /* invalidate buffer */
431
432 if (zzReader != null)
433 zzReader.close();
434 }
435
436
437 /**
438 * Returns the current lexical state.
439 */
440 public final int yystate() {
441 return zzLexicalState;
442 }
443
444
445 /**
446 * Enters a new lexical state
447 *
448 * @param newState the new lexical state
449 */
450 public final void yybegin(int newState) {
451 zzLexicalState = newState;
452 }
453
454
455 /**
456 * Returns the text matched by the current regular expression.
457 */
458 public final String yytext() {
459 return new String( zzBuffer, zzStartRead, zzMarkedPos-zzStartRead );
460 }
461
462
463 /**
464 * Returns the character at position <tt>pos</tt> from the
465 * matched text.
466 *
467 * It is equivalent to yytext().charAt(pos), but faster
468 *
469 * @param pos the position of the character to fetch.
470 * A value from 0 to yylength()-1.
471 *
472 * @return the character at position pos
473 */
474 public final char yycharat(int pos) {
475 return zzBuffer[zzStartRead+pos];
476 }
477
478
479 /**
480 * Returns the length of the matched text region.
481 */
482 public final int yylength() {
483 return zzMarkedPos-zzStartRead;
484 }
485
486
487 /**
488 * Reports an error that occured while scanning.
489 *
490 * In a wellformed scanner (no or only correct usage of
491 * yypushback(int) and a match-all fallback rule) this method
492 * will only be called with things that "Can't Possibly Happen".
493 * If this method is called, something is seriously wrong
494 * (e.g. a JFlex bug producing a faulty scanner etc.).
495 *
496 * Usual syntax/scanner level error handling should be done
497 * in error fallback rules.
498 *
499 * @param errorCode the code of the errormessage to display
500 */
501 private void zzScanError(int errorCode) {
502 String message;
503 try {
504 message = ZZ_ERROR_MSG[errorCode];
505 }
506 catch (ArrayIndexOutOfBoundsException e) {
507 message = ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR];
508 }
509
510 throw new Error(message);
511 }
512
513
514 /**
515 * Pushes the specified amount of characters back into the input stream.
516 *
517 * They will be read again by then next call of the scanning method
518 *
519 * @param number the number of characters to be read again.
520 * This number must not be greater than yylength()!
521 */
522 public void yypushback(int number) {
523 if ( number > yylength() )
524 zzScanError(ZZ_PUSHBACK_2BIG);
525
526 zzMarkedPos -= number;
527 }
528
529
530 /**
531 * Resumes scanning until the next regular expression is matched,
532 * the end of input is encountered or an I/O-Error occurs.
533 *
534 * @return the next token
535 * @exception java.io.IOException if any I/O-Error occurs
536 */
537 public org.fife.ui.rsyntaxtextarea.Token yylex() throws java.io.IOException {
538 int zzInput;
539 int zzAction;
540
541 // cached fields:
542 int zzCurrentPosL;
543 int zzMarkedPosL;
544 int zzEndReadL = zzEndRead;
545 char [] zzBufferL = zzBuffer;
546 char [] zzCMapL = ZZ_CMAP;
547
548 int [] zzTransL = ZZ_TRANS;
549 int [] zzRowMapL = ZZ_ROWMAP;
550 int [] zzAttrL = ZZ_ATTRIBUTE;
551
552 while (true) {
553 zzMarkedPosL = zzMarkedPos;
554
555 zzAction = -1;
556
557 zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;
558
559 zzState = zzLexicalState;
560
561
562 zzForAction: {
563 while (true) {
564
565 if (zzCurrentPosL < zzEndReadL)
566 zzInput = zzBufferL[zzCurrentPosL++];
567 else if (zzAtEOF) {
568 zzInput = YYEOF;
569 break zzForAction;
570 }
571 else {
572 // store back cached positions
573 zzCurrentPos = zzCurrentPosL;
574 zzMarkedPos = zzMarkedPosL;
575 boolean eof = zzRefill();
576 // get translated positions and possibly new buffer
577 zzCurrentPosL = zzCurrentPos;
578 zzMarkedPosL = zzMarkedPos;
579 zzBufferL = zzBuffer;
580 zzEndReadL = zzEndRead;
581 if (eof) {
582 zzInput = YYEOF;
583 break zzForAction;
584 }
585 else {
586 zzInput = zzBufferL[zzCurrentPosL++];
587 }
588 }
589 int zzNext = zzTransL[ zzRowMapL[zzState] + zzCMapL[zzInput] ];
590 if (zzNext == -1) break zzForAction;
591 zzState = zzNext;
592
593 int zzAttributes = zzAttrL[zzState];
594 if ( (zzAttributes & 1) == 1 ) {
595 zzAction = zzState;
596 zzMarkedPosL = zzCurrentPosL;
597 if ( (zzAttributes & 8) == 8 ) break zzForAction;
598 }
599
600 }
601 }
602
603 // store back cached position
604 zzMarkedPos = zzMarkedPosL;
605
606 switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) {
607 case 3:
608 { addToken(Token.WHITESPACE, false);
609 }
610 case 5: break;
611 case 2:
612 { addNullToken(); return firstToken;
613 }
614 case 6: break;
615 case 4:
616 { addToken(Token.IDENTIFIER, true);
617 }
618 case 7: break;
619 case 1:
620 { addToken(Token.IDENTIFIER, false);
621 }
622 case 8: break;
623 default:
624 if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
625 zzAtEOF = true;
626 switch (zzLexicalState) {
627 case YYINITIAL: {
628 addNullToken(); return firstToken;
629 }
630 case 22: break;
631 default:
632 return null;
633 }
634 }
635 else {
636 zzScanError(ZZ_NO_MATCH);
637 }
638 }
639 }
640 }
641
642
643}
Note: See TracBrowser for help on using the repository browser.