source: other-projects/rsyntax-textarea/src/java/org/fife/ui/rsyntaxtextarea/modes/DelphiTokenMaker.flex@ 25584

Last change on this file since 25584 was 25584, checked in by davidb, 12 years ago

Initial cut an a text edit area for GLI that supports color syntax highlighting
File size: 15.7 KB
Line 
1/*
2 * 7/28/2009
3 *
4 * DelphiTokenMaker.java - Scanner for the Delphi programming language.
5 *
6 * This library is distributed under a modified BSD license. See the included
7 * RSyntaxTextArea.License.txt file for details.
8 */
9package org.fife.ui.rsyntaxtextarea.modes;
10
11import java.io.*;
12import javax.swing.text.Segment;
13
14import org.fife.ui.rsyntaxtextarea.*;
15
16
17/**
18 * Scanner for the Delphi programming language.<p>
19 *
20 * This implementation was created using
21 * <a href="http://www.jflex.de/">JFlex</a> 1.4.1; however, the generated file
22 * was modified for performance. Memory allocation needs to be almost
23 * completely removed to be competitive with the handwritten lexers (subclasses
24 * of <code>AbstractTokenMaker</code>, so this class has been modified so that
25 * Strings are never allocated (via yytext()), and the scanner never has to
26 * worry about refilling its buffer (needlessly copying chars around).
27 * We can achieve this because RText always scans exactly 1 line of tokens at a
28 * time, and hands the scanner this line as an array of characters (a Segment
29 * really). Since tokens contain pointers to char arrays instead of Strings
30 * holding their contents, there is no need for allocating new memory for
31 * Strings.<p>
32 *
33 * The actual algorithm generated for scanning has, of course, not been
34 * modified.<p>
35 *
36 * If you wish to regenerate this file yourself, keep in mind the following:
37 * <ul>
38 * <li>The generated DelphiTokenMaker.java</code> file will contain two
39 * definitions of both <code>zzRefill</code> and <code>yyreset</code>.
40 * You should hand-delete the second of each definition (the ones
41 * generated by the lexer), as these generated methods modify the input
42 * buffer, which we'll never have to do.</li>
43 * <li>You should also change the declaration/definition of zzBuffer to NOT
44 * be initialized. This is a needless memory allocation for us since we
45 * will be pointing the array somewhere else anyway.</li>
46 * <li>You should NOT call <code>yylex()</code> on the generated scanner
47 * directly; rather, you should use <code>getTokenList</code> as you would
48 * with any other <code>TokenMaker</code> instance.</li>
49 * </ul>
50 *
51 * @author Robert Futrell
52 * @version 0.5
53 *
54 */
55%%
56
57%public
58%class DelphiTokenMaker
59%extends AbstractJFlexTokenMaker
60%unicode
61%type org.fife.ui.rsyntaxtextarea.Token
62
63
64%{
65
66
67 /**
68 * Token type specific to DelphiTokenMaker; denotes a line ending
69 * with an unterminated "(*" comment.
70 */
71 public static final int INTERNAL_MLC2 = -1;
72
73 /**
74 * Token type specific to DelphiTokenMaker; denotes a line ending
75 * with an unterminated "{$" compiler directive.
76 */
77 public static final int INTERNAL_COMPILER_DIRECTIVE = -2;
78
79 /**
80 * Token type specific to DelphiTokenMaker; denotes a line ending
81 * with an unterminated "(*$" compiler directive.
82 */
83 public static final int INTERNAL_COMPILER_DIRECTIVE2 = -3;
84
85
86 /**
87 * Constructor. This must be here because JFlex does not generate a
88 * no-parameter constructor.
89 */
90 public DelphiTokenMaker() {
91 }
92
93
94 /**
95 * Adds the token specified to the current linked list of tokens as an
96 * "end token;" that is, at <code>zzMarkedPos</code>.
97 *
98 * @param tokenType The token's type.
99 */
100 private void addEndToken(int tokenType) {
101 addToken(zzMarkedPos,zzMarkedPos, tokenType);
102 }
103
104
105 /**
106 * Adds the token specified to the current linked list of tokens.
107 *
108 * @param tokenType The token's type.
109 * @see #addToken(int, int, int)
110 */
111 private void addHyperlinkToken(int start, int end, int tokenType) {
112 int so = start + offsetShift;
113 addToken(zzBuffer, start,end, tokenType, so, true);
114 }
115
116
117 /**
118 * Adds the token specified to the current linked list of tokens.
119 *
120 * @param tokenType The token's type.
121 */
122 private void addToken(int tokenType) {
123 addToken(zzStartRead, zzMarkedPos-1, tokenType);
124 }
125
126
127 /**
128 * Adds the token specified to the current linked list of tokens.
129 *
130 * @param tokenType The token's type.
131 * @see #addHyperlinkToken(int, int, int)
132 */
133 private void addToken(int start, int end, int tokenType) {
134 int so = start + offsetShift;
135 addToken(zzBuffer, start,end, tokenType, so, false);
136 }
137
138
139 /**
140 * Adds the token specified to the current linked list of tokens.
141 *
142 * @param array The character array.
143 * @param start The starting offset in the array.
144 * @param end The ending offset in the array.
145 * @param tokenType The token's type.
146 * @param startOffset The offset in the document at which this token
147 * occurs.
148 * @param hyperlink Whether this token is a hyperlink.
149 */
150 public void addToken(char[] array, int start, int end, int tokenType,
151 int startOffset, boolean hyperlink) {
152 super.addToken(array, start,end, tokenType, startOffset, hyperlink);
153 zzStartRead = zzMarkedPos;
154 }
155
156
157 /**
158 * Returns the text to place at the beginning and end of a
159 * line to "comment" it in a this programming language.
160 *
161 * @return The start and end strings to add to a line to "comment"
162 * it out.
163 */
164 public String[] getLineCommentStartAndEnd() {
165 return new String[] { "//", null };
166 }
167
168
169 /**
170 * Returns the first token in the linked list of tokens generated
171 * from <code>text</code>. This method must be implemented by
172 * subclasses so they can correctly implement syntax highlighting.
173 *
174 * @param text The text from which to get tokens.
175 * @param initialTokenType The token type we should start with.
176 * @param startOffset The offset into the document at which
177 * <code>text</code> starts.
178 * @return The first <code>Token</code> in a linked list representing
179 * the syntax highlighted text.
180 */
181 public Token getTokenList(Segment text, int initialTokenType, int startOffset) {
182
183 resetTokenList();
184 this.offsetShift = -text.offset + startOffset;
185
186 // Start off in the proper state.
187 int state = Token.NULL;
188 switch (initialTokenType) {
189 case Token.COMMENT_MULTILINE:
190 state = MLC;
191 start = text.offset;
192 break;
193 case INTERNAL_MLC2:
194 state = MLC2;
195 start = text.offset;
196 break;
197 case INTERNAL_COMPILER_DIRECTIVE:
198 state = COMPILER_DIRECTIVE;
199 start = text.offset;
200 break;
201 case INTERNAL_COMPILER_DIRECTIVE2:
202 state = COMPILER_DIRECTIVE2;
203 start = text.offset;
204 break;
205 default:
206 state = Token.NULL;
207 }
208
209 s = text;
210 try {
211 yyreset(zzReader);
212 yybegin(state);
213 return yylex();
214 } catch (IOException ioe) {
215 ioe.printStackTrace();
216 return new DefaultToken();
217 }
218
219 }
220
221
222 /**
223 * Refills the input buffer.
224 *
225 * @return <code>true</code> if EOF was reached, otherwise
226 * <code>false</code>.
227 * @exception IOException if any I/O-Error occurs.
228 */
229 private boolean zzRefill() throws java.io.IOException {
230 return zzCurrentPos>=s.offset+s.count;
231 }
232
233
234 /**
235 * Resets the scanner to read from a new input stream.
236 * Does not close the old reader.
237 *
238 * All internal variables are reset, the old input stream
239 * <b>cannot</b> be reused (internal buffer is discarded and lost).
240 * Lexical state is set to <tt>YY_INITIAL</tt>.
241 *
242 * @param reader the new input stream
243 */
244 public final void yyreset(java.io.Reader reader) throws java.io.IOException {
245 // 's' has been updated.
246 zzBuffer = s.array;
247 /*
248 * We replaced the line below with the two below it because zzRefill
249 * no longer "refills" the buffer (since the way we do it, it's always
250 * "full" the first time through, since it points to the segment's
251 * array). So, we assign zzEndRead here.
252 */
253 //zzStartRead = zzEndRead = s.offset;
254 zzStartRead = s.offset;
255 zzEndRead = zzStartRead + s.count - 1;
256 zzCurrentPos = zzMarkedPos = zzPushbackPos = s.offset;
257 zzLexicalState = YYINITIAL;
258 zzReader = reader;
259 zzAtBOL = true;
260 zzAtEOF = false;
261 }
262
263
264%}
265
266Letter = [A-Za-z]
267LetterOrUnderscore = ({Letter}|"_")
268NonzeroDigit = [1-9]
269Digit = ("0"|{NonzeroDigit})
270HexDigit = ({Digit}|[A-Fa-f])
271OctalDigit = ([0-7])
272NonSeparator = ([^\t\f\r\n\ \(\)\{\}\[\]\;\,\.\=\>\<\!\~\?\:\+\-\*\/\&\|\^\%\"\']|"#"|"\\")
273IdentifierStart = ({LetterOrUnderscore}|"$")
274IdentifierPart = ({IdentifierStart}|{Digit})
275
276LineTerminator = (\n)
277WhiteSpace = ([ \t\f])
278
279UnclosedStringLiteral = ([\'][^\']*)
280StringLiteral = ({UnclosedStringLiteral}[\'])
281EscapeSequence = ("#"{Digit}*)
282
283MLCBegin = "{"
284MLCEnd = "}"
285MLC2Begin = "(*"
286MLC2End = "*)"
287CompilerDirective1Begin = ({MLCBegin}"$")
288CompilerDirective2Begin = ({MLC2Begin}"$")
289LineCommentBegin = "//"
290
291IntegerHelper1 = (({NonzeroDigit}{Digit}*)|"0")
292IntegerHelper2 = ("0"(([xX]{HexDigit}+)|({OctalDigit}*)))
293IntegerLiteral = ({IntegerHelper1}[lL]?)
294HexLiteral = ({IntegerHelper2}[lL]?)
295FloatHelper1 = ([fFdD]?)
296FloatHelper2 = ([eE][+-]?{Digit}+{FloatHelper1})
297FloatLiteral1 = ({Digit}+"."({FloatHelper1}|{FloatHelper2}|{Digit}+({FloatHelper1}|{FloatHelper2})))
298FloatLiteral2 = ("."{Digit}+({FloatHelper1}|{FloatHelper2}))
299FloatLiteral3 = ({Digit}+{FloatHelper2})
300FloatLiteral = ({FloatLiteral1}|{FloatLiteral2}|{FloatLiteral3}|({Digit}+[fFdD]))
301ErrorNumberFormat = (({IntegerLiteral}|{HexLiteral}|{FloatLiteral}){NonSeparator}+)
302BooleanLiteral = ("true"|"false")
303
304Separator = ([\(\)\[\]])
305Separator2 = ([\;,.])
306
307Operator = ([\^\@\:\=\<\>\+\-\/\*])
308
309Identifier = ({IdentifierStart}{IdentifierPart}*)
310ErrorIdentifier = ({NonSeparator}+)
311
312URLGenDelim = ([:\/\?#\[\]@])
313URLSubDelim = ([\!\$&'\(\)\*\+,;=])
314URLUnreserved = ({LetterOrUnderscore}|{Digit}|[\-\.\~])
315URLCharacter = ({URLGenDelim}|{URLSubDelim}|{URLUnreserved}|[%])
316URLCharacters = ({URLCharacter}*)
317URLEndCharacter = ([\/\$]|{Letter}|{Digit})
318URL = (((https?|f(tp|ile))"://"|"www.")({URLCharacters}{URLEndCharacter})?)
319
320
321%state MLC
322%state MLC2
323%state COMPILER_DIRECTIVE
324%state COMPILER_DIRECTIVE2
325%state EOL_COMMENT
326
327%%
328
329<YYINITIAL> {
330
331 /* Keywords */
332 "array" |
333 "as" |
334 "at" |
335 "asm" |
336 "begin" |
337 "case" |
338 "class" |
339 "const" |
340 "constructor" |
341 "destructor" |
342 "dispinterface" |
343 "div" |
344 "do" |
345 "downto" |
346 "else" |
347 "end" |
348 "except" |
349 "exports" |
350 "file" |
351 "final" |
352 "finalization" |
353 "finally" |
354 "for" |
355 "function" |
356 "goto" |
357 "if" |
358 "implementation" |
359 "in" |
360 "inherited" |
361 "initialization" |
362 "inline" |
363 "interface" |
364 "is" |
365 "label" |
366 "mod" |
367 "not" |
368 "object" |
369 "of" |
370 "on" |
371 "or" |
372 "out" |
373 "packed" |
374 "procedure" |
375 "program" |
376 "property" |
377 "raise" |
378 "record" |
379 "repeat" |
380 "resourcestring" |
381 "set" |
382 "sealed" |
383 "shl" |
384 "shr" |
385 "static" |
386 "string" |
387 "then" |
388 "threadvar" |
389 "to" |
390 "try" |
391 "type" |
392 "unit" |
393 "unsafe" |
394 "until" |
395 "uses" |
396 "var" |
397 "while" |
398 "with" |
399 "xor" { addToken(Token.RESERVED_WORD); }
400
401 /* Directives. */
402 "absolute" |
403 "abstract" |
404 "assembler" |
405 "automated" |
406 "cdecl" |
407 "contains" |
408 "default" |
409 "deprecated" |
410 "dispid" |
411 "dynamic" |
412 "export" |
413 "external" |
414 "far" |
415 "forward" |
416 "implements" |
417 "index" |
418 "library" |
419 "local" |
420 "message" |
421 "name" |
422 "namespaces" |
423 "near" |
424 "nil" |
425 "nodefault" |
426 "overload" |
427 "override" |
428 "package" |
429 "pascal" |
430 "platform" |
431 "private" |
432 "protected" |
433 "public" |
434 "published" |
435 "read" |
436 "readonly" |
437 "register" |
438 "reintroduce" |
439 "requires" |
440 "resident" |
441 "safecall" |
442 "self" |
443 "stdcall" |
444 "stored" |
445 "varargs" |
446 "virtual" |
447 "write" |
448 "writeonly" { addToken(Token.FUNCTION); }
449
450 /* Data types. */
451 "shortint" |
452 "byte" |
453 "char" |
454 "smallint" |
455 "integer" |
456 "word" |
457 "longint" |
458 "cardinal" |
459 "boolean" |
460 "bytebool" |
461 "wordbool" |
462 "longbool" |
463 "real" |
464 "single" |
465 "double" |
466 "extended" |
467 "comp" |
468 "currency" |
469 "pointer" { addToken(Token.DATA_TYPE); }
470
471 {BooleanLiteral} { addToken(Token.LITERAL_BOOLEAN); }
472
473 {LineTerminator} { addNullToken(); return firstToken; }
474
475 {Identifier} { addToken(Token.IDENTIFIER); }
476
477 {WhiteSpace}+ { addToken(Token.WHITESPACE); }
478
479 /* String/Character literals. */
480 {StringLiteral} { addToken(Token.LITERAL_STRING_DOUBLE_QUOTE); }
481 {UnclosedStringLiteral} { addToken(Token.ERROR_STRING_DOUBLE); addNullToken(); return firstToken; }
482 {EscapeSequence} { addToken(Token.PREPROCESSOR); }
483
484 /* Comment literals. */
485 {CompilerDirective1Begin} {start = zzMarkedPos-2; yybegin(COMPILER_DIRECTIVE); }
486 {CompilerDirective2Begin} {start = zzMarkedPos-3; yybegin(COMPILER_DIRECTIVE2); }
487 {MLCBegin} { start = zzMarkedPos-1; yybegin(MLC); }
488 {MLC2Begin} { start = zzMarkedPos-2; yybegin(MLC2); }
489 {LineCommentBegin} { start = zzMarkedPos-2; yybegin(EOL_COMMENT); }
490
491 {Separator} { addToken(Token.SEPARATOR); }
492 {Separator2} { addToken(Token.IDENTIFIER); }
493 {Operator} { addToken(Token.OPERATOR); }
494
495 {IntegerLiteral} { addToken(Token.LITERAL_NUMBER_DECIMAL_INT); }
496 {HexLiteral} { addToken(Token.LITERAL_NUMBER_HEXADECIMAL); }
497 {FloatLiteral} { addToken(Token.LITERAL_NUMBER_FLOAT); }
498 {ErrorNumberFormat} { addToken(Token.ERROR_NUMBER_FORMAT); }
499
500 {ErrorIdentifier} { addToken(Token.ERROR_IDENTIFIER); }
501
502 /* Ended with a line not in a string or comment. */
503 <<EOF>> { addNullToken(); return firstToken; }
504
505 /* Catch any other (unhandled) characters. */
506 . { addToken(Token.IDENTIFIER); }
507
508}
509
510
511<MLC> {
512
513 [^hwf\n\}]+ {}
514 {URL} { int temp=zzStartRead; addToken(start,zzStartRead-1, Token.COMMENT_MULTILINE); addHyperlinkToken(temp,zzMarkedPos-1, Token.COMMENT_MULTILINE); start = zzMarkedPos; }
515 [hwf] {}
516
517 \n { addToken(start,zzStartRead-1, Token.COMMENT_MULTILINE); return firstToken; }
518 {MLCEnd} { yybegin(YYINITIAL); addToken(start,zzStartRead, Token.COMMENT_MULTILINE); }
519 <<EOF>> { addToken(start,zzStartRead-1, Token.COMMENT_MULTILINE); return firstToken; }
520
521}
522
523
524<MLC2> {
525
526 [^hwf\n\*]+ {}
527 {URL} { int temp=zzStartRead; addToken(start,zzStartRead-1, Token.COMMENT_MULTILINE); addHyperlinkToken(temp,zzMarkedPos-1, Token.COMMENT_MULTILINE); start = zzMarkedPos; }
528 [hwf] {}
529
530 \n { addToken(start,zzStartRead-1, Token.COMMENT_MULTILINE); addEndToken(INTERNAL_MLC2); return firstToken; }
531 {MLC2End} { yybegin(YYINITIAL); addToken(start,zzStartRead+1, Token.COMMENT_MULTILINE); }
532 \* {}
533 <<EOF>> { addToken(start,zzStartRead-1, Token.COMMENT_MULTILINE); addEndToken(INTERNAL_MLC2); return firstToken; }
534
535}
536
537
538<COMPILER_DIRECTIVE> {
539 [^\n\}]+ {}
540 \n { addToken(start,zzStartRead-1, Token.PREPROCESSOR); addEndToken(INTERNAL_COMPILER_DIRECTIVE); return firstToken; }
541 {MLCEnd} { yybegin(YYINITIAL); addToken(start,zzStartRead, Token.PREPROCESSOR); }
542 <<EOF>> { addToken(start,zzStartRead-1, Token.PREPROCESSOR); addEndToken(INTERNAL_COMPILER_DIRECTIVE); return firstToken; }
543}
544
545
546<COMPILER_DIRECTIVE2> {
547 [^\n\*]+ {}
548 \n { addToken(start,zzStartRead-1, Token.PREPROCESSOR); addEndToken(INTERNAL_COMPILER_DIRECTIVE2); return firstToken; }
549 {MLC2End} { yybegin(YYINITIAL); addToken(start,zzStartRead+1, Token.PREPROCESSOR); }
550 \* {}
551 <<EOF>> { addToken(start,zzStartRead-1, Token.PREPROCESSOR); addEndToken(INTERNAL_COMPILER_DIRECTIVE2); return firstToken; }
552}
553
554
555
556<EOL_COMMENT> {
557 [^hwf\n]+ {}
558 {URL} { int temp=zzStartRead; addToken(start,zzStartRead-1, Token.COMMENT_EOL); addHyperlinkToken(temp,zzMarkedPos-1, Token.COMMENT_EOL); start = zzMarkedPos; }
559 [hwf] {}
560 \n { addToken(start,zzStartRead-1, Token.COMMENT_EOL); addNullToken(); return firstToken; }
561 <<EOF>> { addToken(start,zzStartRead-1, Token.COMMENT_EOL); addNullToken(); return firstToken; }
562}
Note: See TracBrowser for help on using the repository browser.