source: other-projects/rsyntax-textarea/src/java/org/fife/ui/rsyntaxtextarea/modes/RubyTokenMaker.flex@ 25584

Last change on this file since 25584 was 25584, checked in by davidb, 12 years ago

Initial cut an a text edit area for GLI that supports color syntax highlighting
File size: 26.7 KB
Line 
1/*
2 * 09/11/2008
3 *
4 * RubyTokenMaker.java - Scanner for Ruby
5 *
6 * This library is distributed under a modified BSD license. See the included
7 * RSyntaxTextArea.License.txt file for details.
8 */
9package org.fife.ui.rsyntaxtextarea.modes;
10
11import java.io.*;
12import javax.swing.text.Segment;
13
14import org.fife.ui.rsyntaxtextarea.*;
15
16
17/**
18 * Scanner for Ruby.<p>
19 *
20 * This implementation was created using
21 * <a href="http://www.jflex.de/">JFlex</a> 1.4.1; however, the generated file
22 * was modified for performance. Memory allocation needs to be almost
23 * completely removed to be competitive with the handwritten lexers (subclasses
24 * of <code>AbstractTokenMaker</code>, so this class has been modified so that
25 * Strings are never allocated (via yytext()), and the scanner never has to
26 * worry about refilling its buffer (needlessly copying chars around).
27 * We can achieve this because RText always scans exactly 1 line of tokens at a
28 * time, and hands the scanner this line as an array of characters (a Segment
29 * really). Since tokens contain pointers to char arrays instead of Strings
30 * holding their contents, there is no need for allocating new memory for
31 * Strings.<p>
32 *
33 * The actual algorithm generated for scanning has, of course, not been
34 * modified.<p>
35 *
36 * If you wish to regenerate this file yourself, keep in mind the following:
37 * <ul>
38 * <li>The generated RubyTokenMaker.java</code> file will contain two
39 * definitions of both <code>zzRefill</code> and <code>yyreset</code>.
40 * You should hand-delete the second of each definition (the ones
41 * generated by the lexer), as these generated methods modify the input
42 * buffer, which we'll never have to do.</li>
43 * <li>You should also change the declaration/definition of zzBuffer to NOT
44 * be initialized. This is a needless memory allocation for us since we
45 * will be pointing the array somewhere else anyway.</li>
46 * <li>You should NOT call <code>yylex()</code> on the generated scanner
47 * directly; rather, you should use <code>getTokenList</code> as you would
48 * with any other <code>TokenMaker</code> instance.</li>
49 * </ul>
50 *
51 * @author Robert Futrell
52 * @version 0.5
53 *
54 */
55%%
56
57%public
58%class RubyTokenMaker
59%extends AbstractJFlexTokenMaker
60%unicode
61%type org.fife.ui.rsyntaxtextarea.Token
62
63
64%{
65
66 /**
67 * Token type specific to RubyTokenMaker; this signals that we are inside
68 * an unquoted/double quoted/backtick EOF heredoc.
69 */
70 public static final int INTERNAL_HEREDOC_EOF_UNQUOTED = -1;
71
72 /**
73 * Token type specific to RubyTokenMaker; this signals that we are inside
74 * an single quoted EOF heredoc.
75 */
76 public static final int INTERNAL_HEREDOC_EOF_SINGLE_QUOTED = -2;
77
78 /**
79 * Token type specific to RubyTokenMaker; this signals that we are inside
80 * an double quoted EOF heredoc.
81 */
82 public static final int INTERNAL_HEREDOC_EOF_DOUBLE_QUOTED = -3;
83
84 /**
85 * Token type specific to RubyTokenMaker; this signals that we are inside
86 * an unquoted/double quoted/backtick EOT heredoc.
87 */
88 public static final int INTERNAL_HEREDOC_EOT_UNQUOTED = -4;
89
90 /**
91 * Token type specific to RubyTokenMaker; this signals that we are inside
92 * an single quoted EOT heredoc.
93 */
94 public static final int INTERNAL_HEREDOC_EOT_SINGLE_QUOTED = -5;
95
96 /**
97 * Token type specific to RubyTokenMaker; this signals that we are inside
98 * an double quoted EOT heredoc.
99 */
100 public static final int INTERNAL_HEREDOC_EOT_DOUBLE_QUOTED = -6;
101
102 /**
103 * Token type specific to RubyTokenMaker; this signals that we are inside
104 * a %Q!...! style double quoted string.
105 */
106 public static final int INTERNAL_STRING_Q_BANG = -7;
107
108 /**
109 * Token type specific to RubyTokenMaker; this signals that we are inside
110 * a %Q{...} style double quoted string.
111 */
112 public static final int INTERNAL_STRING_Q_CURLY_BRACE = -8;
113
114 /**
115 * Token type specific to RubyTokenMaker; this signals that we are inside
116 * a %Q&lt;...&gt; style double quoted string.
117 */
118 public static final int INTERNAL_STRING_Q_LT = -9;
119
120
121 /**
122 * Token type specific to RubyTokenMaker; this signals that we are inside
123 * a %Q(...) style double quoted string.
124 */
125 public static final int INTERNAL_STRING_Q_PAREN = -10;
126
127
128 /**
129 * Token type specific to RubyTokenMaker; this signals that we are inside
130 * a %Q/.../ style double quoted string.
131 */
132 public static final int INTERNAL_STRING_Q_SLASH = -11;
133
134 /**
135 * Token type specific to RubyTokenMaker; this signals that we are inside
136 * a %Q[...] style double quoted string.
137 */
138 public static final int INTERNAL_STRING_Q_SQUARE_BRACKET = -12;
139
140
141 /**
142 * Constructor. This must be here because JFlex does not generate a
143 * no-parameter constructor.
144 */
145 public RubyTokenMaker() {
146 }
147
148
149 /**
150 * Adds the token specified to the current linked list of tokens as an
151 * "end token;" that is, at <code>zzMarkedPos</code>.
152 *
153 * @param tokenType The token's type.
154 */
155 private void addEndToken(int tokenType) {
156 addToken(zzMarkedPos,zzMarkedPos, tokenType);
157 }
158
159
160 /**
161 * Adds the token specified to the current linked list of tokens.
162 *
163 * @param tokenType The token's type.
164 */
165 private void addToken(int tokenType) {
166 addToken(zzStartRead, zzMarkedPos-1, tokenType);
167 }
168
169
170 /**
171 * Adds the token specified to the current linked list of tokens.
172 *
173 * @param tokenType The token's type.
174 */
175 private void addToken(int start, int end, int tokenType) {
176 int so = start + offsetShift;
177 addToken(zzBuffer, start,end, tokenType, so);
178 }
179
180
181 /**
182 * Adds the token specified to the current linked list of tokens.
183 *
184 * @param array The character array.
185 * @param start The starting offset in the array.
186 * @param end The ending offset in the array.
187 * @param tokenType The token's type.
188 * @param startOffset The offset in the document at which this token
189 * occurs.
190 */
191 public void addToken(char[] array, int start, int end, int tokenType, int startOffset) {
192 super.addToken(array, start,end, tokenType, startOffset);
193 zzStartRead = zzMarkedPos;
194 }
195
196
197 /**
198 * Returns the text to place at the beginning and end of a
199 * line to "comment" it in a this programming language.
200 *
201 * @return The start and end strings to add to a line to "comment"
202 * it out.
203 */
204 public String[] getLineCommentStartAndEnd() {
205 return new String[] { "#", null };
206 }
207
208
209 /**
210 * Returns whether tokens of the specified type should have "mark
211 * occurrences" enabled for the current programming language.
212 *
213 * @param type The token type.
214 * @return Whether tokens of this type should have "mark occurrences"
215 * enabled.
216 */
217 public boolean getMarkOccurrencesOfTokenType(int type) {
218 return type==Token.IDENTIFIER || type==Token.VARIABLE;
219 }
220
221
222 /**
223 * Returns the first token in the linked list of tokens generated
224 * from <code>text</code>. This method must be implemented by
225 * subclasses so they can correctly implement syntax highlighting.
226 *
227 * @param text The text from which to get tokens.
228 * @param initialTokenType The token type we should start with.
229 * @param startOffset The offset into the document at which
230 * <code>text</code> starts.
231 * @return The first <code>Token</code> in a linked list representing
232 * the syntax highlighted text.
233 */
234 public Token getTokenList(Segment text, int initialTokenType, int startOffset) {
235
236 resetTokenList();
237 this.offsetShift = -text.offset + startOffset;
238
239 // Start off in the proper state.
240 int state = Token.NULL;
241 switch (initialTokenType) {
242 case Token.COMMENT_DOCUMENTATION:
243 state = DOCCOMMENT;
244 start = text.offset;
245 break;
246 case Token.LITERAL_STRING_DOUBLE_QUOTE:
247 state = STRING;
248 start = text.offset;
249 break;
250 case Token.LITERAL_CHAR:
251 state = CHAR_LITERAL;
252 start = text.offset;
253 break;
254 case Token.LITERAL_BACKQUOTE:
255 state = BACKTICKS;
256 start = text.offset;
257 break;
258 case INTERNAL_HEREDOC_EOF_UNQUOTED:
259 state = HEREDOC_EOF_UNQUOTED;
260 start = text.offset;
261 break;
262 case INTERNAL_HEREDOC_EOF_SINGLE_QUOTED:
263 state = HEREDOC_EOF_SINGLE_QUOTED;
264 start = text.offset;
265 break;
266 case INTERNAL_HEREDOC_EOF_DOUBLE_QUOTED:
267 state = HEREDOC_EOF_DOUBLE_QUOTED;
268 start = text.offset;
269 break;
270 case INTERNAL_HEREDOC_EOT_UNQUOTED:
271 state = HEREDOC_EOT_UNQUOTED;
272 start = text.offset;
273 break;
274 case INTERNAL_HEREDOC_EOT_SINGLE_QUOTED:
275 state = HEREDOC_EOT_SINGLE_QUOTED;
276 start = text.offset;
277 break;
278 case INTERNAL_HEREDOC_EOT_DOUBLE_QUOTED:
279 state = HEREDOC_EOT_DOUBLE_QUOTED;
280 start = text.offset;
281 break;
282 case INTERNAL_STRING_Q_BANG:
283 state = STRING_Q_BANG;
284 start = text.offset;
285 break;
286 case INTERNAL_STRING_Q_CURLY_BRACE:
287 state = STRING_Q_CURLY_BRACE;
288 start = text.offset;
289 break;
290 case INTERNAL_STRING_Q_LT:
291 state = STRING_Q_LT;
292 start = text.offset;
293 break;
294 case INTERNAL_STRING_Q_PAREN:
295 state = STRING_Q_PAREN;
296 start = text.offset;
297 break;
298 case INTERNAL_STRING_Q_SLASH:
299 state = STRING_Q_SLASH;
300 start = text.offset;
301 break;
302 case INTERNAL_STRING_Q_SQUARE_BRACKET:
303 state = STRING_Q_SQUARE_BRACKET;
304 start = text.offset;
305 break;
306 default:
307 state = Token.NULL;
308 }
309
310 s = text;
311 try {
312 yyreset(zzReader);
313 yybegin(state);
314 return yylex();
315 } catch (IOException ioe) {
316 ioe.printStackTrace();
317 return new DefaultToken();
318 }
319
320 }
321
322
323 /**
324 * Refills the input buffer.
325 *
326 * @return <code>true</code> if EOF was reached, otherwise
327 * <code>false</code>.
328 * @exception IOException if any I/O-Error occurs.
329 */
330 private boolean zzRefill() throws java.io.IOException {
331 return zzCurrentPos>=s.offset+s.count;
332 }
333
334
335 /**
336 * Resets the scanner to read from a new input stream.
337 * Does not close the old reader.
338 *
339 * All internal variables are reset, the old input stream
340 * <b>cannot</b> be reused (internal buffer is discarded and lost).
341 * Lexical state is set to <tt>YY_INITIAL</tt>.
342 *
343 * @param reader the new input stream
344 */
345 public final void yyreset(java.io.Reader reader) throws java.io.IOException {
346 // 's' has been updated.
347 zzBuffer = s.array;
348 /*
349 * We replaced the line below with the two below it because zzRefill
350 * no longer "refills" the buffer (since the way we do it, it's always
351 * "full" the first time through, since it points to the segment's
352 * array). So, we assign zzEndRead here.
353 */
354 //zzStartRead = zzEndRead = s.offset;
355 zzStartRead = s.offset;
356 zzEndRead = zzStartRead + s.count - 1;
357 zzCurrentPos = zzMarkedPos = zzPushbackPos = s.offset;
358 zzLexicalState = YYINITIAL;
359 zzReader = reader;
360 zzAtBOL = true;
361 zzAtEOF = false;
362 }
363
364
365%}
366
367Letter = [A-Za-z]
368NonzeroDigit = [1-9]
369Digit = ("0"|{NonzeroDigit})
370BinaryDigit = ([01])
371HexDigit = ({Digit}|[A-Fa-f])
372OctalDigit = ([0-7])
373NonSeparator = ([^\t\f\r\n\ \(\)\{\}\[\]\;\,\.\=\>\<\!\~\?\:\+\-\*\/\&\|\^\%\"\'\`]|"#"|"\\")
374
375IdentifierStart = ({Letter}|"_")
376IdentifierPart = ({IdentifierStart}|{Digit})
377BooleanLiteral = ("true"|"false")
378
379LineTerminator = (\n)
380WhiteSpace = ([ \t\f])
381
382LineCommentBegin = "#"
383DocCommentBegin = "=begin"
384DocCommentEnd = "=end"
385
386DigitOrUnderscore = ({Digit}|[_])
387BinaryIntLiteral = ("0b"{BinaryDigit}([01_]*{BinaryDigit})?)
388OctalLiteral = ("0"([0-7_]*{OctalDigit})?)
389DecimalLiteral1 = ("0d"{Digit}({DigitOrUnderscore}*{Digit})?)
390DecimalLiteral2 = ({NonzeroDigit}({DigitOrUnderscore}*{Digit})?)
391DecimalLiteral = ({BinaryIntLiteral}|{OctalLiteral}|{DecimalLiteral1}|{DecimalLiteral2})
392HexLiteral = ("0x"{HexDigit}([0-9a-zA-Z_]*{HexDigit})?)
393FloatLiteral = ({NonzeroDigit}({DigitOrUnderscore}*{Digit})?[Ee]({Digit}({DigitOrUnderscore}*{Digit})?)?)
394
395Separator = ([\(\)\{\}])
396Operator1 = ("::"|"."|"["|"]"|"-"|"+"|"!"|"~"|"*"|"/"|"%"|"<<"|">>"|"&"|"|"|"^")
397Operator2 = (">"|">="|"<"|"<="|"<=>"|"=="|"==="|"!="|"=~"|"!~"|"&&"|"||")
398Operator3 = (".."|"..."|"="|"+="|"-="|"*="|"/="|"%=")
399Operator = ({Operator1}|{Operator2}|{Operator3})
400
401Identifier = ({IdentifierStart}{IdentifierPart}*)
402Symbol = ([:]{Identifier})
403ErrorIdentifier = ({NonSeparator}+)
404
405PreDefinedVariable = ("$"([!@&`\'+0-9~=/\,;.<>_*$?:\"]|"DEBUG"|"FILENAME"|"LOAD_PATH"|"stderr"|"stdin"|"stdout"|"VERBOSE"|([\-][0adFiIlpwv])))
406Variable = ({PreDefinedVariable}|([@][@]?|[$]){Identifier})
407
408%state STRING
409%state STRING_Q_BANG
410%state STRING_Q_CURLY_BRACE
411%state STRING_Q_PAREN
412%state STRING_Q_SLASH
413%state STRING_Q_SQUARE_BRACKET
414%state STRING_Q_LT
415%state CHAR_LITERAL
416%state BACKTICKS
417%state HEREDOC_EOF_UNQUOTED
418%state HEREDOC_EOF_SINGLE_QUOTED
419%state HEREDOC_EOF_DOUBLE_QUOTED
420%state HEREDOC_EOT_UNQUOTED
421%state HEREDOC_EOT_SINGLE_QUOTED
422%state HEREDOC_EOT_DOUBLE_QUOTED
423%state DOCCOMMENT
424
425%%
426
427<YYINITIAL> {
428
429 /* Keywords */
430 "alias" |
431 "BEGIN" |
432 "begin" |
433 "break" |
434 "case" |
435 "class" |
436 "def" |
437 "defined" |
438 "do" |
439 "else" |
440 "elsif" |
441 "END" |
442 "end" |
443 "ensure" |
444 "for" |
445 "if" |
446 "in" |
447 "module" |
448 "next" |
449 "nil" |
450 "redo" |
451 "rescue" |
452 "retry" |
453 "return" |
454 "self" |
455 "super" |
456 "then" |
457 "undef" |
458 "unless" |
459 "until" |
460 "when" |
461 "while" |
462 "yield" { addToken(Token.RESERVED_WORD); }
463
464 "Array" |
465 "Float" |
466 "Integer" |
467 "String" |
468 "at_exit" |
469 "autoload" |
470 "binding" |
471 "caller" |
472 "catch" |
473 "chop" |
474 "chop!" |
475 "chomp" |
476 "chomp!" |
477 "eval" |
478 "exec" |
479 "exit" |
480 "exit!" |
481 "fail" |
482 "fork" |
483 "format" |
484 "gets" |
485 "global_variables" |
486 "gsub" |
487 "gsub!" |
488 "iterator?" |
489 "lambda" |
490 "load" |
491 "local_variables" |
492 "loop" |
493 "open" |
494 "p" |
495 "print" |
496 "printf" |
497 "proc" |
498 "putc" |
499 "puts" |
500 "raise" |
501 "rand" |
502 "readline" |
503 "readlines" |
504 "require" |
505 "select" |
506 "sleep" |
507 "split" |
508 "sprintf" |
509 "srand" |
510 "sub" |
511 "sub!" |
512 "syscall" |
513 "system" |
514 "test" |
515 "trace_var" |
516 "trap" |
517 "untrace_var" { addToken(Token.FUNCTION); }
518
519 "and" |
520 "or" |
521 "not" { addToken(Token.OPERATOR); }
522
523 {BooleanLiteral} { addToken(Token.LITERAL_BOOLEAN); }
524
525 {Variable} { addToken(Token.VARIABLE); }
526 {Symbol} { addToken(Token.PREPROCESSOR); }
527
528 {LineTerminator} { addNullToken(); return firstToken; }
529 {Identifier} { addToken(Token.IDENTIFIER); }
530 {WhiteSpace}+ { addToken(Token.WHITESPACE); }
531
532 /* String/Character literals. */
533 \" { start = zzMarkedPos-1; yybegin(STRING); }
534 \' { start = zzMarkedPos-1; yybegin(CHAR_LITERAL); }
535 [\%][QqWwx]?[\(] { start = zzMarkedPos-yylength(); yybegin(STRING_Q_PAREN); }
536 [\%][QqWwx]?[\{] { start = zzMarkedPos-yylength(); yybegin(STRING_Q_CURLY_BRACE); }
537 [\%][QqWwx]?[\[] { start = zzMarkedPos-yylength(); yybegin(STRING_Q_SQUARE_BRACKET); }
538 [\%][QqWwx]?[\<] { start = zzMarkedPos-yylength(); yybegin(STRING_Q_LT); }
539 [\%][QqWwx]?[\!] { start = zzMarkedPos-yylength(); yybegin(STRING_Q_BANG); }
540 [\%][QqWwx]?[\/] { start = zzMarkedPos-yylength(); yybegin(STRING_Q_SLASH); }
541 \` { start = zzMarkedPos-1; yybegin(BACKTICKS); }
542
543 /* Comment literals. */
544 {LineCommentBegin}.* { addToken(Token.COMMENT_EOL); addNullToken(); return firstToken; }
545 {DocCommentBegin} { start = zzMarkedPos-6; yybegin(DOCCOMMENT); }
546
547 /* "Here-document" syntax. This is only implemented for the common */
548 /* cases. */
549 "<<EOF" { start = zzStartRead; yybegin(HEREDOC_EOF_UNQUOTED); }
550 "<<" {WhiteSpace}* \""EOF"\" { start = zzStartRead; yybegin(HEREDOC_EOF_UNQUOTED); }
551 "<<" {WhiteSpace}* \'"EOF"\' { start = zzStartRead; yybegin(HEREDOC_EOF_SINGLE_QUOTED); }
552 "<<" {WhiteSpace}* \`"EOF"\` { start = zzStartRead; yybegin(HEREDOC_EOF_UNQUOTED); }
553 "<<EOT" { start = zzStartRead; yybegin(HEREDOC_EOT_UNQUOTED); }
554 "<<" {WhiteSpace}* \""EOT"\" { start = zzStartRead; yybegin(HEREDOC_EOT_UNQUOTED); }
555 "<<" {WhiteSpace}* \'"EOT"\' { start = zzStartRead; yybegin(HEREDOC_EOT_SINGLE_QUOTED); }
556 "<<" {WhiteSpace}* \`"EOT"\` { start = zzStartRead; yybegin(HEREDOC_EOT_UNQUOTED); }
557
558 /* Separators and operators. */
559 {Separator} { addToken(Token.SEPARATOR); }
560 {Operator} { addToken(Token.OPERATOR); }
561
562 /* Numbers */
563 {DecimalLiteral} { addToken(Token.LITERAL_NUMBER_DECIMAL_INT); }
564 {HexLiteral} { addToken(Token.LITERAL_NUMBER_HEXADECIMAL); }
565 {FloatLiteral} { addToken(Token.LITERAL_NUMBER_FLOAT); }
566
567 {ErrorIdentifier} { addToken(Token.ERROR_IDENTIFIER); }
568
569 /* Ended with a line not in a string or comment. */
570 <<EOF>> { addNullToken(); return firstToken; }
571
572 /* Catch any other (unhandled) characters. */
573 . { addToken(Token.IDENTIFIER); }
574
575}
576
577
578<STRING> {
579 [^\n\\\"]+ {}
580 \n { addToken(start,zzStartRead-1, Token.LITERAL_STRING_DOUBLE_QUOTE); return firstToken; }
581 \\.? { /* Skip escaped chars. */ }
582 \" { yybegin(YYINITIAL); addToken(start,zzStartRead, Token.LITERAL_STRING_DOUBLE_QUOTE); }
583 <<EOF>> { addToken(start,zzStartRead-1, Token.LITERAL_STRING_DOUBLE_QUOTE); return firstToken; }
584}
585
586
587<STRING_Q_BANG> {
588 [^\n\\\!]+ {}
589 \n { addToken(start,zzStartRead-1, Token.LITERAL_STRING_DOUBLE_QUOTE); addEndToken(INTERNAL_STRING_Q_BANG); return firstToken; }
590 \\.? { /* Skip escaped chars. */ }
591 \! { yybegin(YYINITIAL); addToken(start,zzStartRead, Token.LITERAL_STRING_DOUBLE_QUOTE); }
592 <<EOF>> { addToken(start,zzStartRead-1, Token.LITERAL_STRING_DOUBLE_QUOTE); addEndToken(INTERNAL_STRING_Q_BANG); return firstToken; }
593}
594
595
596<STRING_Q_CURLY_BRACE> {
597 [^\n\\\}]+ {}
598 \n { addToken(start,zzStartRead-1, Token.LITERAL_STRING_DOUBLE_QUOTE); addEndToken(INTERNAL_STRING_Q_CURLY_BRACE); return firstToken; }
599 \\.? { /* Skip escaped chars. */ }
600 \} { yybegin(YYINITIAL); addToken(start,zzStartRead, Token.LITERAL_STRING_DOUBLE_QUOTE); }
601 <<EOF>> { addToken(start,zzStartRead-1, Token.LITERAL_STRING_DOUBLE_QUOTE); addEndToken(INTERNAL_STRING_Q_CURLY_BRACE); return firstToken; }
602}
603
604
605<STRING_Q_LT> {
606 [^\n\\\>]+ {}
607 \n { addToken(start,zzStartRead-1, Token.LITERAL_STRING_DOUBLE_QUOTE); addEndToken(INTERNAL_STRING_Q_LT); return firstToken; }
608 \\.? { /* Skip escaped chars. */ }
609 \> { yybegin(YYINITIAL); addToken(start,zzStartRead, Token.LITERAL_STRING_DOUBLE_QUOTE); }
610 <<EOF>> { addToken(start,zzStartRead-1, Token.LITERAL_STRING_DOUBLE_QUOTE); addEndToken(INTERNAL_STRING_Q_LT); return firstToken; }
611}
612
613
614<STRING_Q_PAREN> {
615 [^\n\\\)]+ {}
616 \n { addToken(start,zzStartRead-1, Token.LITERAL_STRING_DOUBLE_QUOTE); addEndToken(INTERNAL_STRING_Q_PAREN); return firstToken; }
617 \\.? { /* Skip escaped chars. */ }
618 \) { yybegin(YYINITIAL); addToken(start,zzStartRead, Token.LITERAL_STRING_DOUBLE_QUOTE); }
619 <<EOF>> { addToken(start,zzStartRead-1, Token.LITERAL_STRING_DOUBLE_QUOTE); addEndToken(INTERNAL_STRING_Q_PAREN); return firstToken; }
620}
621
622
623<STRING_Q_SLASH> {
624 [^\n\\\/]+ {}
625 \n { addToken(start,zzStartRead-1, Token.LITERAL_STRING_DOUBLE_QUOTE); addEndToken(INTERNAL_STRING_Q_SLASH); return firstToken; }
626 \\.? { /* Skip escaped chars. */ }
627 \/ { yybegin(YYINITIAL); addToken(start,zzStartRead, Token.LITERAL_STRING_DOUBLE_QUOTE); }
628 <<EOF>> { addToken(start,zzStartRead-1, Token.LITERAL_STRING_DOUBLE_QUOTE); addEndToken(INTERNAL_STRING_Q_SLASH); return firstToken; }
629}
630
631
632<STRING_Q_SQUARE_BRACKET> {
633 [^\n\\\]]+ {}
634 \n { addToken(start,zzStartRead-1, Token.LITERAL_STRING_DOUBLE_QUOTE); addEndToken(INTERNAL_STRING_Q_SQUARE_BRACKET); return firstToken; }
635 \\.? { /* Skip escaped chars. */ }
636 \] { yybegin(YYINITIAL); addToken(start,zzStartRead, Token.LITERAL_STRING_DOUBLE_QUOTE); }
637 <<EOF>> { addToken(start,zzStartRead-1, Token.LITERAL_STRING_DOUBLE_QUOTE); addEndToken(INTERNAL_STRING_Q_SQUARE_BRACKET); return firstToken; }
638}
639
640
641<CHAR_LITERAL> {
642 [^\n\\\']+ {}
643 \\.? { /* Skip escaped single quotes only, but this should still work. */ }
644 \n { addToken(start,zzStartRead-1, Token.LITERAL_CHAR); return firstToken; }
645 \' { yybegin(YYINITIAL); addToken(start,zzStartRead, Token.LITERAL_CHAR); }
646 <<EOF>> { addToken(start,zzStartRead-1, Token.LITERAL_CHAR); return firstToken; }
647}
648
649
650<BACKTICKS> {
651 [^\n\\\`]+ {}
652 \n { addToken(start,zzStartRead-1, Token.LITERAL_BACKQUOTE); return firstToken; }
653 \\.? { /* Skip escaped chars. */ }
654 \` { yybegin(YYINITIAL); addToken(start,zzStartRead, Token.LITERAL_BACKQUOTE); }
655 <<EOF>> { addToken(start,zzStartRead-1, Token.LITERAL_BACKQUOTE); return firstToken; }
656}
657
658
659<HEREDOC_EOF_UNQUOTED> {
660 /* NOTE: The closing "EOF" is supposed to be on a line by itself - */
661 /* no surrounding whitespace or other chars. However, the way */
662 /* we're hacking the JFLex scanning, something like ^"EOF"$ doesn't */
663 /* work. Fortunately we don't need the start- and end-line anchors */
664 /* since the production after "EOF" will match any line containing */
665 /* EOF and any other chars. */
666 /* NOTE2: This case is used for unquoted <<EOF, double quoted */
667 /* <<"EOF" and backticks <<`EOF`, since they all follow the same */
668 /* syntactic rules. */
669 "EOF" { if (start==zzStartRead) { addToken(Token.PREPROCESSOR); addNullToken(); return firstToken; } }
670 [^\n\\]+ {}
671 \n { addToken(start,zzStartRead-1, Token.PREPROCESSOR); addEndToken(INTERNAL_HEREDOC_EOF_UNQUOTED); return firstToken; }
672 \\.? { /* Skip escaped chars. */ }
673 <<EOF>> { addToken(start,zzStartRead-1, Token.PREPROCESSOR); addEndToken(INTERNAL_HEREDOC_EOF_UNQUOTED); return firstToken; }
674}
675
676
677<HEREDOC_EOF_SINGLE_QUOTED> {
678 /* NOTE: The closing "EOF" is supposed to be on a line by itself - */
679 /* no surrounding whitespace or other chars. However, the way */
680 /* we're hacking the JFLex scanning, something like ^"EOF"$ doesn't */
681 /* work. Fortunately we don't need the start- and end-line anchors */
682 /* since the production after "EOF" will match any line containing */
683 /* EOF and any other chars. */
684 "EOF" { if (start==zzStartRead) { addToken(Token.PREPROCESSOR); addNullToken(); return firstToken; } }
685 [^\n\\]+ {}
686 \n { addToken(start,zzStartRead-1, Token.PREPROCESSOR); addEndToken(INTERNAL_HEREDOC_EOF_SINGLE_QUOTED); return firstToken; }
687 \\.? { /* Skip escaped chars. */ }
688 <<EOF>> { addToken(start,zzStartRead-1, Token.PREPROCESSOR); addEndToken(INTERNAL_HEREDOC_EOF_SINGLE_QUOTED); return firstToken; }
689}
690
691
692<HEREDOC_EOF_DOUBLE_QUOTED> {
693 /* NOTE: The closing "EOF" is supposed to be on a line by itself - */
694 /* no surrounding whitespace or other chars. However, the way */
695 /* we're hacking the JFLex scanning, something like ^"EOF"$ doesn't */
696 /* work. Fortunately we don't need the start- and end-line anchors */
697 /* since the production after "EOF" will match any line containing */
698 /* EOF and any other chars. */
699 "EOF" { if (start==zzStartRead) { addToken(Token.PREPROCESSOR); addNullToken(); return firstToken; } }
700 [^\n\\]+ {}
701 \n { addToken(start,zzStartRead-1, Token.PREPROCESSOR); addEndToken(INTERNAL_HEREDOC_EOF_SINGLE_QUOTED); return firstToken; }
702 \\.? { /* Skip escaped chars. */ }
703 <<EOF>> { addToken(start,zzStartRead-1, Token.PREPROCESSOR); addEndToken(INTERNAL_HEREDOC_EOF_SINGLE_QUOTED); return firstToken; }
704}
705
706
707<HEREDOC_EOT_UNQUOTED> {
708 /* NOTE: The closing "EOT" is supposed to be on a line by itself - */
709 /* no surrounding whitespace or other chars. However, the way */
710 /* we're hacking the JFLex scanning, something like ^"EOT"$ doesn't */
711 /* work. Fortunately we don't need the start- and end-line anchors */
712 /* since the production after "EOT" will match any line containing */
713 /* EOF and any other chars. */
714 /* NOTE2: This case is used for unquoted <<EOT, double quoted */
715 /* <<"EOT" and backticks <<`EOT`, since they all follow the same */
716 /* syntactic rules. */
717 "EOT" { if (start==zzStartRead) { addToken(Token.PREPROCESSOR); addNullToken(); return firstToken; } }
718 [^\n\\]+ {}
719 \n { addToken(start,zzStartRead-1, Token.PREPROCESSOR); addEndToken(INTERNAL_HEREDOC_EOT_UNQUOTED); return firstToken; }
720 \\.? { /* Skip escaped chars. */ }
721 <<EOF>> { addToken(start,zzStartRead-1, Token.PREPROCESSOR); addEndToken(INTERNAL_HEREDOC_EOT_UNQUOTED); return firstToken; }
722}
723
724
725<HEREDOC_EOT_SINGLE_QUOTED> {
726 /* NOTE: The closing "EOT" is supposed to be on a line by itself - */
727 /* no surrounding whitespace or other chars. However, the way */
728 /* we're hacking the JFLex scanning, something like ^"EOT"$ doesn't */
729 /* work. Fortunately we don't need the start- and end-line anchors */
730 /* since the production after "EOT" will match any line containing */
731 /* EOT and any other chars. */
732 "EOT" { if (start==zzStartRead) { addToken(Token.PREPROCESSOR); addNullToken(); return firstToken; } }
733 [^\n\\]+ {}
734 \n { addToken(start,zzStartRead-1, Token.PREPROCESSOR); addEndToken(INTERNAL_HEREDOC_EOT_SINGLE_QUOTED); return firstToken; }
735 \\.? { /* Skip escaped chars. */ }
736 <<EOF>> { addToken(start,zzStartRead-1, Token.PREPROCESSOR); addEndToken(INTERNAL_HEREDOC_EOT_SINGLE_QUOTED); return firstToken; }
737}
738
739
740<HEREDOC_EOT_DOUBLE_QUOTED> {
741 /* NOTE: The closing "EOT" is supposed to be on a line by itself - */
742 /* no surrounding whitespace or other chars. However, the way */
743 /* we're hacking the JFLex scanning, something like ^"EOT"$ doesn't */
744 /* work. Fortunately we don't need the start- and end-line anchors */
745 /* since the production after "EOT" will match any line containing */
746 /* EOT and any other chars. */
747 "EOT" { if (start==zzStartRead) { addToken(Token.PREPROCESSOR); addNullToken(); return firstToken; } }
748 [^\n\\]+ {}
749 \n { addToken(start,zzStartRead-1, Token.PREPROCESSOR); addEndToken(INTERNAL_HEREDOC_EOT_SINGLE_QUOTED); return firstToken; }
750 \\.? { /* Skip escaped chars. */ }
751 <<EOF>> { addToken(start,zzStartRead-1, Token.PREPROCESSOR); addEndToken(INTERNAL_HEREDOC_EOT_SINGLE_QUOTED); return firstToken; }
752}
753
754
755<DOCCOMMENT> {
756
757 [^\n\=]+ {}
758 {DocCommentEnd} { yybegin(YYINITIAL); addToken(start,zzStartRead+3, Token.COMMENT_DOCUMENTATION); }
759 = {}
760 \n { addToken(start,zzStartRead-1, Token.COMMENT_DOCUMENTATION); return firstToken; }
761 <<EOF>> { yybegin(YYINITIAL); addToken(start,zzEndRead, Token.COMMENT_DOCUMENTATION); return firstToken; }
762
763}
Note: See TracBrowser for help on using the repository browser.