source: other-projects/rsyntax-textarea/src/java/org/fife/ui/rsyntaxtextarea/modes/LuaTokenMaker.flex@ 25584

Last change on this file since 25584 was 25584, checked in by davidb, 12 years ago

Initial cut an a text edit area for GLI that supports color syntax highlighting
File size: 12.5 KB
Line 
1/*
2 * 07/14/2006
3 *
4 * LuaTokenMaker.java - Scanner for the Lua programming language.
5 *
6 * This library is distributed under a modified BSD license. See the included
7 * RSyntaxTextArea.License.txt file for details.
8 */
9package org.fife.ui.rsyntaxtextarea.modes;
10
11import java.io.*;
12import javax.swing.text.Segment;
13
14import org.fife.ui.rsyntaxtextarea.*;
15
16
17/**
18 * Scanner for the Lua programming language.<p>
19 *
20 * This implementation was created using
21 * <a href="http://www.jflex.de/">JFlex</a> 1.4.1; however, the generated file
22 * was modified for performance. Memory allocation needs to be almost
23 * completely removed to be competitive with the handwritten lexers (subclasses
24 * of <code>AbstractTokenMaker</code>, so this class has been modified so that
25 * Strings are never allocated (via yytext()), and the scanner never has to
26 * worry about refilling its buffer (needlessly copying chars around).
27 * We can achieve this because RText always scans exactly 1 line of tokens at a
28 * time, and hands the scanner this line as an array of characters (a Segment
29 * really). Since tokens contain pointers to char arrays instead of Strings
30 * holding their contents, there is no need for allocating new memory for
31 * Strings.<p>
32 *
33 * The actual algorithm generated for scanning has, of course, not been
34 * modified.<p>
35 *
36 * If you wish to regenerate this file yourself, keep in mind the following:
37 * <ul>
38 * <li>The generated <code>LuaTokenMaker.java</code> file will contain two
39 * definitions of both <code>zzRefill</code> and <code>yyreset</code>.
40 * You should hand-delete the second of each definition (the ones
41 * generated by the lexer), as these generated methods modify the input
42 * buffer, which we'll never have to do.</li>
43 * <li>You should also change the declaration/definition of zzBuffer to NOT
44 * be initialized. This is a needless memory allocation for us since we
45 * will be pointing the array somewhere else anyway.</li>
46 * <li>You should NOT call <code>yylex()</code> on the generated scanner
47 * directly; rather, you should use <code>getTokenList</code> as you would
48 * with any other <code>TokenMaker</code> instance.</li>
49 * </ul>
50 *
51 * @author Robert Futrell
52 * @version 0.4
53 *
54 */
55%%
56
57%public
58%class LuaTokenMaker
59%extends AbstractJFlexTokenMaker
60%implements TokenMaker
61%unicode
62%type org.fife.ui.rsyntaxtextarea.Token
63
64
65%{
66
67
68 /**
69 * Constructor. This must be here because JFlex does not generate a
70 * no-parameter constructor.
71 */
72 public LuaTokenMaker() {
73 }
74
75
76 /**
77 * Adds the token specified to the current linked list of tokens.
78 *
79 * @param tokenType The token's type.
80 */
81 private void addToken(int tokenType) {
82 addToken(zzStartRead, zzMarkedPos-1, tokenType);
83 }
84
85
86 /**
87 * Adds the token specified to the current linked list of tokens.
88 *
89 * @param tokenType The token's type.
90 */
91 private void addToken(int start, int end, int tokenType) {
92 int so = start + offsetShift;
93 addToken(zzBuffer, start,end, tokenType, so);
94 }
95
96
97 /**
98 * Adds the token specified to the current linked list of tokens.
99 *
100 * @param array The character array.
101 * @param start The starting offset in the array.
102 * @param end The ending offset in the array.
103 * @param tokenType The token's type.
104 * @param startOffset The offset in the document at which this token
105 * occurs.
106 */
107 public void addToken(char[] array, int start, int end, int tokenType, int startOffset) {
108 super.addToken(array, start,end, tokenType, startOffset);
109 zzStartRead = zzMarkedPos;
110 }
111
112
113 /**
114 * Returns the text to place at the beginning and end of a
115 * line to "comment" it in a this programming language.
116 *
117 * @return The start and end strings to add to a line to "comment"
118 * it out.
119 */
120 public String[] getLineCommentStartAndEnd() {
121 return new String[] { "--", null };
122 }
123
124
125 /**
126 * Returns the first token in the linked list of tokens generated
127 * from <code>text</code>. This method must be implemented by
128 * subclasses so they can correctly implement syntax highlighting.
129 *
130 * @param text The text from which to get tokens.
131 * @param initialTokenType The token type we should start with.
132 * @param startOffset The offset into the document at which
133 * <code>text</code> starts.
134 * @return The first <code>Token</code> in a linked list representing
135 * the syntax highlighted text.
136 */
137 public Token getTokenList(Segment text, int initialTokenType, int startOffset) {
138
139 resetTokenList();
140 this.offsetShift = -text.offset + startOffset;
141
142 // Start off in the proper state.
143 int state = Token.NULL;
144 switch (initialTokenType) {
145 case Token.COMMENT_MULTILINE:
146 state = MLC;
147 start = text.offset;
148 break;
149 case Token.LITERAL_STRING_DOUBLE_QUOTE:
150 state = LONGSTRING;
151 start = text.offset;
152 break;
153 default:
154 state = Token.NULL;
155 }
156
157 s = text;
158 try {
159 yyreset(zzReader);
160 yybegin(state);
161 return yylex();
162 } catch (IOException ioe) {
163 ioe.printStackTrace();
164 return new DefaultToken();
165 }
166
167 }
168
169
170 /**
171 * Refills the input buffer.
172 *
173 * @return <code>true</code> if EOF was reached, otherwise
174 * <code>false</code>.
175 * @exception IOException if any I/O-Error occurs.
176 */
177 private boolean zzRefill() throws java.io.IOException {
178 return zzCurrentPos>=s.offset+s.count;
179 }
180
181
182 /**
183 * Resets the scanner to read from a new input stream.
184 * Does not close the old reader.
185 *
186 * All internal variables are reset, the old input stream
187 * <b>cannot</b> be reused (internal buffer is discarded and lost).
188 * Lexical state is set to <tt>YY_INITIAL</tt>.
189 *
190 * @param reader the new input stream
191 */
192 public final void yyreset(java.io.Reader reader) throws java.io.IOException {
193 // 's' has been updated.
194 zzBuffer = s.array;
195 /*
196 * We replaced the line below with the two below it because zzRefill
197 * no longer "refills" the buffer (since the way we do it, it's always
198 * "full" the first time through, since it points to the segment's
199 * array). So, we assign zzEndRead here.
200 */
201 //zzStartRead = zzEndRead = s.offset;
202 zzStartRead = s.offset;
203 zzEndRead = zzStartRead + s.count - 1;
204 zzCurrentPos = zzMarkedPos = zzPushbackPos = s.offset;
205 zzLexicalState = YYINITIAL;
206 zzReader = reader;
207 zzAtBOL = true;
208 zzAtEOF = false;
209 }
210
211
212%}
213
214Letter = [A-Za-z_]
215Digit = [0-9]
216
217LineTerminator = (\n)
218WhiteSpace = ([ \t\f])
219
220UnclosedCharLiteral = ([\']([^\'\n]|"\\'")*)
221CharLiteral = ({UnclosedCharLiteral}"'")
222UnclosedStringLiteral = ([\"]([^\"\n]|"\\\"")*)
223StringLiteral = ({UnclosedStringLiteral}[\"])
224LongStringBegin = ("[[")
225LongStringEnd = ("]]")
226
227LineCommentBegin = ("--")
228MLCBegin = ({LineCommentBegin}{LongStringBegin})
229
230Number = ( "."? {Digit} ({Digit}|".")* ([eE][+-]?)? ({Letter}|{Digit})* )
231BooleanLiteral = ("true"|"false")
232
233Separator = ([\(\)\{\}\[\]\]])
234Separator2 = ([\;:,.])
235
236ArithmeticOperator = ("+"|"-"|"*"|"/"|"^"|"%")
237RelationalOperator = ("<"|">"|"<="|">="|"=="|"~=")
238LogicalOperator = ("and"|"or"|"not"|"#")
239ConcatenationOperator = ("..")
240Elipsis = ({ConcatenationOperator}".")
241Operator = ({ArithmeticOperator}|{RelationalOperator}|{LogicalOperator}|{ConcatenationOperator}|{Elipsis})
242
243Identifier = ({Letter}({Letter}|{Digit})*)
244
245
246%state MLC
247%state LONGSTRING
248%state LINECOMMENT
249
250
251%%
252
253/* Keywords */
254<YYINITIAL> "break" { addToken(Token.RESERVED_WORD); }
255<YYINITIAL> "do" { addToken(Token.RESERVED_WORD); }
256<YYINITIAL> "else" { addToken(Token.RESERVED_WORD); }
257<YYINITIAL> "elseif" { addToken(Token.RESERVED_WORD); }
258<YYINITIAL> "end" { addToken(Token.RESERVED_WORD); }
259<YYINITIAL> "for" { addToken(Token.RESERVED_WORD); }
260<YYINITIAL> "function" { addToken(Token.RESERVED_WORD); }
261<YYINITIAL> "if" { addToken(Token.RESERVED_WORD); }
262<YYINITIAL> "local" { addToken(Token.RESERVED_WORD); }
263<YYINITIAL> "nil" { addToken(Token.RESERVED_WORD); }
264<YYINITIAL> "repeat" { addToken(Token.RESERVED_WORD); }
265<YYINITIAL> "return" { addToken(Token.RESERVED_WORD); }
266<YYINITIAL> "then" { addToken(Token.RESERVED_WORD); }
267<YYINITIAL> "until" { addToken(Token.RESERVED_WORD); }
268<YYINITIAL> "while" { addToken(Token.RESERVED_WORD); }
269
270/* Data types. */
271<YYINITIAL> "<number>" { addToken(Token.DATA_TYPE); }
272<YYINITIAL> "<name>" { addToken(Token.DATA_TYPE); }
273<YYINITIAL> "<string>" { addToken(Token.DATA_TYPE); }
274<YYINITIAL> "<eof>" { addToken(Token.DATA_TYPE); }
275<YYINITIAL> "NULL" { addToken(Token.DATA_TYPE); }
276
277/* Functions. */
278<YYINITIAL> "_G" { addToken(Token.FUNCTION); }
279<YYINITIAL> "_VERSION" { addToken(Token.FUNCTION); }
280<YYINITIAL> "assert" { addToken(Token.FUNCTION); }
281<YYINITIAL> "collectgarbage" { addToken(Token.FUNCTION); }
282<YYINITIAL> "dofile" { addToken(Token.FUNCTION); }
283<YYINITIAL> "error" { addToken(Token.FUNCTION); }
284<YYINITIAL> "getfenv" { addToken(Token.FUNCTION); }
285<YYINITIAL> "getmetatable" { addToken(Token.FUNCTION); }
286<YYINITIAL> "ipairs" { addToken(Token.FUNCTION); }
287<YYINITIAL> "load" { addToken(Token.FUNCTION); }
288<YYINITIAL> "loadfile" { addToken(Token.FUNCTION); }
289<YYINITIAL> "loadstring" { addToken(Token.FUNCTION); }
290<YYINITIAL> "module" { addToken(Token.FUNCTION); }
291<YYINITIAL> "next" { addToken(Token.FUNCTION); }
292<YYINITIAL> "pairs" { addToken(Token.FUNCTION); }
293<YYINITIAL> "pcall" { addToken(Token.FUNCTION); }
294<YYINITIAL> "print" { addToken(Token.FUNCTION); }
295<YYINITIAL> "rawequal" { addToken(Token.FUNCTION); }
296<YYINITIAL> "rawget" { addToken(Token.FUNCTION); }
297<YYINITIAL> "rawset" { addToken(Token.FUNCTION); }
298<YYINITIAL> "require" { addToken(Token.FUNCTION); }
299<YYINITIAL> "select" { addToken(Token.FUNCTION); }
300<YYINITIAL> "setfenv" { addToken(Token.FUNCTION); }
301<YYINITIAL> "setmetatable" { addToken(Token.FUNCTION); }
302<YYINITIAL> "tonumber" { addToken(Token.FUNCTION); }
303<YYINITIAL> "tostring" { addToken(Token.FUNCTION); }
304<YYINITIAL> "type" { addToken(Token.FUNCTION); }
305<YYINITIAL> "unpack" { addToken(Token.FUNCTION); }
306<YYINITIAL> "xpcall" { addToken(Token.FUNCTION); }
307
308/* Booleans. */
309<YYINITIAL> {BooleanLiteral} { addToken(Token.LITERAL_BOOLEAN); }
310
311
312<YYINITIAL> {
313
314 {LineTerminator} { addNullToken(); return firstToken; }
315
316 {WhiteSpace}+ { addToken(Token.WHITESPACE); }
317
318 /* String/Character literals. */
319 {CharLiteral} { addToken(Token.LITERAL_CHAR); }
320 {UnclosedCharLiteral} { addToken(Token.ERROR_CHAR); addNullToken(); return firstToken; }
321 {StringLiteral} { addToken(Token.LITERAL_STRING_DOUBLE_QUOTE); }
322 {UnclosedStringLiteral} { addToken(Token.ERROR_STRING_DOUBLE); addNullToken(); return firstToken; }
323 {LongStringBegin} { start = zzMarkedPos-2; yybegin(LONGSTRING); }
324
325 /* Comment literals. */
326 {MLCBegin} { start = zzMarkedPos-4; yybegin(MLC); }
327 {LineCommentBegin} { start = zzMarkedPos-2; yybegin(LINECOMMENT); }
328
329 /* Separators. */
330 {Separator} { addToken(Token.SEPARATOR); }
331 {Separator2} { addToken(Token.IDENTIFIER); }
332
333 /* Operators. */
334 {Operator} { addToken(Token.OPERATOR); }
335
336 /* Identifiers - Comes after Operators for "and", "not" and "or". */
337 {Identifier} { addToken(Token.IDENTIFIER); }
338
339 /* Numbers */
340 {Number} { addToken(Token.LITERAL_NUMBER_FLOAT); }
341
342 /* Ended with a line not in a string or comment. */
343 <<EOF>> { addNullToken(); return firstToken; }
344
345 /* Catch any other (unhandled) characters. */
346 . { addToken(Token.IDENTIFIER); }
347
348}
349
350
351<MLC> {
352 [^\n\]]+ {}
353 \n { addToken(start,zzStartRead-1, Token.COMMENT_MULTILINE); return firstToken; }
354 {LongStringEnd} { yybegin(YYINITIAL); addToken(start,zzStartRead+1, Token.COMMENT_MULTILINE); }
355 \] {}
356 <<EOF>> { addToken(start,zzStartRead-1, Token.COMMENT_MULTILINE); return firstToken; }
357}
358
359
360<LONGSTRING> {
361 [^\n\]]+ {}
362 \n { addToken(start,zzStartRead-1, Token.LITERAL_STRING_DOUBLE_QUOTE); return firstToken; }
363 {LongStringEnd} { yybegin(YYINITIAL); addToken(start,zzStartRead+1, Token.LITERAL_STRING_DOUBLE_QUOTE); }
364 \] {}
365 <<EOF>> { addToken(start,zzStartRead-1, Token.LITERAL_STRING_DOUBLE_QUOTE); return firstToken; }
366}
367
368
369<LINECOMMENT> {
370 [^\n]+ {}
371 \n { addToken(start,zzStartRead-1, Token.COMMENT_EOL); return firstToken; }
372 <<EOF>> { addToken(start,zzStartRead-1, Token.COMMENT_EOL); return firstToken; }
373}
Note: See TracBrowser for help on using the repository browser.