source: other-projects/rsyntax-textarea/src/java/org/fife/ui/rsyntaxtextarea/modes/TclTokenMaker.flex@ 25584

Last change on this file since 25584 was 25584, checked in by davidb, 12 years ago

Initial cut an a text edit area for GLI that supports color syntax highlighting
File size: 16.0 KB
Line 
1/*
2 * 10/03/2007
3 *
4 * TclTokenMaker.java - Scanner for the Tcl programming language.
5 *
6 * This library is distributed under a modified BSD license. See the included
7 * RSyntaxTextArea.License.txt file for details.
8 */
9package org.fife.ui.rsyntaxtextarea.modes;
10
11import java.io.*;
12import javax.swing.text.Segment;
13
14import org.fife.ui.rsyntaxtextarea.*;
15
16
17/**
18 * Scanner for the Tcl programming language.<p>
19 *
20 * This implementation was created using
21 * <a href="http://www.jflex.de/">JFlex</a> 1.4.1; however, the generated file
22 * was modified for performance. Memory allocation needs to be almost
23 * completely removed to be competitive with the handwritten lexers (subclasses
24 * of <code>AbstractTokenMaker</code>, so this class has been modified so that
25 * Strings are never allocated (via yytext()), and the scanner never has to
26 * worry about refilling its buffer (needlessly copying chars around).
27 * We can achieve this because RText always scans exactly 1 line of tokens at a
28 * time, and hands the scanner this line as an array of characters (a Segment
29 * really). Since tokens contain pointers to char arrays instead of Strings
30 * holding their contents, there is no need for allocating new memory for
31 * Strings.<p>
32 *
33 * The actual algorithm generated for scanning has, of course, not been
34 * modified.<p>
35 *
36 * If you wish to regenerate this file yourself, keep in mind the following:
37 * <ul>
38 * <li>The generated TclTokenMaker.java</code> file will contain two
39 * definitions of both <code>zzRefill</code> and <code>yyreset</code>.
40 * You should hand-delete the second of each definition (the ones
41 * generated by the lexer), as these generated methods modify the input
42 * buffer, which we'll never have to do.</li>
43 * <li>You should also change the declaration/definition of zzBuffer to NOT
44 * be initialized. This is a needless memory allocation for us since we
45 * will be pointing the array somewhere else anyway.</li>
46 * <li>You should NOT call <code>yylex()</code> on the generated scanner
47 * directly; rather, you should use <code>getTokenList</code> as you would
48 * with any other <code>TokenMaker</code> instance.</li>
49 * </ul>
50 *
51 * @author Robert Futrell
52 * @version 0.5
53 *
54 */
55%%
56
57%public
58%class TclTokenMaker
59%extends AbstractJFlexCTokenMaker
60%unicode
61%type org.fife.ui.rsyntaxtextarea.Token
62
63
64%{
65
66
67 /**
68 * Constructor. This must be here because JFlex does not generate a
69 * no-parameter constructor.
70 */
71 public TclTokenMaker() {
72 }
73
74
75 /**
76 * Adds the token specified to the current linked list of tokens.
77 *
78 * @param tokenType The token's type.
79 */
80 private void addToken(int tokenType) {
81 addToken(zzStartRead, zzMarkedPos-1, tokenType);
82 }
83
84
85 /**
86 * Adds the token specified to the current linked list of tokens.
87 *
88 * @param tokenType The token's type.
89 */
90 private void addToken(int start, int end, int tokenType) {
91 int so = start + offsetShift;
92 addToken(zzBuffer, start,end, tokenType, so);
93 }
94
95
96 /**
97 * Adds the token specified to the current linked list of tokens.
98 *
99 * @param array The character array.
100 * @param start The starting offset in the array.
101 * @param end The ending offset in the array.
102 * @param tokenType The token's type.
103 * @param startOffset The offset in the document at which this token
104 * occurs.
105 */
106 public void addToken(char[] array, int start, int end, int tokenType, int startOffset) {
107 super.addToken(array, start,end, tokenType, startOffset);
108 zzStartRead = zzMarkedPos;
109 }
110
111
112 /**
113 * Returns the text to place at the beginning and end of a
114 * line to "comment" it in a this programming language.
115 *
116 * @return The start and end strings to add to a line to "comment"
117 * it out.
118 */
119 public String[] getLineCommentStartAndEnd() {
120 return new String[] { "//", null };
121 }
122
123
124 /**
125 * Returns the first token in the linked list of tokens generated
126 * from <code>text</code>. This method must be implemented by
127 * subclasses so they can correctly implement syntax highlighting.
128 *
129 * @param text The text from which to get tokens.
130 * @param initialTokenType The token type we should start with.
131 * @param startOffset The offset into the document at which
132 * <code>text</code> starts.
133 * @return The first <code>Token</code> in a linked list representing
134 * the syntax highlighted text.
135 */
136 public Token getTokenList(Segment text, int initialTokenType, int startOffset) {
137
138 resetTokenList();
139 this.offsetShift = -text.offset + startOffset;
140
141 // Start off in the proper state.
142 int state = Token.NULL;
143
144 s = text;
145 try {
146 yyreset(zzReader);
147 yybegin(state);
148 return yylex();
149 } catch (IOException ioe) {
150 ioe.printStackTrace();
151 return new DefaultToken();
152 }
153
154 }
155
156
157 /**
158 * Refills the input buffer.
159 *
160 * @return <code>true</code> if EOF was reached, otherwise
161 * <code>false</code>.
162 * @exception IOException if any I/O-Error occurs.
163 */
164 private boolean zzRefill() throws java.io.IOException {
165 return zzCurrentPos>=s.offset+s.count;
166 }
167
168
169 /**
170 * Resets the scanner to read from a new input stream.
171 * Does not close the old reader.
172 *
173 * All internal variables are reset, the old input stream
174 * <b>cannot</b> be reused (internal buffer is discarded and lost).
175 * Lexical state is set to <tt>YY_INITIAL</tt>.
176 *
177 * @param reader the new input stream
178 */
179 public final void yyreset(java.io.Reader reader) throws java.io.IOException {
180 // 's' has been updated.
181 zzBuffer = s.array;
182 /*
183 * We replaced the line below with the two below it because zzRefill
184 * no longer "refills" the buffer (since the way we do it, it's always
185 * "full" the first time through, since it points to the segment's
186 * array). So, we assign zzEndRead here.
187 */
188 //zzStartRead = zzEndRead = s.offset;
189 zzStartRead = s.offset;
190 zzEndRead = zzStartRead + s.count - 1;
191 zzCurrentPos = zzMarkedPos = zzPushbackPos = s.offset;
192 zzLexicalState = YYINITIAL;
193 zzReader = reader;
194 zzAtBOL = true;
195 zzAtEOF = false;
196 }
197
198
199%}
200
201Letter = [A-Za-z]
202NonzeroDigit = [1-9]
203Digit = ("0"|{NonzeroDigit})
204HexDigit = ({Digit}|[A-Fa-f])
205OctalDigit = ([0-7])
206EscapedSourceCharacter = ("u"{HexDigit}{HexDigit}{HexDigit}{HexDigit})
207NonSeparator = ([^\t\f\r\n\ \(\)\{\}\[\]\;\,\.\=\>\<\!\~\?\:\+\-\*\/\&\|\^\%\"\']|"#"|"\\")
208IdentifierStart = ({Letter}|"_"|"$")
209IdentifierPart = ({IdentifierStart}|{Digit}|("\\"{EscapedSourceCharacter}))
210
211LineTerminator = (\n)
212WhiteSpace = ([ \t\f])
213
214UnclosedStringLiteral = ([\"]([\\].|[^\\\"])*[^\"]?)
215StringLiteral = ({UnclosedStringLiteral}[\"])
216
217LineCommentBegin = "#"
218
219IntegerHelper1 = (({NonzeroDigit}{Digit}*)|"0")
220IntegerHelper2 = ("0"(([xX]{HexDigit}+)|({OctalDigit}*)))
221IntegerLiteral = ({IntegerHelper1}[lL]?)
222HexLiteral = ({IntegerHelper2}[lL]?)
223FloatHelper1 = ([fFdD]?)
224FloatHelper2 = ([eE][+-]?{Digit}+{FloatHelper1})
225FloatLiteral1 = ({Digit}+"."({FloatHelper1}|{FloatHelper2}|{Digit}+({FloatHelper1}|{FloatHelper2})))
226FloatLiteral2 = ("."{Digit}+({FloatHelper1}|{FloatHelper2}))
227FloatLiteral3 = ({Digit}+{FloatHelper2})
228FloatLiteral = ({FloatLiteral1}|{FloatLiteral2}|{FloatLiteral3}|({Digit}+[fFdD]))
229ErrorNumberFormat = (({IntegerLiteral}|{HexLiteral}|{FloatLiteral}){NonSeparator}+)
230
231Separator = ([\(\)\{\}\[\]])
232Separator2 = ([\;,.])
233
234Operator = ("="|"!"|"+"|"-"|"*"|"/"|">"=?|"<"=?|"%"|"&"|"|"|"^"|"~")
235
236Identifier = ({IdentifierStart}{IdentifierPart}*)
237ErrorIdentifier = ({NonSeparator}+)
238
239
240%%
241
242/* Keywords */
243<YYINITIAL> "append" { addToken(Token.RESERVED_WORD); }
244<YYINITIAL> "array" { addToken(Token.RESERVED_WORD); }
245<YYINITIAL> "auto_mkindex" { addToken(Token.RESERVED_WORD); }
246<YYINITIAL> "concat" { addToken(Token.RESERVED_WORD); }
247<YYINITIAL> "console" { addToken(Token.RESERVED_WORD); }
248<YYINITIAL> "eval" { addToken(Token.RESERVED_WORD); }
249<YYINITIAL> "expr" { addToken(Token.RESERVED_WORD); }
250<YYINITIAL> "format" { addToken(Token.RESERVED_WORD); }
251<YYINITIAL> "global" { addToken(Token.RESERVED_WORD); }
252<YYINITIAL> "set" { addToken(Token.RESERVED_WORD); }
253<YYINITIAL> "trace" { addToken(Token.RESERVED_WORD); }
254<YYINITIAL> "unset" { addToken(Token.RESERVED_WORD); }
255<YYINITIAL> "upvar" { addToken(Token.RESERVED_WORD); }
256<YYINITIAL> "join" { addToken(Token.RESERVED_WORD); }
257<YYINITIAL> "lappend" { addToken(Token.RESERVED_WORD); }
258<YYINITIAL> "lindex" { addToken(Token.RESERVED_WORD); }
259<YYINITIAL> "linsert" { addToken(Token.RESERVED_WORD); }
260<YYINITIAL> "list" { addToken(Token.RESERVED_WORD); }
261<YYINITIAL> "llength" { addToken(Token.RESERVED_WORD); }
262<YYINITIAL> "lrange" { addToken(Token.RESERVED_WORD); }
263<YYINITIAL> "lreplace" { addToken(Token.RESERVED_WORD); }
264<YYINITIAL> "lsearch" { addToken(Token.RESERVED_WORD); }
265<YYINITIAL> "lsort" { addToken(Token.RESERVED_WORD); }
266<YYINITIAL> "split" { addToken(Token.RESERVED_WORD); }
267<YYINITIAL> "scan" { addToken(Token.RESERVED_WORD); }
268<YYINITIAL> "string" { addToken(Token.RESERVED_WORD); }
269<YYINITIAL> "regexp" { addToken(Token.RESERVED_WORD); }
270<YYINITIAL> "regsub" { addToken(Token.RESERVED_WORD); }
271<YYINITIAL> "if" { addToken(Token.RESERVED_WORD); }
272<YYINITIAL> "else" { addToken(Token.RESERVED_WORD); }
273<YYINITIAL> "elseif" { addToken(Token.RESERVED_WORD); }
274<YYINITIAL> "switch" { addToken(Token.RESERVED_WORD); }
275<YYINITIAL> "for" { addToken(Token.RESERVED_WORD); }
276<YYINITIAL> "foreach" { addToken(Token.RESERVED_WORD); }
277<YYINITIAL> "while" { addToken(Token.RESERVED_WORD); }
278<YYINITIAL> "break" { addToken(Token.RESERVED_WORD); }
279<YYINITIAL> "continue" { addToken(Token.RESERVED_WORD); }
280<YYINITIAL> "proc" { addToken(Token.RESERVED_WORD); }
281<YYINITIAL> "return" { addToken(Token.RESERVED_WORD); }
282<YYINITIAL> "source" { addToken(Token.RESERVED_WORD); }
283<YYINITIAL> "unkown" { addToken(Token.RESERVED_WORD); }
284<YYINITIAL> "uplevel" { addToken(Token.RESERVED_WORD); }
285<YYINITIAL> "cd" { addToken(Token.RESERVED_WORD); }
286<YYINITIAL> "close" { addToken(Token.RESERVED_WORD); }
287<YYINITIAL> "eof" { addToken(Token.RESERVED_WORD); }
288<YYINITIAL> "file" { addToken(Token.RESERVED_WORD); }
289<YYINITIAL> "flush" { addToken(Token.RESERVED_WORD); }
290<YYINITIAL> "gets" { addToken(Token.RESERVED_WORD); }
291<YYINITIAL> "glob" { addToken(Token.RESERVED_WORD); }
292<YYINITIAL> "open" { addToken(Token.RESERVED_WORD); }
293<YYINITIAL> "read" { addToken(Token.RESERVED_WORD); }
294<YYINITIAL> "puts" { addToken(Token.RESERVED_WORD); }
295<YYINITIAL> "pwd" { addToken(Token.RESERVED_WORD); }
296<YYINITIAL> "seek" { addToken(Token.RESERVED_WORD); }
297<YYINITIAL> "tell" { addToken(Token.RESERVED_WORD); }
298<YYINITIAL> "catch" { addToken(Token.RESERVED_WORD); }
299<YYINITIAL> "error" { addToken(Token.RESERVED_WORD); }
300<YYINITIAL> "exec" { addToken(Token.RESERVED_WORD); }
301<YYINITIAL> "pid" { addToken(Token.RESERVED_WORD); }
302<YYINITIAL> "after" { addToken(Token.RESERVED_WORD); }
303<YYINITIAL> "time" { addToken(Token.RESERVED_WORD); }
304<YYINITIAL> "exit" { addToken(Token.RESERVED_WORD); }
305<YYINITIAL> "history" { addToken(Token.RESERVED_WORD); }
306<YYINITIAL> "rename" { addToken(Token.RESERVED_WORD); }
307<YYINITIAL> "info" { addToken(Token.RESERVED_WORD); }
308<YYINITIAL> "ceil" { addToken(Token.RESERVED_WORD); }
309<YYINITIAL> "floor" { addToken(Token.RESERVED_WORD); }
310<YYINITIAL> "round" { addToken(Token.RESERVED_WORD); }
311<YYINITIAL> "incr" { addToken(Token.RESERVED_WORD); }
312<YYINITIAL> "hypot" { addToken(Token.RESERVED_WORD); }
313<YYINITIAL> "abs" { addToken(Token.RESERVED_WORD); }
314<YYINITIAL> "acos" { addToken(Token.RESERVED_WORD); }
315<YYINITIAL> "cos" { addToken(Token.RESERVED_WORD); }
316<YYINITIAL> "cosh" { addToken(Token.RESERVED_WORD); }
317<YYINITIAL> "asin" { addToken(Token.RESERVED_WORD); }
318<YYINITIAL> "sin" { addToken(Token.RESERVED_WORD); }
319<YYINITIAL> "sinh" { addToken(Token.RESERVED_WORD); }
320<YYINITIAL> "atan" { addToken(Token.RESERVED_WORD); }
321<YYINITIAL> "atan2" { addToken(Token.RESERVED_WORD); }
322<YYINITIAL> "tan" { addToken(Token.RESERVED_WORD); }
323<YYINITIAL> "tanh" { addToken(Token.RESERVED_WORD); }
324<YYINITIAL> "log" { addToken(Token.RESERVED_WORD); }
325<YYINITIAL> "log10" { addToken(Token.RESERVED_WORD); }
326<YYINITIAL> "fmod" { addToken(Token.RESERVED_WORD); }
327<YYINITIAL> "pow" { addToken(Token.RESERVED_WORD); }
328<YYINITIAL> "hypot" { addToken(Token.RESERVED_WORD); }
329<YYINITIAL> "sqrt" { addToken(Token.RESERVED_WORD); }
330<YYINITIAL> "double" { addToken(Token.RESERVED_WORD); }
331<YYINITIAL> "int" { addToken(Token.RESERVED_WORD); }
332
333<YYINITIAL> "bind" { addToken(Token.RESERVED_WORD); }
334<YYINITIAL> "button" { addToken(Token.RESERVED_WORD); }
335<YYINITIAL> "canvas" { addToken(Token.RESERVED_WORD); }
336<YYINITIAL> "checkbutton" { addToken(Token.RESERVED_WORD); }
337<YYINITIAL> "destroy" { addToken(Token.RESERVED_WORD); }
338<YYINITIAL> "entry" { addToken(Token.RESERVED_WORD); }
339<YYINITIAL> "focus" { addToken(Token.RESERVED_WORD); }
340<YYINITIAL> "frame" { addToken(Token.RESERVED_WORD); }
341<YYINITIAL> "grab" { addToken(Token.RESERVED_WORD); }
342<YYINITIAL> "image" { addToken(Token.RESERVED_WORD); }
343<YYINITIAL> "label" { addToken(Token.RESERVED_WORD); }
344<YYINITIAL> "listbox" { addToken(Token.RESERVED_WORD); }
345<YYINITIAL> "lower" { addToken(Token.RESERVED_WORD); }
346<YYINITIAL> "menu" { addToken(Token.RESERVED_WORD); }
347<YYINITIAL> "menubutton" { addToken(Token.RESERVED_WORD); }
348<YYINITIAL> "message" { addToken(Token.RESERVED_WORD); }
349<YYINITIAL> "option" { addToken(Token.RESERVED_WORD); }
350<YYINITIAL> "pack" { addToken(Token.RESERVED_WORD); }
351<YYINITIAL> "placer" { addToken(Token.RESERVED_WORD); }
352<YYINITIAL> "radiobutton" { addToken(Token.RESERVED_WORD); }
353<YYINITIAL> "raise" { addToken(Token.RESERVED_WORD); }
354<YYINITIAL> "scale" { addToken(Token.RESERVED_WORD); }
355<YYINITIAL> "scrollbar" { addToken(Token.RESERVED_WORD); }
356<YYINITIAL> "selection" { addToken(Token.RESERVED_WORD); }
357<YYINITIAL> "send" { addToken(Token.RESERVED_WORD); }
358<YYINITIAL> "text" { addToken(Token.RESERVED_WORD); }
359<YYINITIAL> "tk" { addToken(Token.RESERVED_WORD); }
360<YYINITIAL> "tkerror" { addToken(Token.RESERVED_WORD); }
361<YYINITIAL> "tkwait" { addToken(Token.RESERVED_WORD); }
362<YYINITIAL> "toplevel" { addToken(Token.RESERVED_WORD); }
363<YYINITIAL> "update" { addToken(Token.RESERVED_WORD); }
364<YYINITIAL> "winfo" { addToken(Token.RESERVED_WORD); }
365<YYINITIAL> "wm" { addToken(Token.RESERVED_WORD); }
366
367
368<YYINITIAL> {
369
370 {LineTerminator} { addNullToken(); return firstToken; }
371
372 {Identifier} { addToken(Token.IDENTIFIER); }
373
374 {WhiteSpace}+ { addToken(Token.WHITESPACE); }
375
376 /* String/Character literals. */
377 {StringLiteral} { addToken(Token.LITERAL_STRING_DOUBLE_QUOTE); }
378 {UnclosedStringLiteral} { addToken(Token.ERROR_STRING_DOUBLE); addNullToken(); return firstToken; }
379
380 /* Comment literals. */
381 {LineCommentBegin}.* { addToken(Token.COMMENT_EOL); addNullToken(); return firstToken; }
382
383 /* Separators. */
384 {Separator} { addToken(Token.SEPARATOR); }
385 {Separator2} { addToken(Token.IDENTIFIER); }
386
387 /* Operators. */
388 {Operator} { addToken(Token.OPERATOR); }
389
390 /* Numbers */
391 {IntegerLiteral} { addToken(Token.LITERAL_NUMBER_DECIMAL_INT); }
392 {HexLiteral} { addToken(Token.LITERAL_NUMBER_HEXADECIMAL); }
393 {FloatLiteral} { addToken(Token.LITERAL_NUMBER_FLOAT); }
394 {ErrorNumberFormat} { addToken(Token.ERROR_NUMBER_FORMAT); }
395
396 {ErrorIdentifier} { addToken(Token.ERROR_IDENTIFIER); }
397
398 /* Ended with a line not in a string or comment. */
399 <<EOF>> { addNullToken(); return firstToken; }
400
401 /* Catch any other (unhandled) characters and flag them as bad. */
402 . { addToken(Token.ERROR_IDENTIFIER); }
403
404}
Note: See TracBrowser for help on using the repository browser.