source: other-projects/rsyntax-textarea/src/java/org/fife/ui/rsyntaxtextarea/modes/ScalaTokenMaker.flex@ 25584

Last change on this file since 25584 was 25584, checked in by davidb, 12 years ago

Initial cut an a text edit area for GLI that supports color syntax highlighting
File size: 11.3 KB
Line 
1/*
2 * 8/19/2009
3 *
4 * ScalaTokenMaker.java - Scanner for the Scala programming language.
5 *
6 * This library is distributed under a modified BSD license. See the included
7 * RSyntaxTextArea.License.txt file for details.
8 */
9package org.fife.ui.rsyntaxtextarea.modes;
10
11import java.io.*;
12import javax.swing.text.Segment;
13
14import org.fife.ui.rsyntaxtextarea.*;
15
16
17/**
18 * Scanner for the Scala programming language.<p>
19 *
20 * This implementation was created using
21 * <a href="http://www.jflex.de/">JFlex</a> 1.4.1; however, the generated file
22 * was modified for performance. Memory allocation needs to be almost
23 * completely removed to be competitive with the handwritten lexers (subclasses
24 * of <code>AbstractTokenMaker</code>, so this class has been modified so that
25 * Strings are never allocated (via yytext()), and the scanner never has to
26 * worry about refilling its buffer (needlessly copying chars around).
27 * We can achieve this because RText always scans exactly 1 line of tokens at a
28 * time, and hands the scanner this line as an array of characters (a Segment
29 * really). Since tokens contain pointers to char arrays instead of Strings
30 * holding their contents, there is no need for allocating new memory for
31 * Strings.<p>
32 *
33 * The actual algorithm generated for scanning has, of course, not been
34 * modified.<p>
35 *
36 * If you wish to regenerate this file yourself, keep in mind the following:
37 * <ul>
38 * <li>The generated ScalaTokenMaker.java</code> file will contain two
39 * definitions of both <code>zzRefill</code> and <code>yyreset</code>.
40 * You should hand-delete the second of each definition (the ones
41 * generated by the lexer), as these generated methods modify the input
42 * buffer, which we'll never have to do.</li>
43 * <li>You should also change the declaration/definition of zzBuffer to NOT
44 * be initialized. This is a needless memory allocation for us since we
45 * will be pointing the array somewhere else anyway.</li>
46 * <li>You should NOT call <code>yylex()</code> on the generated scanner
47 * directly; rather, you should use <code>getTokenList</code> as you would
48 * with any other <code>TokenMaker</code> instance.</li>
49 * </ul>
50 *
51 * @author Robert Futrell
52 * @version 0.5
53 *
54 */
55%%
56
57%public
58%class ScalaTokenMaker
59%extends AbstractJFlexCTokenMaker
60%unicode
61%type org.fife.ui.rsyntaxtextarea.Token
62
63
64%{
65
66
67 /**
68 * Constructor. This must be here because JFlex does not generate a
69 * no-parameter constructor.
70 */
71 public ScalaTokenMaker() {
72 }
73
74
75 /**
76 * Adds the token specified to the current linked list of tokens.
77 *
78 * @param tokenType The token's type.
79 * @see #addToken(int, int, int)
80 */
81 private void addHyperlinkToken(int start, int end, int tokenType) {
82 int so = start + offsetShift;
83 addToken(zzBuffer, start,end, tokenType, so, true);
84 }
85
86
87 /**
88 * Adds the token specified to the current linked list of tokens.
89 *
90 * @param tokenType The token's type.
91 */
92 private void addToken(int tokenType) {
93 addToken(zzStartRead, zzMarkedPos-1, tokenType);
94 }
95
96
97 /**
98 * Adds the token specified to the current linked list of tokens.
99 *
100 * @param tokenType The token's type.
101 * @see #addHyperlinkToken(int, int, int)
102 */
103 private void addToken(int start, int end, int tokenType) {
104 int so = start + offsetShift;
105 addToken(zzBuffer, start,end, tokenType, so, false);
106 }
107
108
109 /**
110 * Adds the token specified to the current linked list of tokens.
111 *
112 * @param array The character array.
113 * @param start The starting offset in the array.
114 * @param end The ending offset in the array.
115 * @param tokenType The token's type.
116 * @param startOffset The offset in the document at which this token
117 * occurs.
118 * @param hyperlink Whether this token is a hyperlink.
119 */
120 public void addToken(char[] array, int start, int end, int tokenType,
121 int startOffset, boolean hyperlink) {
122 super.addToken(array, start,end, tokenType, startOffset, hyperlink);
123 zzStartRead = zzMarkedPos;
124 }
125
126
127 /**
128 * Returns the text to place at the beginning and end of a
129 * line to "comment" it in a this programming language.
130 *
131 * @return The start and end strings to add to a line to "comment"
132 * it out.
133 */
134 public String[] getLineCommentStartAndEnd() {
135 return new String[] { "//", null };
136 }
137
138
139 /**
140 * Returns the first token in the linked list of tokens generated
141 * from <code>text</code>. This method must be implemented by
142 * subclasses so they can correctly implement syntax highlighting.
143 *
144 * @param text The text from which to get tokens.
145 * @param initialTokenType The token type we should start with.
146 * @param startOffset The offset into the document at which
147 * <code>text</code> starts.
148 * @return The first <code>Token</code> in a linked list representing
149 * the syntax highlighted text.
150 */
151 public Token getTokenList(Segment text, int initialTokenType, int startOffset) {
152
153 resetTokenList();
154 this.offsetShift = -text.offset + startOffset;
155
156 // Start off in the proper state.
157 int state = Token.NULL;
158 switch (initialTokenType) {
159 case Token.COMMENT_MULTILINE:
160 state = MLC;
161 start = text.offset;
162 break;
163 default:
164 state = Token.NULL;
165 }
166
167 s = text;
168 try {
169 yyreset(zzReader);
170 yybegin(state);
171 return yylex();
172 } catch (IOException ioe) {
173 ioe.printStackTrace();
174 return new DefaultToken();
175 }
176
177 }
178
179
180 /**
181 * Refills the input buffer.
182 *
183 * @return <code>true</code> if EOF was reached, otherwise
184 * <code>false</code>.
185 * @exception IOException if any I/O-Error occurs.
186 */
187 private boolean zzRefill() throws java.io.IOException {
188 return zzCurrentPos>=s.offset+s.count;
189 }
190
191
192 /**
193 * Resets the scanner to read from a new input stream.
194 * Does not close the old reader.
195 *
196 * All internal variables are reset, the old input stream
197 * <b>cannot</b> be reused (internal buffer is discarded and lost).
198 * Lexical state is set to <tt>YY_INITIAL</tt>.
199 *
200 * @param reader the new input stream
201 */
202 public final void yyreset(java.io.Reader reader) throws IOException {
203 // 's' has been updated.
204 zzBuffer = s.array;
205 /*
206 * We replaced the line below with the two below it because zzRefill
207 * no longer "refills" the buffer (since the way we do it, it's always
208 * "full" the first time through, since it points to the segment's
209 * array). So, we assign zzEndRead here.
210 */
211 //zzStartRead = zzEndRead = s.offset;
212 zzStartRead = s.offset;
213 zzEndRead = zzStartRead + s.count - 1;
214 zzCurrentPos = zzMarkedPos = zzPushbackPos = s.offset;
215 zzLexicalState = YYINITIAL;
216 zzReader = reader;
217 zzAtBOL = true;
218 zzAtEOF = false;
219 }
220
221
222%}
223
224
225/***** BEGIN SCALA-SPECIFIC CHANGES *********/
226Upper = ([A-Z\$\_]) /* Plus Unicode category Lu */
227Lower = ([a-z]) /* Plus Unicode category Ll */
228Letter = ({Upper}|{Lower}) /*Plus Unicode categories Lo, Lt, Nl */
229Digit = ([0-9])
230OpChar = ([^A-Z\$\_a-z0-9\(\[\]\)\. \t\f])
231Op = ({OpChar}+)
232IdRest = (({Letter}|{Digit})*([\_]{Op})?)
233VarId = ({Lower}{IdRest})
234PlainId = ({Upper}{IdRest}|{VarId}) /*|{Op})*/
235Id = ({PlainId}) /*({PlainId}|[\']{StringLit}[\'])*/
236
237IntegerLiteral = ({Digit}+[Ll]?)
238HexDigit = ({Digit}|[A-Fa-f])
239HexLiteral = ("0x"{HexDigit}+)
240
241ExponentPart = ([Ee][+\-]?{Digit}+)
242FloatType = ([FfDd])
243FloatingPointLiteral = ({Digit}+[\.]{Digit}*{ExponentPart}?{FloatType}? |
244 [\.]{Digit}+{ExponentPart}?{FloatType}? |
245 {Digit}+{ExponentPart}{FloatType}? |
246 {Digit}+{ExponentPart}?{FloatType})
247
248UnclosedCharLiteral = ([\']([\\].|[^\\\'])*[^\']?)
249CharLiteral = ({UnclosedCharLiteral}[\'])
250UnclosedStringLiteral = ([\"]([\\].|[^\\\"])*[^\"]?)
251StringLiteral = ({UnclosedStringLiteral}[\"])
252UnclosedBacktickLiteral = ([\`][^\`]+)
253BacktickLiteral = ({UnclosedBacktickLiteral}[\`])
254/* TODO: Multiline strings */
255
256MLCBegin = ("/*")
257MLCEnd = ("*/")
258LineCommentBegin = ("//")
259
260/***** END SCALA-SPECIFIC CHANGES *********/
261
262Whitespace = ([ \t\f]+)
263LineTerminator = ([\n])
264Separator = ([\(\)\{\}\[\]])
265
266URLGenDelim = ([:\/\?#\[\]@])
267URLSubDelim = ([\!\$&'\(\)\*\+,;=])
268URLUnreserved = ({Letter}|[\_]|{Digit}|[\-\.\~])
269URLCharacter = ({URLGenDelim}|{URLSubDelim}|{URLUnreserved}|[%])
270URLCharacters = ({URLCharacter}*)
271URLEndCharacter = ([\/\$]|{Letter}|{Digit})
272URL = (((https?|f(tp|ile))"://"|"www.")({URLCharacters}{URLEndCharacter})?)
273
274
275%state MLC
276%state EOL_COMMENT
277
278%%
279
280<YYINITIAL> {
281
282 /* Keywords */
283 "abstract" |
284 "case" |
285 "catch" |
286 "class" |
287 "def" |
288 "do" |
289 "else" |
290 "extends" |
291 "false" |
292 "final" |
293 "finally" |
294 "for" |
295 "forSome" |
296 "if" |
297 "implicit" |
298 "import" |
299 "lazy" |
300 "match" |
301 "new" |
302 "null" |
303 "object" |
304 "override" |
305 "package" |
306 "private" |
307 "protected" |
308 "requires" |
309 "return" |
310 "sealed" |
311 "super" |
312 "this" |
313 "throw" |
314 "trait" |
315 "try" |
316 "true" |
317 "type" |
318 "val" |
319 "var" |
320 "while" |
321 "with" |
322 "yield" { addToken(Token.RESERVED_WORD); }
323
324 {LineTerminator} { addNullToken(); return firstToken; }
325
326 {Id} { addToken(Token.IDENTIFIER); }
327
328 {Whitespace} { addToken(Token.WHITESPACE); }
329
330 /* String/Character literals. */
331 {UnclosedCharLiteral} { addToken(Token.ERROR_CHAR); addNullToken(); return firstToken; }
332 {CharLiteral} { addToken(Token.LITERAL_CHAR); }
333 {UnclosedStringLiteral} { addToken(Token.ERROR_STRING_DOUBLE); addNullToken(); return firstToken; }
334 {StringLiteral} { addToken(Token.LITERAL_STRING_DOUBLE_QUOTE); }
335 {UnclosedBacktickLiteral} { addToken(Token.ERROR_STRING_DOUBLE); addNullToken(); return firstToken; }
336 {BacktickLiteral} { addToken(Token.LITERAL_BACKQUOTE); }
337
338 /* Comment literals. */
339 {MLCBegin} { start = zzMarkedPos-2; yybegin(MLC); }
340 {LineCommentBegin} { start = zzMarkedPos-2; yybegin(EOL_COMMENT); }
341
342 {Separator} { addToken(Token.SEPARATOR); }
343
344 {IntegerLiteral} { addToken(Token.LITERAL_NUMBER_DECIMAL_INT); }
345 {HexLiteral} { addToken(Token.LITERAL_NUMBER_HEXADECIMAL); }
346 {FloatingPointLiteral} { addToken(Token.LITERAL_NUMBER_FLOAT); }
347
348 /* Ended with a line not in a string or comment. */
349 <<EOF>> { addNullToken(); return firstToken; }
350
351 /* Catch any other (unhandled) characters. */
352 . { addToken(Token.IDENTIFIER); }
353
354}
355
356
357<MLC> {
358
359 [^hwf\n\*]+ {}
360 {URL} { int temp=zzStartRead; addToken(start,zzStartRead-1, Token.COMMENT_MULTILINE); addHyperlinkToken(temp,zzMarkedPos-1, Token.COMMENT_MULTILINE); start = zzMarkedPos; }
361 [hwf] {}
362
363 \n { addToken(start,zzStartRead-1, Token.COMMENT_MULTILINE); return firstToken; }
364 {MLCEnd} { yybegin(YYINITIAL); addToken(start,zzStartRead+1, Token.COMMENT_MULTILINE); }
365 \* {}
366 <<EOF>> { addToken(start,zzStartRead-1, Token.COMMENT_MULTILINE); return firstToken; }
367
368}
369
370
371<EOL_COMMENT> {
372 [^hwf\n]+ {}
373 {URL} { int temp=zzStartRead; addToken(start,zzStartRead-1, Token.COMMENT_EOL); addHyperlinkToken(temp,zzMarkedPos-1, Token.COMMENT_EOL); start = zzMarkedPos; }
374 [hwf] {}
375 \n { addToken(start,zzStartRead-1, Token.COMMENT_EOL); addNullToken(); return firstToken; }
376 <<EOF>> { addToken(start,zzStartRead-1, Token.COMMENT_EOL); addNullToken(); return firstToken; }
377}
Note: See TracBrowser for help on using the repository browser.