source: other-projects/rsyntax-textarea/src/java/org/fife/ui/rsyntaxtextarea/modes/ClojureTokenMaker.flex@ 25584

Last change on this file since 25584 was 25584, checked in by davidb, 12 years ago

Initial cut an a text edit area for GLI that supports color syntax highlighting
File size: 14.1 KB
Line 
1/*
2 * 12/23/2010
3 *
4 * ClojureTokenMaker.java - Scanner for Clojure.
5 *
6 * This library is distributed under a modified BSD license. See the included
7 * RSyntaxTextArea.License.txt file for details.
8 */
9package org.fife.ui.rsyntaxtextarea.modes;
10
11import java.io.*;
12import javax.swing.text.Segment;
13
14import org.fife.ui.rsyntaxtextarea.*;
15
16
17/**
18 * Scanner for the Clojure programming language.<p>
19 *
20 * This was graciously donated by the folks at the
21 * <a href="http://pacific.mpi-cbg.de/wiki/index.php/Fiji">Fiji</a> project.
22 * Its original location was
23 * <a href="http://pacific.mpi-cbg.de/cgi-bin/gitweb.cgi?p=fiji.git;a=tree;f=src-plugins/Script_Editor/fiji/scripting;hb=935d85d9d88dd780c6d5f2765937ddc18b5008ca">here</a>.
24 * <p>
25 *
26 * This implementation was created using
27 * <a href="http://www.jflex.de/">JFlex</a> 1.4.1; however, the generated file
28 * was modified for performance. Memory allocation needs to be almost
29 * completely removed to be competitive with the handwritten lexers (subclasses
30 * of <code>AbstractTokenMaker</code>, so this class has been modified so that
31 * Strings are never allocated (via yytext()), and the scanner never has to
32 * worry about refilling its buffer (needlessly copying chars around).
33 * We can achieve this because RText always scans exactly 1 line of tokens at a
34 * time, and hands the scanner this line as an array of characters (a Segment
35 * really). Since tokens contain pointers to char arrays instead of Strings
36 * holding their contents, there is no need for allocating new memory for
37 * Strings.<p>
38 *
39 * The actual algorithm generated for scanning has, of course, not been
40 * modified.<p>
41 *
42 * If you wish to regenerate this file yourself, keep in mind the following:
43 * <ul>
44 * <li>The generated ClojureTokenMaker.java</code> file will contain two
45 * definitions of both <code>zzRefill</code> and <code>yyreset</code>.
46 * You should hand-delete the second of each definition (the ones
47 * generated by the lexer), as these generated methods modify the input
48 * buffer, which we'll never have to do.</li>
49 * <li>You should also change the declaration/definition of zzBuffer to NOT
50 * be initialized. This is a needless memory allocation for us since we
51 * will be pointing the array somewhere else anyway.</li>
52 * <li>You should NOT call <code>yylex()</code> on the generated scanner
53 * directly; rather, you should use <code>getTokenList</code> as you would
54 * with any other <code>TokenMaker</code> instance.</li>
55 * </ul>
56 *
57 *
58 */
59%%
60
61%public
62%class ClojureTokenMaker
63%extends AbstractJFlexTokenMaker
64%unicode
65%type org.fife.ui.rsyntaxtextarea.Token
66
67
68%{
69
70
71 /**
72 * Constructor. This must be here because JFlex does not generate a
73 * no-parameter constructor.
74 */
75 public ClojureTokenMaker() {
76 }
77
78
79 /**
80 * Adds the token specified to the current linked list of tokens.
81 *
82 * @param tokenType The token's type.
83 * @see #addToken(int, int, int)
84 */
85 private void addHyperlinkToken(int start, int end, int tokenType) {
86 int so = start + offsetShift;
87 addToken(zzBuffer, start,end, tokenType, so, true);
88 }
89
90
91 /**
92 * Adds the token specified to the current linked list of tokens.
93 *
94 * @param tokenType The token's type.
95 */
96 private void addToken(int tokenType) {
97 addToken(zzStartRead, zzMarkedPos-1, tokenType);
98 }
99
100
101 /**
102 * Adds the token specified to the current linked list of tokens.
103 *
104 * @param tokenType The token's type.
105 * @see #addHyperlinkToken(int, int, int)
106 */
107 private void addToken(int start, int end, int tokenType) {
108 int so = start + offsetShift;
109 addToken(zzBuffer, start,end, tokenType, so, false);
110 }
111
112
113 /**
114 * Adds the token specified to the current linked list of tokens.
115 *
116 * @param array The character array.
117 * @param start The starting offset in the array.
118 * @param end The ending offset in the array.
119 * @param tokenType The token's type.
120 * @param startOffset The offset in the document at which this token
121 * occurs.
122 * @param hyperlink Whether this token is a hyperlink.
123 */
124 public void addToken(char[] array, int start, int end, int tokenType,
125 int startOffset, boolean hyperlink) {
126 super.addToken(array, start,end, tokenType, startOffset, hyperlink);
127 zzStartRead = zzMarkedPos;
128 }
129
130
131 /**
132 * Returns the text to place at the beginning and end of a
133 * line to "comment" it in a this programming language.
134 *
135 * @return The start and end strings to add to a line to "comment"
136 * it out.
137 */
138 public String[] getLineCommentStartAndEnd() {
139 return new String[] { ";", null };
140 }
141
142
143 /**
144 * Returns the first token in the linked list of tokens generated
145 * from <code>text</code>. This method must be implemented by
146 * subclasses so they can correctly implement syntax highlighting.
147 *
148 * @param text The text from which to get tokens.
149 * @param initialTokenType The token type we should start with.
150 * @param startOffset The offset into the document at which
151 * <code>text</code> starts.
152 * @return The first <code>Token</code> in a linked list representing
153 * the syntax highlighted text.
154 */
155 public Token getTokenList(Segment text, int initialTokenType, int startOffset) {
156
157 resetTokenList();
158 this.offsetShift = -text.offset + startOffset;
159
160 // Start off in the proper state.
161 int state = Token.NULL;
162 switch (initialTokenType) {
163 /*case Token.COMMENT_MULTILINE:
164 state = MLC;
165 start = text.offset;
166 break;
167 case Token.COMMENT_DOCUMENTATION:
168 state = DOCCOMMENT;
169 start = text.offset;
170 break;*/
171 case Token.LITERAL_STRING_DOUBLE_QUOTE:
172 state = STRING;
173 start = text.offset;
174 break;
175 default:
176 state = Token.NULL;
177 }
178
179 s = text;
180 try {
181 yyreset(zzReader);
182 yybegin(state);
183 return yylex();
184 } catch (IOException ioe) {
185 ioe.printStackTrace();
186 return new DefaultToken();
187 }
188
189 }
190
191
192 /**
193 * Refills the input buffer.
194 *
195 * @return <code>true</code> if EOF was reached, otherwise
196 * <code>false</code>.
197 * @exception IOException if any I/O-Error occurs.
198 */
199 private boolean zzRefill() throws java.io.IOException {
200 return zzCurrentPos>=s.offset+s.count;
201 }
202
203
204 /**
205 * Resets the scanner to read from a new input stream.
206 * Does not close the old reader.
207 *
208 * All internal variables are reset, the old input stream
209 * <b>cannot</b> be reused (internal buffer is discarded and lost).
210 * Lexical state is set to <tt>YY_INITIAL</tt>.
211 *
212 * @param reader the new input stream
213 */
214 public final void yyreset(java.io.Reader reader) throws java.io.IOException {
215 // 's' has been updated.
216 zzBuffer = s.array;
217 /*
218 * We replaced the line below with the two below it because zzRefill
219 * no longer "refills" the buffer (since the way we do it, it's always
220 * "full" the first time through, since it points to the segment's
221 * array). So, we assign zzEndRead here.
222 */
223 //zzStartRead = zzEndRead = s.offset;
224 zzStartRead = s.offset;
225 zzEndRead = zzStartRead + s.count - 1;
226 zzCurrentPos = zzMarkedPos = s.offset;
227 zzLexicalState = YYINITIAL;
228 zzReader = reader;
229 zzAtBOL = true;
230 zzAtEOF = false;
231 }
232
233
234%}
235
236
237LineCommentBegin = (";")
238Keyword = ([:][a-zA-Z?!\-+*/][a-zA-Z0-9?!\-+*/]*)
239NonzeroDigit = [1-9]
240Digit = ("0"|{NonzeroDigit})
241HexDigit = ({Digit}|[A-Fa-f])
242OctalDigit = ([0-7])
243EscapedSourceCharacter = ("u"{HexDigit}{HexDigit}{HexDigit}{HexDigit})
244Escape = ("\\"(([btnfr\"'\\])|([0123]{OctalDigit}?{OctalDigit}?)|({OctalDigit}{OctalDigit}?)|{EscapedSourceCharacter}))
245AnyCharacterButDoubleQuoteOrBackSlash = ([^\\\"\n])
246UnclosedStringLiteral = ([\"]([\\].|[^\\\"])*[^\"]?)
247ErrorStringLiteral = ({UnclosedStringLiteral}[\"])
248StringLiteral = ([\"])
249CharLiteral = ("\\."|"\\space"|"\\tab"|"\\newline")
250AnyCharacter = ([.]*)
251Separator = ([\(\)\{\}\[\]])
252NonSeparator = ([^\t\f\r\n\ \(\)\{\}\[\]\;\,\.\=\>\<\!\~\?\:\+\-\*\/\&\|\^\%\"\'])
253
254BooleanLiteral =("true"|"false")
255
256LineTerminator = (\n)
257WhiteSpace = ([ \t\f])
258
259IntegerHelper1 = (({NonzeroDigit}{Digit}*)|"0")
260IntegerHelper2 = ("0"(([xX]{HexDigit}+)|({OctalDigit}*)))
261IntegerLiteral = ({IntegerHelper1}[lL]?)
262HexLiteral = ({IntegerHelper2}[lL]?)
263FloatHelper1 = ([fFdD]?)
264FloatHelper2 = ([eE][+-]?{Digit}+{FloatHelper1})
265FloatLiteral1 = ({Digit}+"."({FloatHelper1}|{FloatHelper2}|{Digit}+({FloatHelper1}|{FloatHelper2})))
266FloatLiteral2 = ("."{Digit}+({FloatHelper1}|{FloatHelper2}))
267FloatLiteral3 = ({Digit}+{FloatHelper2})
268FloatLiteral = ({FloatLiteral1}|{FloatLiteral2}|{FloatLiteral3}|({Digit}+[fFdD]))
269ErrorNumberFormat = (({IntegerLiteral}|{HexLiteral}|{FloatLiteral}){NonSeparator}+)
270
271
272Nil = ("nil")
273Quote = (\('\|`\))
274Unquote = (\(\~@\|\~\))
275DispatchStart = ("#^"|"#^{")
276Dispatch = ({DispatchStart}[^\s\t\n;\"}]*([ \t\n;\"]|"}"))
277VarQuote = ("#'"[.]*[ \t\n;(\"])
278DefName = (\s*[a-zA-Z0-9?!\-+*\./<>]*)
279
280NonAssignmentOperator = ("+"|"-"|"<="|"^"|"<"|"*"|">="|"%"|">"|"/"|"!="|"?"|">>"|"!"|"&"|"=="|":"|">>"|"~"|">>>")
281AssignmentOperator = ("=")
282Operator = ({NonAssignmentOperator}|{AssignmentOperator})
283
284Letter = [A-Za-z]
285LetterOrUnderscore = ({Letter}|[_])
286Digit = [0-9]
287URLGenDelim = ([:\/\?#\[\]@])
288URLSubDelim = ([\!\$&'\(\)\*\+,;=])
289URLUnreserved = ({LetterOrUnderscore}|{Digit}|[\-\.\~])
290URLCharacter = ({URLGenDelim}|{URLSubDelim}|{URLUnreserved}|[%])
291URLCharacters = ({URLCharacter}*)
292URLEndCharacter = ([\/\$]|{Letter}|{Digit})
293URL = (((https?|f(tp|ile))"://"|"www.")({URLCharacters}{URLEndCharacter})?)
294
295
296%state STRING
297%state EOL_COMMENT
298
299%%
300
301<YYINITIAL> {
302
303"fn" |
304"if" |
305"cond" |
306"def" |
307"defn" |
308"let" |
309"new" |
310"defmacro" |
311"recur" |
312"loop" |
313"thisfn" |
314"do" |
315"quote" |
316"the-var" |
317"class" |
318"instance?" |
319"throw" |
320"try-finally" |
321"set!" |
322"monitor-enter" |
323"monitor-exit" { addToken(Token.RESERVED_WORD); }
324
325"load-file" |
326"eql-ref?" |
327"list" |
328"cons" |
329"conj" |
330"vector" |
331"hash-map" |
332"sorted-map" |
333"sorted-map-by" |
334"meta" |
335"with-meta" |
336"when" |
337"when-not" |
338"nil?" |
339"not" |
340"first" |
341"rest" |
342"second" |
343"ffirst" |
344"frest" |
345"rfirst" |
346"rrest" |
347"eql?" |
348"str" |
349"strcat" |
350"gensym" |
351"apply" |
352"list*" |
353"delay" |
354"lazy-cons" |
355"fnseq" |
356"concat" |
357"inc" |
358"dec" |
359"pos?" |
360"neg?" |
361"zero?" |
362"quot" |
363"rem" |
364"complement" |
365"constantly" |
366"identity" |
367"seq" |
368"count" |
369"assoc" |
370"dissoc" |
371"find" |
372"keys" |
373"vals" |
374"merge" |
375"rseq" |
376"sym" |
377"name" |
378"namespace" |
379"locking" |
380".." |
381"->" |
382"defmulti" |
383"defmethod" |
384"remove-method" |
385"binding" |
386"find-var" |
387"ref" |
388"deref" |
389"commute" |
390"alter" |
391"set" |
392"ensure" |
393"sync" |
394"agent" |
395"agent-of" |
396"agent-errors" |
397"clear-agent-errors" |
398"reduce" |
399"reverse" |
400"comp" |
401"appl" |
402"every" |
403"not-every" |
404"any" |
405"not-any" |
406"map" |
407"pmap" |
408"mapcat" |
409"filter" |
410"take" |
411"take-while" |
412"drop" |
413"drop-while" |
414"zipmap" |
415"cycle" |
416"split-at" |
417"split-with" |
418"repeat" |
419"replicate" |
420"iterate" |
421"range" |
422"doseq" |
423"dotimes" |
424"into" |
425"eval" |
426"import" |
427"unimport" |
428"refer" |
429"unrefer" |
430"in-namespace" |
431"unintern" |
432"into-array" |
433"array" |
434"make-proxy" |
435"implement" |
436"prn" |
437"print" |
438"newline" |
439"*out*" |
440"*current-namespace*" |
441"*print-meta*"
442"doto" |
443"memfn" |
444"read" |
445"*in*" |
446"time" |
447"key" |
448"val" |
449"int" |
450"long" |
451"float" |
452"double" |
453"short" |
454"byte" |
455"boolean" |
456"char" |
457"aget" |
458"aset" |
459"aset-boolean" |
460"aset-int" |
461"aset-long" |
462"aset-float" |
463"aset-double" |
464"aset-short" |
465"aset-byte" |
466"make-array" |
467"peek" |
468"pop" |
469"nth" |
470"get" |
471"and" |
472"or" { addToken(Token.FUNCTION); }
473
474{LineTerminator} { addNullToken(); return firstToken; }
475
476{WhiteSpace}+ { addToken(Token.WHITESPACE); }
477
478{CharLiteral} { addToken(Token.LITERAL_CHAR); }
479{StringLiteral} { start = zzMarkedPos-1; yybegin(STRING); }
480 //{UnclosedStringLiteral} { addToken(Token.ERROR_STRING_DOUBLE); addNullToken(); return firstToken; }
481 //{ErrorStringLiteral} { addToken(Token.ERROR_STRING_DOUBLE); }
482
483{Nil} { addToken(Token.DATA_TYPE); }
484
485{BooleanLiteral} { addToken(Token.LITERAL_BOOLEAN); }
486
487
488{Quote} { addToken(Token.SEPARATOR); }
489{Unquote} { addToken(Token.SEPARATOR); }
490{VarQuote} { addToken(Token.SEPARATOR); }
491{Dispatch} { addToken(Token.DATA_TYPE); }
492
493{LineCommentBegin} { start = zzMarkedPos-1; yybegin(EOL_COMMENT); }
494
495{Separator} { addToken(Token.SEPARATOR); }
496
497{Operator} { addToken(Token.OPERATOR); }
498
499{IntegerLiteral} { addToken(Token.LITERAL_NUMBER_DECIMAL_INT); }
500{HexLiteral} { addToken(Token.LITERAL_NUMBER_HEXADECIMAL); }
501{FloatLiteral} { addToken(Token.LITERAL_NUMBER_FLOAT); }
502{ErrorNumberFormat} { addToken(Token.ERROR_NUMBER_FORMAT); }
503{Keyword} { addToken(Token.PREPROCESSOR); }
504{DefName} { addToken(Token.IDENTIFIER); }
505
506<<EOF>> { addNullToken(); return firstToken; }
507
508. { addToken(Token.ERROR_IDENTIFIER); }
509
510}
511
512<STRING> {
513
514 [^\n\"]+ {}
515 \n { addToken(start,zzStartRead-1, Token.LITERAL_STRING_DOUBLE_QUOTE); return firstToken; }
516 "\"\"" {}
517 "\"" { yybegin(YYINITIAL); addToken(start,zzStartRead, Token.LITERAL_STRING_DOUBLE_QUOTE); }
518 <<EOF>> { addToken(start,zzStartRead-1, Token.LITERAL_STRING_DOUBLE_QUOTE); return firstToken; }
519
520}
521
522<EOL_COMMENT> {
523 [^hwf\n]+ {}
524 {URL} { int temp=zzStartRead; addToken(start,zzStartRead-1, Token.COMMENT_EOL); addHyperlinkToken(temp,zzMarkedPos-1, Token.COMMENT_EOL); start = zzMarkedPos; }
525 [hwf] {}
526 \n { addToken(start,zzStartRead-1, Token.COMMENT_EOL); addNullToken(); return firstToken; }
527 <<EOF>> { addToken(start,zzStartRead-1, Token.COMMENT_EOL); addNullToken(); return firstToken; }
528
529}
Note: See TracBrowser for help on using the repository browser.