1 | #
|
---|
2 | # irb/ruby-token.rb - ruby tokens
|
---|
3 | # $Release Version: 0.9.5$
|
---|
4 | # $Revision: 11708 $
|
---|
5 | # $Date: 2007-02-13 08:01:19 +0900 (Tue, 13 Feb 2007) $
|
---|
6 | # by Keiju ISHITSUKA([email protected])
|
---|
7 | #
|
---|
8 | # --
|
---|
9 | #
|
---|
10 | #
|
---|
11 | #
|
---|
12 | module RubyToken
|
---|
13 | EXPR_BEG = :EXPR_BEG
|
---|
14 | EXPR_MID = :EXPR_MID
|
---|
15 | EXPR_END = :EXPR_END
|
---|
16 | EXPR_ARG = :EXPR_ARG
|
---|
17 | EXPR_FNAME = :EXPR_FNAME
|
---|
18 | EXPR_DOT = :EXPR_DOT
|
---|
19 | EXPR_CLASS = :EXPR_CLASS
|
---|
20 |
|
---|
21 | # for ruby 1.4X
|
---|
22 | if !defined?(Symbol)
|
---|
23 | Symbol = Integer
|
---|
24 | end
|
---|
25 |
|
---|
26 | class Token
|
---|
27 | def initialize(seek, line_no, char_no)
|
---|
28 | @seek = seek
|
---|
29 | @line_no = line_no
|
---|
30 | @char_no = char_no
|
---|
31 | end
|
---|
32 | attr :seek
|
---|
33 | attr :line_no
|
---|
34 | attr :char_no
|
---|
35 | end
|
---|
36 |
|
---|
37 | class TkNode < Token
|
---|
38 | def initialize(seek, line_no, char_no)
|
---|
39 | super
|
---|
40 | end
|
---|
41 | attr :node
|
---|
42 | end
|
---|
43 |
|
---|
44 | class TkId < Token
|
---|
45 | def initialize(seek, line_no, char_no, name)
|
---|
46 | super(seek, line_no, char_no)
|
---|
47 | @name = name
|
---|
48 | end
|
---|
49 | attr :name
|
---|
50 | end
|
---|
51 |
|
---|
52 | class TkVal < Token
|
---|
53 | def initialize(seek, line_no, char_no, value = nil)
|
---|
54 | super(seek, line_no, char_no)
|
---|
55 | @value = value
|
---|
56 | end
|
---|
57 | attr :value
|
---|
58 | end
|
---|
59 |
|
---|
60 | class TkOp < Token
|
---|
61 | attr :name, true
|
---|
62 | end
|
---|
63 |
|
---|
64 | class TkOPASGN < TkOp
|
---|
65 | def initialize(seek, line_no, char_no, op)
|
---|
66 | super(seek, line_no, char_no)
|
---|
67 | op = TkReading2Token[op][0] unless op.kind_of?(Symbol)
|
---|
68 | @op = op
|
---|
69 | end
|
---|
70 | attr :op
|
---|
71 | end
|
---|
72 |
|
---|
73 | class TkUnknownChar < Token
|
---|
74 | def initialize(seek, line_no, char_no, id)
|
---|
75 | super(seek, line_no, char_no)
|
---|
76 | @name = name
|
---|
77 | end
|
---|
78 | attr :name
|
---|
79 | end
|
---|
80 |
|
---|
81 | class TkError < Token
|
---|
82 | end
|
---|
83 |
|
---|
84 | def Token(token, value = nil)
|
---|
85 | case token
|
---|
86 | when String
|
---|
87 | if (tk = TkReading2Token[token]).nil?
|
---|
88 | IRB.fail TkReading2TokenNoKey, token
|
---|
89 | end
|
---|
90 | tk = Token(tk[0], value)
|
---|
91 | if tk.kind_of?(TkOp)
|
---|
92 | tk.name = token
|
---|
93 | end
|
---|
94 | return tk
|
---|
95 | when Symbol
|
---|
96 | if (tk = TkSymbol2Token[token]).nil?
|
---|
97 | IRB.fail TkSymbol2TokenNoKey, token
|
---|
98 | end
|
---|
99 | return Token(tk[0], value)
|
---|
100 | else
|
---|
101 | if (token.ancestors & [TkId, TkVal, TkOPASGN, TkUnknownChar]).empty?
|
---|
102 | token.new(@prev_seek, @prev_line_no, @prev_char_no)
|
---|
103 | else
|
---|
104 | token.new(@prev_seek, @prev_line_no, @prev_char_no, value)
|
---|
105 | end
|
---|
106 | end
|
---|
107 | end
|
---|
108 |
|
---|
109 | TokenDefinitions = [
|
---|
110 | [:TkCLASS, TkId, "class", EXPR_CLASS],
|
---|
111 | [:TkMODULE, TkId, "module", EXPR_BEG],
|
---|
112 | [:TkDEF, TkId, "def", EXPR_FNAME],
|
---|
113 | [:TkUNDEF, TkId, "undef", EXPR_FNAME],
|
---|
114 | [:TkBEGIN, TkId, "begin", EXPR_BEG],
|
---|
115 | [:TkRESCUE, TkId, "rescue", EXPR_MID],
|
---|
116 | [:TkENSURE, TkId, "ensure", EXPR_BEG],
|
---|
117 | [:TkEND, TkId, "end", EXPR_END],
|
---|
118 | [:TkIF, TkId, "if", EXPR_BEG, :TkIF_MOD],
|
---|
119 | [:TkUNLESS, TkId, "unless", EXPR_BEG, :TkUNLESS_MOD],
|
---|
120 | [:TkTHEN, TkId, "then", EXPR_BEG],
|
---|
121 | [:TkELSIF, TkId, "elsif", EXPR_BEG],
|
---|
122 | [:TkELSE, TkId, "else", EXPR_BEG],
|
---|
123 | [:TkCASE, TkId, "case", EXPR_BEG],
|
---|
124 | [:TkWHEN, TkId, "when", EXPR_BEG],
|
---|
125 | [:TkWHILE, TkId, "while", EXPR_BEG, :TkWHILE_MOD],
|
---|
126 | [:TkUNTIL, TkId, "until", EXPR_BEG, :TkUNTIL_MOD],
|
---|
127 | [:TkFOR, TkId, "for", EXPR_BEG],
|
---|
128 | [:TkBREAK, TkId, "break", EXPR_END],
|
---|
129 | [:TkNEXT, TkId, "next", EXPR_END],
|
---|
130 | [:TkREDO, TkId, "redo", EXPR_END],
|
---|
131 | [:TkRETRY, TkId, "retry", EXPR_END],
|
---|
132 | [:TkIN, TkId, "in", EXPR_BEG],
|
---|
133 | [:TkDO, TkId, "do", EXPR_BEG],
|
---|
134 | [:TkRETURN, TkId, "return", EXPR_MID],
|
---|
135 | [:TkYIELD, TkId, "yield", EXPR_END],
|
---|
136 | [:TkSUPER, TkId, "super", EXPR_END],
|
---|
137 | [:TkSELF, TkId, "self", EXPR_END],
|
---|
138 | [:TkNIL, TkId, "nil", EXPR_END],
|
---|
139 | [:TkTRUE, TkId, "true", EXPR_END],
|
---|
140 | [:TkFALSE, TkId, "false", EXPR_END],
|
---|
141 | [:TkAND, TkId, "and", EXPR_BEG],
|
---|
142 | [:TkOR, TkId, "or", EXPR_BEG],
|
---|
143 | [:TkNOT, TkId, "not", EXPR_BEG],
|
---|
144 | [:TkIF_MOD, TkId],
|
---|
145 | [:TkUNLESS_MOD, TkId],
|
---|
146 | [:TkWHILE_MOD, TkId],
|
---|
147 | [:TkUNTIL_MOD, TkId],
|
---|
148 | [:TkALIAS, TkId, "alias", EXPR_FNAME],
|
---|
149 | [:TkDEFINED, TkId, "defined?", EXPR_END],
|
---|
150 | [:TklBEGIN, TkId, "BEGIN", EXPR_END],
|
---|
151 | [:TklEND, TkId, "END", EXPR_END],
|
---|
152 | [:Tk__LINE__, TkId, "__LINE__", EXPR_END],
|
---|
153 | [:Tk__FILE__, TkId, "__FILE__", EXPR_END],
|
---|
154 |
|
---|
155 | [:TkIDENTIFIER, TkId],
|
---|
156 | [:TkFID, TkId],
|
---|
157 | [:TkGVAR, TkId],
|
---|
158 | [:TkCVAR, TkId],
|
---|
159 | [:TkIVAR, TkId],
|
---|
160 | [:TkCONSTANT, TkId],
|
---|
161 |
|
---|
162 | [:TkINTEGER, TkVal],
|
---|
163 | [:TkFLOAT, TkVal],
|
---|
164 | [:TkSTRING, TkVal],
|
---|
165 | [:TkXSTRING, TkVal],
|
---|
166 | [:TkREGEXP, TkVal],
|
---|
167 | [:TkSYMBOL, TkVal],
|
---|
168 |
|
---|
169 | [:TkDSTRING, TkNode],
|
---|
170 | [:TkDXSTRING, TkNode],
|
---|
171 | [:TkDREGEXP, TkNode],
|
---|
172 | [:TkNTH_REF, TkNode],
|
---|
173 | [:TkBACK_REF, TkNode],
|
---|
174 |
|
---|
175 | [:TkUPLUS, TkOp, "+@"],
|
---|
176 | [:TkUMINUS, TkOp, "-@"],
|
---|
177 | [:TkPOW, TkOp, "**"],
|
---|
178 | [:TkCMP, TkOp, "<=>"],
|
---|
179 | [:TkEQ, TkOp, "=="],
|
---|
180 | [:TkEQQ, TkOp, "==="],
|
---|
181 | [:TkNEQ, TkOp, "!="],
|
---|
182 | [:TkGEQ, TkOp, ">="],
|
---|
183 | [:TkLEQ, TkOp, "<="],
|
---|
184 | [:TkANDOP, TkOp, "&&"],
|
---|
185 | [:TkOROP, TkOp, "||"],
|
---|
186 | [:TkMATCH, TkOp, "=~"],
|
---|
187 | [:TkNMATCH, TkOp, "!~"],
|
---|
188 | [:TkDOT2, TkOp, ".."],
|
---|
189 | [:TkDOT3, TkOp, "..."],
|
---|
190 | [:TkAREF, TkOp, "[]"],
|
---|
191 | [:TkASET, TkOp, "[]="],
|
---|
192 | [:TkLSHFT, TkOp, "<<"],
|
---|
193 | [:TkRSHFT, TkOp, ">>"],
|
---|
194 | [:TkCOLON2, TkOp],
|
---|
195 | [:TkCOLON3, TkOp],
|
---|
196 | # [:OPASGN, TkOp], # +=, -= etc. #
|
---|
197 | [:TkASSOC, TkOp, "=>"],
|
---|
198 | [:TkQUESTION, TkOp, "?"], #?
|
---|
199 | [:TkCOLON, TkOp, ":"], #:
|
---|
200 |
|
---|
201 | [:TkfLPAREN], # func( #
|
---|
202 | [:TkfLBRACK], # func[ #
|
---|
203 | [:TkfLBRACE], # func{ #
|
---|
204 | [:TkSTAR], # *arg
|
---|
205 | [:TkAMPER], # &arg #
|
---|
206 | [:TkSYMBEG], # :SYMBOL
|
---|
207 |
|
---|
208 | [:TkGT, TkOp, ">"],
|
---|
209 | [:TkLT, TkOp, "<"],
|
---|
210 | [:TkPLUS, TkOp, "+"],
|
---|
211 | [:TkMINUS, TkOp, "-"],
|
---|
212 | [:TkMULT, TkOp, "*"],
|
---|
213 | [:TkDIV, TkOp, "/"],
|
---|
214 | [:TkMOD, TkOp, "%"],
|
---|
215 | [:TkBITOR, TkOp, "|"],
|
---|
216 | [:TkBITXOR, TkOp, "^"],
|
---|
217 | [:TkBITAND, TkOp, "&"],
|
---|
218 | [:TkBITNOT, TkOp, "~"],
|
---|
219 | [:TkNOTOP, TkOp, "!"],
|
---|
220 |
|
---|
221 | [:TkBACKQUOTE, TkOp, "`"],
|
---|
222 |
|
---|
223 | [:TkASSIGN, Token, "="],
|
---|
224 | [:TkDOT, Token, "."],
|
---|
225 | [:TkLPAREN, Token, "("], #(exp)
|
---|
226 | [:TkLBRACK, Token, "["], #[arry]
|
---|
227 | [:TkLBRACE, Token, "{"], #{hash}
|
---|
228 | [:TkRPAREN, Token, ")"],
|
---|
229 | [:TkRBRACK, Token, "]"],
|
---|
230 | [:TkRBRACE, Token, "}"],
|
---|
231 | [:TkCOMMA, Token, ","],
|
---|
232 | [:TkSEMICOLON, Token, ";"],
|
---|
233 |
|
---|
234 | [:TkCOMMENT],
|
---|
235 | [:TkRD_COMMENT],
|
---|
236 | [:TkSPACE],
|
---|
237 | [:TkNL],
|
---|
238 | [:TkEND_OF_SCRIPT],
|
---|
239 |
|
---|
240 | [:TkBACKSLASH, TkUnknownChar, "\\"],
|
---|
241 | [:TkAT, TkUnknownChar, "@"],
|
---|
242 | [:TkDOLLAR, TkUnknownChar, "$"],
|
---|
243 | ]
|
---|
244 |
|
---|
245 | # {reading => token_class}
|
---|
246 | # {reading => [token_class, *opt]}
|
---|
247 | TkReading2Token = {}
|
---|
248 | TkSymbol2Token = {}
|
---|
249 |
|
---|
250 | def RubyToken.def_token(token_n, super_token = Token, reading = nil, *opts)
|
---|
251 | token_n = token_n.id2name if token_n.kind_of?(Symbol)
|
---|
252 | if RubyToken.const_defined?(token_n)
|
---|
253 | IRB.fail AlreadyDefinedToken, token_n
|
---|
254 | end
|
---|
255 | token_c = eval("class #{token_n} < #{super_token}; end; #{token_n}")
|
---|
256 |
|
---|
257 | if reading
|
---|
258 | if TkReading2Token[reading]
|
---|
259 | IRB.fail TkReading2TokenDuplicateError, token_n, reading
|
---|
260 | end
|
---|
261 | if opts.empty?
|
---|
262 | TkReading2Token[reading] = [token_c]
|
---|
263 | else
|
---|
264 | TkReading2Token[reading] = [token_c].concat(opts)
|
---|
265 | end
|
---|
266 | end
|
---|
267 | TkSymbol2Token[token_n.intern] = token_c
|
---|
268 | end
|
---|
269 |
|
---|
270 | for defs in TokenDefinitions
|
---|
271 | def_token(*defs)
|
---|
272 | end
|
---|
273 | end
|
---|