1 | #!/usr/local/bin/ruby
|
---|
2 |
|
---|
3 | # Parse a Ruby source file, building a set of objects
|
---|
4 | # representing the modules, classes, methods,
|
---|
5 | # requires, and includes we find (these classes
|
---|
6 | # are defined in code_objects.rb).
|
---|
7 |
|
---|
8 | # This file contains stuff stolen outright from:
|
---|
9 | #
|
---|
10 | # rtags.rb -
|
---|
11 | # ruby-lex.rb - ruby lexcal analizer
|
---|
12 | # ruby-token.rb - ruby tokens
|
---|
13 | # by Keiju ISHITSUKA (Nippon Rational Inc.)
|
---|
14 | #
|
---|
15 |
|
---|
16 | require "e2mmap"
|
---|
17 | require "irb/slex"
|
---|
18 |
|
---|
19 | require "rdoc/code_objects"
|
---|
20 | require "rdoc/tokenstream"
|
---|
21 |
|
---|
22 | require "rdoc/markup/simple_markup/preprocess"
|
---|
23 |
|
---|
24 | require "rdoc/parsers/parserfactory"
|
---|
25 |
|
---|
26 | $TOKEN_DEBUG = $DEBUG
|
---|
27 |
|
---|
28 | # Definitions of all tokens involved in the lexical analysis
|
---|
29 |
|
---|
30 | module RubyToken
|
---|
31 | EXPR_BEG = :EXPR_BEG
|
---|
32 | EXPR_MID = :EXPR_MID
|
---|
33 | EXPR_END = :EXPR_END
|
---|
34 | EXPR_ARG = :EXPR_ARG
|
---|
35 | EXPR_FNAME = :EXPR_FNAME
|
---|
36 | EXPR_DOT = :EXPR_DOT
|
---|
37 | EXPR_CLASS = :EXPR_CLASS
|
---|
38 |
|
---|
39 | class Token
|
---|
40 | NO_TEXT = "??".freeze
|
---|
41 | attr :text
|
---|
42 |
|
---|
43 | def initialize(line_no, char_no)
|
---|
44 | @line_no = line_no
|
---|
45 | @char_no = char_no
|
---|
46 | @text = NO_TEXT
|
---|
47 | end
|
---|
48 |
|
---|
49 | # Because we're used in contexts that expect to return a token,
|
---|
50 | # we set the text string and then return ourselves
|
---|
51 | def set_text(text)
|
---|
52 | @text = text
|
---|
53 | self
|
---|
54 | end
|
---|
55 |
|
---|
56 | attr_reader :line_no, :char_no, :text
|
---|
57 | end
|
---|
58 |
|
---|
59 | class TkNode < Token
|
---|
60 | attr :node
|
---|
61 | end
|
---|
62 |
|
---|
63 | class TkId < Token
|
---|
64 | def initialize(line_no, char_no, name)
|
---|
65 | super(line_no, char_no)
|
---|
66 | @name = name
|
---|
67 | end
|
---|
68 | attr :name
|
---|
69 | end
|
---|
70 |
|
---|
71 | class TkKW < TkId
|
---|
72 | end
|
---|
73 |
|
---|
74 | class TkVal < Token
|
---|
75 | def initialize(line_no, char_no, value = nil)
|
---|
76 | super(line_no, char_no)
|
---|
77 | set_text(value)
|
---|
78 | end
|
---|
79 | end
|
---|
80 |
|
---|
81 | class TkOp < Token
|
---|
82 | def name
|
---|
83 | self.class.op_name
|
---|
84 | end
|
---|
85 | end
|
---|
86 |
|
---|
87 | class TkOPASGN < TkOp
|
---|
88 | def initialize(line_no, char_no, op)
|
---|
89 | super(line_no, char_no)
|
---|
90 | op = TkReading2Token[op] unless op.kind_of?(Symbol)
|
---|
91 | @op = op
|
---|
92 | end
|
---|
93 | attr :op
|
---|
94 | end
|
---|
95 |
|
---|
96 | class TkUnknownChar < Token
|
---|
97 | def initialize(line_no, char_no, id)
|
---|
98 | super(line_no, char_no)
|
---|
99 | @name = char_no.chr
|
---|
100 | end
|
---|
101 | attr :name
|
---|
102 | end
|
---|
103 |
|
---|
104 | class TkError < Token
|
---|
105 | end
|
---|
106 |
|
---|
107 | def set_token_position(line, char)
|
---|
108 | @prev_line_no = line
|
---|
109 | @prev_char_no = char
|
---|
110 | end
|
---|
111 |
|
---|
112 | def Token(token, value = nil)
|
---|
113 | tk = nil
|
---|
114 | case token
|
---|
115 | when String, Symbol
|
---|
116 | source = token.kind_of?(String) ? TkReading2Token : TkSymbol2Token
|
---|
117 | if (tk = source[token]).nil?
|
---|
118 | IRB.fail TkReading2TokenNoKey, token
|
---|
119 | end
|
---|
120 | tk = Token(tk[0], value)
|
---|
121 | else
|
---|
122 | tk = if (token.ancestors & [TkId, TkVal, TkOPASGN, TkUnknownChar]).empty?
|
---|
123 | token.new(@prev_line_no, @prev_char_no)
|
---|
124 | else
|
---|
125 | token.new(@prev_line_no, @prev_char_no, value)
|
---|
126 | end
|
---|
127 | end
|
---|
128 | tk
|
---|
129 | end
|
---|
130 |
|
---|
131 | TokenDefinitions = [
|
---|
132 | [:TkCLASS, TkKW, "class", EXPR_CLASS],
|
---|
133 | [:TkMODULE, TkKW, "module", EXPR_BEG],
|
---|
134 | [:TkDEF, TkKW, "def", EXPR_FNAME],
|
---|
135 | [:TkUNDEF, TkKW, "undef", EXPR_FNAME],
|
---|
136 | [:TkBEGIN, TkKW, "begin", EXPR_BEG],
|
---|
137 | [:TkRESCUE, TkKW, "rescue", EXPR_MID],
|
---|
138 | [:TkENSURE, TkKW, "ensure", EXPR_BEG],
|
---|
139 | [:TkEND, TkKW, "end", EXPR_END],
|
---|
140 | [:TkIF, TkKW, "if", EXPR_BEG, :TkIF_MOD],
|
---|
141 | [:TkUNLESS, TkKW, "unless", EXPR_BEG, :TkUNLESS_MOD],
|
---|
142 | [:TkTHEN, TkKW, "then", EXPR_BEG],
|
---|
143 | [:TkELSIF, TkKW, "elsif", EXPR_BEG],
|
---|
144 | [:TkELSE, TkKW, "else", EXPR_BEG],
|
---|
145 | [:TkCASE, TkKW, "case", EXPR_BEG],
|
---|
146 | [:TkWHEN, TkKW, "when", EXPR_BEG],
|
---|
147 | [:TkWHILE, TkKW, "while", EXPR_BEG, :TkWHILE_MOD],
|
---|
148 | [:TkUNTIL, TkKW, "until", EXPR_BEG, :TkUNTIL_MOD],
|
---|
149 | [:TkFOR, TkKW, "for", EXPR_BEG],
|
---|
150 | [:TkBREAK, TkKW, "break", EXPR_END],
|
---|
151 | [:TkNEXT, TkKW, "next", EXPR_END],
|
---|
152 | [:TkREDO, TkKW, "redo", EXPR_END],
|
---|
153 | [:TkRETRY, TkKW, "retry", EXPR_END],
|
---|
154 | [:TkIN, TkKW, "in", EXPR_BEG],
|
---|
155 | [:TkDO, TkKW, "do", EXPR_BEG],
|
---|
156 | [:TkRETURN, TkKW, "return", EXPR_MID],
|
---|
157 | [:TkYIELD, TkKW, "yield", EXPR_END],
|
---|
158 | [:TkSUPER, TkKW, "super", EXPR_END],
|
---|
159 | [:TkSELF, TkKW, "self", EXPR_END],
|
---|
160 | [:TkNIL, TkKW, "nil", EXPR_END],
|
---|
161 | [:TkTRUE, TkKW, "true", EXPR_END],
|
---|
162 | [:TkFALSE, TkKW, "false", EXPR_END],
|
---|
163 | [:TkAND, TkKW, "and", EXPR_BEG],
|
---|
164 | [:TkOR, TkKW, "or", EXPR_BEG],
|
---|
165 | [:TkNOT, TkKW, "not", EXPR_BEG],
|
---|
166 | [:TkIF_MOD, TkKW],
|
---|
167 | [:TkUNLESS_MOD, TkKW],
|
---|
168 | [:TkWHILE_MOD, TkKW],
|
---|
169 | [:TkUNTIL_MOD, TkKW],
|
---|
170 | [:TkALIAS, TkKW, "alias", EXPR_FNAME],
|
---|
171 | [:TkDEFINED, TkKW, "defined?", EXPR_END],
|
---|
172 | [:TklBEGIN, TkKW, "BEGIN", EXPR_END],
|
---|
173 | [:TklEND, TkKW, "END", EXPR_END],
|
---|
174 | [:Tk__LINE__, TkKW, "__LINE__", EXPR_END],
|
---|
175 | [:Tk__FILE__, TkKW, "__FILE__", EXPR_END],
|
---|
176 |
|
---|
177 | [:TkIDENTIFIER, TkId],
|
---|
178 | [:TkFID, TkId],
|
---|
179 | [:TkGVAR, TkId],
|
---|
180 | [:TkIVAR, TkId],
|
---|
181 | [:TkCONSTANT, TkId],
|
---|
182 |
|
---|
183 | [:TkINTEGER, TkVal],
|
---|
184 | [:TkFLOAT, TkVal],
|
---|
185 | [:TkSTRING, TkVal],
|
---|
186 | [:TkXSTRING, TkVal],
|
---|
187 | [:TkREGEXP, TkVal],
|
---|
188 | [:TkCOMMENT, TkVal],
|
---|
189 |
|
---|
190 | [:TkDSTRING, TkNode],
|
---|
191 | [:TkDXSTRING, TkNode],
|
---|
192 | [:TkDREGEXP, TkNode],
|
---|
193 | [:TkNTH_REF, TkId],
|
---|
194 | [:TkBACK_REF, TkId],
|
---|
195 |
|
---|
196 | [:TkUPLUS, TkOp, "+@"],
|
---|
197 | [:TkUMINUS, TkOp, "-@"],
|
---|
198 | [:TkPOW, TkOp, "**"],
|
---|
199 | [:TkCMP, TkOp, "<=>"],
|
---|
200 | [:TkEQ, TkOp, "=="],
|
---|
201 | [:TkEQQ, TkOp, "==="],
|
---|
202 | [:TkNEQ, TkOp, "!="],
|
---|
203 | [:TkGEQ, TkOp, ">="],
|
---|
204 | [:TkLEQ, TkOp, "<="],
|
---|
205 | [:TkANDOP, TkOp, "&&"],
|
---|
206 | [:TkOROP, TkOp, "||"],
|
---|
207 | [:TkMATCH, TkOp, "=~"],
|
---|
208 | [:TkNMATCH, TkOp, "!~"],
|
---|
209 | [:TkDOT2, TkOp, ".."],
|
---|
210 | [:TkDOT3, TkOp, "..."],
|
---|
211 | [:TkAREF, TkOp, "[]"],
|
---|
212 | [:TkASET, TkOp, "[]="],
|
---|
213 | [:TkLSHFT, TkOp, "<<"],
|
---|
214 | [:TkRSHFT, TkOp, ">>"],
|
---|
215 | [:TkCOLON2, TkOp],
|
---|
216 | [:TkCOLON3, TkOp],
|
---|
217 | # [:OPASGN, TkOp], # +=, -= etc. #
|
---|
218 | [:TkASSOC, TkOp, "=>"],
|
---|
219 | [:TkQUESTION, TkOp, "?"], #?
|
---|
220 | [:TkCOLON, TkOp, ":"], #:
|
---|
221 |
|
---|
222 | [:TkfLPAREN], # func( #
|
---|
223 | [:TkfLBRACK], # func[ #
|
---|
224 | [:TkfLBRACE], # func{ #
|
---|
225 | [:TkSTAR], # *arg
|
---|
226 | [:TkAMPER], # &arg #
|
---|
227 | [:TkSYMBOL, TkId], # :SYMBOL
|
---|
228 | [:TkSYMBEG, TkId],
|
---|
229 | [:TkGT, TkOp, ">"],
|
---|
230 | [:TkLT, TkOp, "<"],
|
---|
231 | [:TkPLUS, TkOp, "+"],
|
---|
232 | [:TkMINUS, TkOp, "-"],
|
---|
233 | [:TkMULT, TkOp, "*"],
|
---|
234 | [:TkDIV, TkOp, "/"],
|
---|
235 | [:TkMOD, TkOp, "%"],
|
---|
236 | [:TkBITOR, TkOp, "|"],
|
---|
237 | [:TkBITXOR, TkOp, "^"],
|
---|
238 | [:TkBITAND, TkOp, "&"],
|
---|
239 | [:TkBITNOT, TkOp, "~"],
|
---|
240 | [:TkNOTOP, TkOp, "!"],
|
---|
241 |
|
---|
242 | [:TkBACKQUOTE, TkOp, "`"],
|
---|
243 |
|
---|
244 | [:TkASSIGN, Token, "="],
|
---|
245 | [:TkDOT, Token, "."],
|
---|
246 | [:TkLPAREN, Token, "("], #(exp)
|
---|
247 | [:TkLBRACK, Token, "["], #[arry]
|
---|
248 | [:TkLBRACE, Token, "{"], #{hash}
|
---|
249 | [:TkRPAREN, Token, ")"],
|
---|
250 | [:TkRBRACK, Token, "]"],
|
---|
251 | [:TkRBRACE, Token, "}"],
|
---|
252 | [:TkCOMMA, Token, ","],
|
---|
253 | [:TkSEMICOLON, Token, ";"],
|
---|
254 |
|
---|
255 | [:TkRD_COMMENT],
|
---|
256 | [:TkSPACE],
|
---|
257 | [:TkNL],
|
---|
258 | [:TkEND_OF_SCRIPT],
|
---|
259 |
|
---|
260 | [:TkBACKSLASH, TkUnknownChar, "\\"],
|
---|
261 | [:TkAT, TkUnknownChar, "@"],
|
---|
262 | [:TkDOLLAR, TkUnknownChar, "\$"], #"
|
---|
263 | ]
|
---|
264 |
|
---|
265 | # {reading => token_class}
|
---|
266 | # {reading => [token_class, *opt]}
|
---|
267 | TkReading2Token = {}
|
---|
268 | TkSymbol2Token = {}
|
---|
269 |
|
---|
270 | def RubyToken.def_token(token_n, super_token = Token, reading = nil, *opts)
|
---|
271 | token_n = token_n.id2name unless token_n.kind_of?(String)
|
---|
272 | if RubyToken.const_defined?(token_n)
|
---|
273 | IRB.fail AlreadyDefinedToken, token_n
|
---|
274 | end
|
---|
275 |
|
---|
276 | token_c = Class.new super_token
|
---|
277 | RubyToken.const_set token_n, token_c
|
---|
278 | # token_c.inspect
|
---|
279 |
|
---|
280 | if reading
|
---|
281 | if TkReading2Token[reading]
|
---|
282 | IRB.fail TkReading2TokenDuplicateError, token_n, reading
|
---|
283 | end
|
---|
284 | if opts.empty?
|
---|
285 | TkReading2Token[reading] = [token_c]
|
---|
286 | else
|
---|
287 | TkReading2Token[reading] = [token_c].concat(opts)
|
---|
288 | end
|
---|
289 | end
|
---|
290 | TkSymbol2Token[token_n.intern] = token_c
|
---|
291 |
|
---|
292 | if token_c <= TkOp
|
---|
293 | token_c.class_eval %{
|
---|
294 | def self.op_name; "#{reading}"; end
|
---|
295 | }
|
---|
296 | end
|
---|
297 | end
|
---|
298 |
|
---|
299 | for defs in TokenDefinitions
|
---|
300 | def_token(*defs)
|
---|
301 | end
|
---|
302 |
|
---|
303 | NEWLINE_TOKEN = TkNL.new(0,0)
|
---|
304 | NEWLINE_TOKEN.set_text("\n")
|
---|
305 |
|
---|
306 | end
|
---|
307 |
|
---|
308 |
|
---|
309 |
|
---|
310 | # Lexical analyzer for Ruby source
|
---|
311 |
|
---|
312 | class RubyLex
|
---|
313 |
|
---|
314 | ######################################################################
|
---|
315 | #
|
---|
316 | # Read an input stream character by character. We allow for unlimited
|
---|
317 | # ungetting of characters just read.
|
---|
318 | #
|
---|
319 | # We simplify the implementation greatly by reading the entire input
|
---|
320 | # into a buffer initially, and then simply traversing it using
|
---|
321 | # pointers.
|
---|
322 | #
|
---|
323 | # We also have to allow for the <i>here document diversion</i>. This
|
---|
324 | # little gem comes about when the lexer encounters a here
|
---|
325 | # document. At this point we effectively need to split the input
|
---|
326 | # stream into two parts: one to read the body of the here document,
|
---|
327 | # the other to read the rest of the input line where the here
|
---|
328 | # document was initially encountered. For example, we might have
|
---|
329 | #
|
---|
330 | # do_something(<<-A, <<-B)
|
---|
331 | # stuff
|
---|
332 | # for
|
---|
333 | # A
|
---|
334 | # stuff
|
---|
335 | # for
|
---|
336 | # B
|
---|
337 | #
|
---|
338 | # When the lexer encounters the <<A, it reads until the end of the
|
---|
339 | # line, and keeps it around for later. It then reads the body of the
|
---|
340 | # here document. Once complete, it needs to read the rest of the
|
---|
341 | # original line, but then skip the here document body.
|
---|
342 | #
|
---|
343 |
|
---|
344 | class BufferedReader
|
---|
345 |
|
---|
346 | attr_reader :line_num
|
---|
347 |
|
---|
348 | def initialize(content)
|
---|
349 | if /\t/ =~ content
|
---|
350 | tab_width = Options.instance.tab_width
|
---|
351 | content = content.split(/\n/).map do |line|
|
---|
352 | 1 while line.gsub!(/\t+/) { ' ' * (tab_width*$&.length - $`.length % tab_width)} && $~ #`
|
---|
353 | line
|
---|
354 | end .join("\n")
|
---|
355 | end
|
---|
356 | @content = content
|
---|
357 | @content << "\n" unless @content[-1,1] == "\n"
|
---|
358 | @size = @content.size
|
---|
359 | @offset = 0
|
---|
360 | @hwm = 0
|
---|
361 | @line_num = 1
|
---|
362 | @read_back_offset = 0
|
---|
363 | @last_newline = 0
|
---|
364 | @newline_pending = false
|
---|
365 | end
|
---|
366 |
|
---|
367 | def column
|
---|
368 | @offset - @last_newline
|
---|
369 | end
|
---|
370 |
|
---|
371 | def getc
|
---|
372 | return nil if @offset >= @size
|
---|
373 | ch = @content[@offset, 1]
|
---|
374 |
|
---|
375 | @offset += 1
|
---|
376 | @hwm = @offset if @hwm < @offset
|
---|
377 |
|
---|
378 | if @newline_pending
|
---|
379 | @line_num += 1
|
---|
380 | @last_newline = @offset - 1
|
---|
381 | @newline_pending = false
|
---|
382 | end
|
---|
383 |
|
---|
384 | if ch == "\n"
|
---|
385 | @newline_pending = true
|
---|
386 | end
|
---|
387 | ch
|
---|
388 | end
|
---|
389 |
|
---|
390 | def getc_already_read
|
---|
391 | getc
|
---|
392 | end
|
---|
393 |
|
---|
394 | def ungetc(ch)
|
---|
395 | raise "unget past beginning of file" if @offset <= 0
|
---|
396 | @offset -= 1
|
---|
397 | if @content[@offset] == ?\n
|
---|
398 | @newline_pending = false
|
---|
399 | end
|
---|
400 | end
|
---|
401 |
|
---|
402 | def get_read
|
---|
403 | res = @content[@read_back_offset...@offset]
|
---|
404 | @read_back_offset = @offset
|
---|
405 | res
|
---|
406 | end
|
---|
407 |
|
---|
408 | def peek(at)
|
---|
409 | pos = @offset + at
|
---|
410 | if pos >= @size
|
---|
411 | nil
|
---|
412 | else
|
---|
413 | @content[pos, 1]
|
---|
414 | end
|
---|
415 | end
|
---|
416 |
|
---|
417 | def peek_equal(str)
|
---|
418 | @content[@offset, str.length] == str
|
---|
419 | end
|
---|
420 |
|
---|
421 | def divert_read_from(reserve)
|
---|
422 | @content[@offset, 0] = reserve
|
---|
423 | @size = @content.size
|
---|
424 | end
|
---|
425 | end
|
---|
426 |
|
---|
427 | # end of nested class BufferedReader
|
---|
428 |
|
---|
429 | extend Exception2MessageMapper
|
---|
430 | def_exception(:AlreadyDefinedToken, "Already defined token(%s)")
|
---|
431 | def_exception(:TkReading2TokenNoKey, "key nothing(key='%s')")
|
---|
432 | def_exception(:TkSymbol2TokenNoKey, "key nothing(key='%s')")
|
---|
433 | def_exception(:TkReading2TokenDuplicateError,
|
---|
434 | "key duplicate(token_n='%s', key='%s')")
|
---|
435 | def_exception(:SyntaxError, "%s")
|
---|
436 |
|
---|
437 | include RubyToken
|
---|
438 | include IRB
|
---|
439 |
|
---|
440 | attr_reader :continue
|
---|
441 | attr_reader :lex_state
|
---|
442 |
|
---|
443 | def RubyLex.debug?
|
---|
444 | false
|
---|
445 | end
|
---|
446 |
|
---|
447 | def initialize(content)
|
---|
448 | lex_init
|
---|
449 |
|
---|
450 | @reader = BufferedReader.new(content)
|
---|
451 |
|
---|
452 | @exp_line_no = @line_no = 1
|
---|
453 | @base_char_no = 0
|
---|
454 | @indent = 0
|
---|
455 |
|
---|
456 | @ltype = nil
|
---|
457 | @quoted = nil
|
---|
458 | @lex_state = EXPR_BEG
|
---|
459 | @space_seen = false
|
---|
460 |
|
---|
461 | @continue = false
|
---|
462 | @line = ""
|
---|
463 |
|
---|
464 | @skip_space = false
|
---|
465 | @read_auto_clean_up = false
|
---|
466 | @exception_on_syntax_error = true
|
---|
467 | end
|
---|
468 |
|
---|
469 | attr :skip_space, true
|
---|
470 | attr :read_auto_clean_up, true
|
---|
471 | attr :exception_on_syntax_error, true
|
---|
472 |
|
---|
473 | attr :indent
|
---|
474 |
|
---|
475 | # io functions
|
---|
476 | def line_no
|
---|
477 | @reader.line_num
|
---|
478 | end
|
---|
479 |
|
---|
480 | def char_no
|
---|
481 | @reader.column
|
---|
482 | end
|
---|
483 |
|
---|
484 | def get_read
|
---|
485 | @reader.get_read
|
---|
486 | end
|
---|
487 |
|
---|
488 | def getc
|
---|
489 | @reader.getc
|
---|
490 | end
|
---|
491 |
|
---|
492 | def getc_of_rests
|
---|
493 | @reader.getc_already_read
|
---|
494 | end
|
---|
495 |
|
---|
496 | def gets
|
---|
497 | c = getc or return
|
---|
498 | l = ""
|
---|
499 | begin
|
---|
500 | l.concat c unless c == "\r"
|
---|
501 | break if c == "\n"
|
---|
502 | end while c = getc
|
---|
503 | l
|
---|
504 | end
|
---|
505 |
|
---|
506 |
|
---|
507 | def ungetc(c = nil)
|
---|
508 | @reader.ungetc(c)
|
---|
509 | end
|
---|
510 |
|
---|
511 | def peek_equal?(str)
|
---|
512 | @reader.peek_equal(str)
|
---|
513 | end
|
---|
514 |
|
---|
515 | def peek(i = 0)
|
---|
516 | @reader.peek(i)
|
---|
517 | end
|
---|
518 |
|
---|
519 | def lex
|
---|
520 | until (((tk = token).kind_of?(TkNL) || tk.kind_of?(TkEND_OF_SCRIPT)) &&
|
---|
521 | !@continue or
|
---|
522 | tk.nil?)
|
---|
523 | end
|
---|
524 | line = get_read
|
---|
525 |
|
---|
526 | if line == "" and tk.kind_of?(TkEND_OF_SCRIPT) || tk.nil?
|
---|
527 | nil
|
---|
528 | else
|
---|
529 | line
|
---|
530 | end
|
---|
531 | end
|
---|
532 |
|
---|
533 | def token
|
---|
534 | set_token_position(line_no, char_no)
|
---|
535 | begin
|
---|
536 | begin
|
---|
537 | tk = @OP.match(self)
|
---|
538 | @space_seen = tk.kind_of?(TkSPACE)
|
---|
539 | rescue SyntaxError
|
---|
540 | abort if @exception_on_syntax_error
|
---|
541 | tk = TkError.new(line_no, char_no)
|
---|
542 | end
|
---|
543 | end while @skip_space and tk.kind_of?(TkSPACE)
|
---|
544 | if @read_auto_clean_up
|
---|
545 | get_read
|
---|
546 | end
|
---|
547 | # throw :eof unless tk
|
---|
548 | p tk if $DEBUG
|
---|
549 | tk
|
---|
550 | end
|
---|
551 |
|
---|
552 | ENINDENT_CLAUSE = [
|
---|
553 | "case", "class", "def", "do", "for", "if",
|
---|
554 | "module", "unless", "until", "while", "begin" #, "when"
|
---|
555 | ]
|
---|
556 | DEINDENT_CLAUSE = ["end" #, "when"
|
---|
557 | ]
|
---|
558 |
|
---|
559 | PERCENT_LTYPE = {
|
---|
560 | "q" => "\'",
|
---|
561 | "Q" => "\"",
|
---|
562 | "x" => "\`",
|
---|
563 | "r" => "/",
|
---|
564 | "w" => "]"
|
---|
565 | }
|
---|
566 |
|
---|
567 | PERCENT_PAREN = {
|
---|
568 | "{" => "}",
|
---|
569 | "[" => "]",
|
---|
570 | "<" => ">",
|
---|
571 | "(" => ")"
|
---|
572 | }
|
---|
573 |
|
---|
574 | Ltype2Token = {
|
---|
575 | "\'" => TkSTRING,
|
---|
576 | "\"" => TkSTRING,
|
---|
577 | "\`" => TkXSTRING,
|
---|
578 | "/" => TkREGEXP,
|
---|
579 | "]" => TkDSTRING
|
---|
580 | }
|
---|
581 | Ltype2Token.default = TkSTRING
|
---|
582 |
|
---|
583 | DLtype2Token = {
|
---|
584 | "\"" => TkDSTRING,
|
---|
585 | "\`" => TkDXSTRING,
|
---|
586 | "/" => TkDREGEXP,
|
---|
587 | }
|
---|
588 |
|
---|
589 | def lex_init()
|
---|
590 | @OP = SLex.new
|
---|
591 | @OP.def_rules("\0", "\004", "\032") do |chars, io|
|
---|
592 | Token(TkEND_OF_SCRIPT).set_text(chars)
|
---|
593 | end
|
---|
594 |
|
---|
595 | @OP.def_rules(" ", "\t", "\f", "\r", "\13") do |chars, io|
|
---|
596 | @space_seen = TRUE
|
---|
597 | while (ch = getc) =~ /[ \t\f\r\13]/
|
---|
598 | chars << ch
|
---|
599 | end
|
---|
600 | ungetc
|
---|
601 | Token(TkSPACE).set_text(chars)
|
---|
602 | end
|
---|
603 |
|
---|
604 | @OP.def_rule("#") do
|
---|
605 | |op, io|
|
---|
606 | identify_comment
|
---|
607 | end
|
---|
608 |
|
---|
609 | @OP.def_rule("=begin", proc{@prev_char_no == 0 && peek(0) =~ /\s/}) do
|
---|
610 | |op, io|
|
---|
611 | str = op
|
---|
612 | @ltype = "="
|
---|
613 |
|
---|
614 |
|
---|
615 | begin
|
---|
616 | line = ""
|
---|
617 | begin
|
---|
618 | ch = getc
|
---|
619 | line << ch
|
---|
620 | end until ch == "\n"
|
---|
621 | str << line
|
---|
622 | end until line =~ /^=end/
|
---|
623 |
|
---|
624 | ungetc
|
---|
625 |
|
---|
626 | @ltype = nil
|
---|
627 |
|
---|
628 | if str =~ /\A=begin\s+rdoc/i
|
---|
629 | str.sub!(/\A=begin.*\n/, '')
|
---|
630 | str.sub!(/^=end.*/m, '')
|
---|
631 | Token(TkCOMMENT).set_text(str)
|
---|
632 | else
|
---|
633 | Token(TkRD_COMMENT)#.set_text(str)
|
---|
634 | end
|
---|
635 | end
|
---|
636 |
|
---|
637 | @OP.def_rule("\n") do
|
---|
638 | print "\\n\n" if RubyLex.debug?
|
---|
639 | case @lex_state
|
---|
640 | when EXPR_BEG, EXPR_FNAME, EXPR_DOT
|
---|
641 | @continue = TRUE
|
---|
642 | else
|
---|
643 | @continue = FALSE
|
---|
644 | @lex_state = EXPR_BEG
|
---|
645 | end
|
---|
646 | Token(TkNL).set_text("\n")
|
---|
647 | end
|
---|
648 |
|
---|
649 | @OP.def_rules("*", "**",
|
---|
650 | "!", "!=", "!~",
|
---|
651 | "=", "==", "===",
|
---|
652 | "=~", "<=>",
|
---|
653 | "<", "<=",
|
---|
654 | ">", ">=", ">>") do
|
---|
655 | |op, io|
|
---|
656 | @lex_state = EXPR_BEG
|
---|
657 | Token(op).set_text(op)
|
---|
658 | end
|
---|
659 |
|
---|
660 | @OP.def_rules("<<") do
|
---|
661 | |op, io|
|
---|
662 | tk = nil
|
---|
663 | if @lex_state != EXPR_END && @lex_state != EXPR_CLASS &&
|
---|
664 | (@lex_state != EXPR_ARG || @space_seen)
|
---|
665 | c = peek(0)
|
---|
666 | if /[-\w_\"\'\`]/ =~ c
|
---|
667 | tk = identify_here_document
|
---|
668 | end
|
---|
669 | end
|
---|
670 | if !tk
|
---|
671 | @lex_state = EXPR_BEG
|
---|
672 | tk = Token(op).set_text(op)
|
---|
673 | end
|
---|
674 | tk
|
---|
675 | end
|
---|
676 |
|
---|
677 | @OP.def_rules("'", '"') do
|
---|
678 | |op, io|
|
---|
679 | identify_string(op)
|
---|
680 | end
|
---|
681 |
|
---|
682 | @OP.def_rules("`") do
|
---|
683 | |op, io|
|
---|
684 | if @lex_state == EXPR_FNAME
|
---|
685 | Token(op).set_text(op)
|
---|
686 | else
|
---|
687 | identify_string(op)
|
---|
688 | end
|
---|
689 | end
|
---|
690 |
|
---|
691 | @OP.def_rules('?') do
|
---|
692 | |op, io|
|
---|
693 | if @lex_state == EXPR_END
|
---|
694 | @lex_state = EXPR_BEG
|
---|
695 | Token(TkQUESTION).set_text(op)
|
---|
696 | else
|
---|
697 | ch = getc
|
---|
698 | if @lex_state == EXPR_ARG && ch !~ /\s/
|
---|
699 | ungetc
|
---|
700 | @lex_state = EXPR_BEG;
|
---|
701 | Token(TkQUESTION).set_text(op)
|
---|
702 | else
|
---|
703 | str = op
|
---|
704 | str << ch
|
---|
705 | if (ch == '\\') #'
|
---|
706 | str << read_escape
|
---|
707 | end
|
---|
708 | @lex_state = EXPR_END
|
---|
709 | Token(TkINTEGER).set_text(str)
|
---|
710 | end
|
---|
711 | end
|
---|
712 | end
|
---|
713 |
|
---|
714 | @OP.def_rules("&", "&&", "|", "||") do
|
---|
715 | |op, io|
|
---|
716 | @lex_state = EXPR_BEG
|
---|
717 | Token(op).set_text(op)
|
---|
718 | end
|
---|
719 |
|
---|
720 | @OP.def_rules("+=", "-=", "*=", "**=",
|
---|
721 | "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do
|
---|
722 | |op, io|
|
---|
723 | @lex_state = EXPR_BEG
|
---|
724 | op =~ /^(.*)=$/
|
---|
725 | Token(TkOPASGN, $1).set_text(op)
|
---|
726 | end
|
---|
727 |
|
---|
728 | @OP.def_rule("+@", proc{@lex_state == EXPR_FNAME}) do |op, io|
|
---|
729 | Token(TkUPLUS).set_text(op)
|
---|
730 | end
|
---|
731 |
|
---|
732 | @OP.def_rule("-@", proc{@lex_state == EXPR_FNAME}) do |op, io|
|
---|
733 | Token(TkUMINUS).set_text(op)
|
---|
734 | end
|
---|
735 |
|
---|
736 | @OP.def_rules("+", "-") do
|
---|
737 | |op, io|
|
---|
738 | catch(:RET) do
|
---|
739 | if @lex_state == EXPR_ARG
|
---|
740 | if @space_seen and peek(0) =~ /[0-9]/
|
---|
741 | throw :RET, identify_number(op)
|
---|
742 | else
|
---|
743 | @lex_state = EXPR_BEG
|
---|
744 | end
|
---|
745 | elsif @lex_state != EXPR_END and peek(0) =~ /[0-9]/
|
---|
746 | throw :RET, identify_number(op)
|
---|
747 | else
|
---|
748 | @lex_state = EXPR_BEG
|
---|
749 | end
|
---|
750 | Token(op).set_text(op)
|
---|
751 | end
|
---|
752 | end
|
---|
753 |
|
---|
754 | @OP.def_rule(".") do
|
---|
755 | @lex_state = EXPR_BEG
|
---|
756 | if peek(0) =~ /[0-9]/
|
---|
757 | ungetc
|
---|
758 | identify_number("")
|
---|
759 | else
|
---|
760 | # for obj.if
|
---|
761 | @lex_state = EXPR_DOT
|
---|
762 | Token(TkDOT).set_text(".")
|
---|
763 | end
|
---|
764 | end
|
---|
765 |
|
---|
766 | @OP.def_rules("..", "...") do
|
---|
767 | |op, io|
|
---|
768 | @lex_state = EXPR_BEG
|
---|
769 | Token(op).set_text(op)
|
---|
770 | end
|
---|
771 |
|
---|
772 | lex_int2
|
---|
773 | end
|
---|
774 |
|
---|
775 | def lex_int2
|
---|
776 | @OP.def_rules("]", "}", ")") do
|
---|
777 | |op, io|
|
---|
778 | @lex_state = EXPR_END
|
---|
779 | @indent -= 1
|
---|
780 | Token(op).set_text(op)
|
---|
781 | end
|
---|
782 |
|
---|
783 | @OP.def_rule(":") do
|
---|
784 | if @lex_state == EXPR_END || peek(0) =~ /\s/
|
---|
785 | @lex_state = EXPR_BEG
|
---|
786 | tk = Token(TkCOLON)
|
---|
787 | else
|
---|
788 | @lex_state = EXPR_FNAME;
|
---|
789 | tk = Token(TkSYMBEG)
|
---|
790 | end
|
---|
791 | tk.set_text(":")
|
---|
792 | end
|
---|
793 |
|
---|
794 | @OP.def_rule("::") do
|
---|
795 | # p @lex_state.id2name, @space_seen
|
---|
796 | if @lex_state == EXPR_BEG or @lex_state == EXPR_ARG && @space_seen
|
---|
797 | @lex_state = EXPR_BEG
|
---|
798 | tk = Token(TkCOLON3)
|
---|
799 | else
|
---|
800 | @lex_state = EXPR_DOT
|
---|
801 | tk = Token(TkCOLON2)
|
---|
802 | end
|
---|
803 | tk.set_text("::")
|
---|
804 | end
|
---|
805 |
|
---|
806 | @OP.def_rule("/") do
|
---|
807 | |op, io|
|
---|
808 | if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
|
---|
809 | identify_string(op)
|
---|
810 | elsif peek(0) == '='
|
---|
811 | getc
|
---|
812 | @lex_state = EXPR_BEG
|
---|
813 | Token(TkOPASGN, :/).set_text("/=") #")
|
---|
814 | elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
|
---|
815 | identify_string(op)
|
---|
816 | else
|
---|
817 | @lex_state = EXPR_BEG
|
---|
818 | Token("/").set_text(op)
|
---|
819 | end
|
---|
820 | end
|
---|
821 |
|
---|
822 | @OP.def_rules("^") do
|
---|
823 | @lex_state = EXPR_BEG
|
---|
824 | Token("^").set_text("^")
|
---|
825 | end
|
---|
826 |
|
---|
827 | # @OP.def_rules("^=") do
|
---|
828 | # @lex_state = EXPR_BEG
|
---|
829 | # Token(TkOPASGN, :^)
|
---|
830 | # end
|
---|
831 |
|
---|
832 | @OP.def_rules(",", ";") do
|
---|
833 | |op, io|
|
---|
834 | @lex_state = EXPR_BEG
|
---|
835 | Token(op).set_text(op)
|
---|
836 | end
|
---|
837 |
|
---|
838 | @OP.def_rule("~") do
|
---|
839 | @lex_state = EXPR_BEG
|
---|
840 | Token("~").set_text("~")
|
---|
841 | end
|
---|
842 |
|
---|
843 | @OP.def_rule("~@", proc{@lex_state = EXPR_FNAME}) do
|
---|
844 | @lex_state = EXPR_BEG
|
---|
845 | Token("~").set_text("~@")
|
---|
846 | end
|
---|
847 |
|
---|
848 | @OP.def_rule("(") do
|
---|
849 | @indent += 1
|
---|
850 | if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
|
---|
851 | @lex_state = EXPR_BEG
|
---|
852 | tk = Token(TkfLPAREN)
|
---|
853 | else
|
---|
854 | @lex_state = EXPR_BEG
|
---|
855 | tk = Token(TkLPAREN)
|
---|
856 | end
|
---|
857 | tk.set_text("(")
|
---|
858 | end
|
---|
859 |
|
---|
860 | @OP.def_rule("[]", proc{@lex_state == EXPR_FNAME}) do
|
---|
861 | Token("[]").set_text("[]")
|
---|
862 | end
|
---|
863 |
|
---|
864 | @OP.def_rule("[]=", proc{@lex_state == EXPR_FNAME}) do
|
---|
865 | Token("[]=").set_text("[]=")
|
---|
866 | end
|
---|
867 |
|
---|
868 | @OP.def_rule("[") do
|
---|
869 | @indent += 1
|
---|
870 | if @lex_state == EXPR_FNAME
|
---|
871 | t = Token(TkfLBRACK)
|
---|
872 | else
|
---|
873 | if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
|
---|
874 | t = Token(TkLBRACK)
|
---|
875 | elsif @lex_state == EXPR_ARG && @space_seen
|
---|
876 | t = Token(TkLBRACK)
|
---|
877 | else
|
---|
878 | t = Token(TkfLBRACK)
|
---|
879 | end
|
---|
880 | @lex_state = EXPR_BEG
|
---|
881 | end
|
---|
882 | t.set_text("[")
|
---|
883 | end
|
---|
884 |
|
---|
885 | @OP.def_rule("{") do
|
---|
886 | @indent += 1
|
---|
887 | if @lex_state != EXPR_END && @lex_state != EXPR_ARG
|
---|
888 | t = Token(TkLBRACE)
|
---|
889 | else
|
---|
890 | t = Token(TkfLBRACE)
|
---|
891 | end
|
---|
892 | @lex_state = EXPR_BEG
|
---|
893 | t.set_text("{")
|
---|
894 | end
|
---|
895 |
|
---|
896 | @OP.def_rule('\\') do #'
|
---|
897 | if getc == "\n"
|
---|
898 | @space_seen = true
|
---|
899 | @continue = true
|
---|
900 | Token(TkSPACE).set_text("\\\n")
|
---|
901 | else
|
---|
902 | ungetc
|
---|
903 | Token("\\").set_text("\\") #"
|
---|
904 | end
|
---|
905 | end
|
---|
906 |
|
---|
907 | @OP.def_rule('%') do
|
---|
908 | |op, io|
|
---|
909 | if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
|
---|
910 | identify_quotation('%')
|
---|
911 | elsif peek(0) == '='
|
---|
912 | getc
|
---|
913 | Token(TkOPASGN, "%").set_text("%=")
|
---|
914 | elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
|
---|
915 | identify_quotation('%')
|
---|
916 | else
|
---|
917 | @lex_state = EXPR_BEG
|
---|
918 | Token("%").set_text("%")
|
---|
919 | end
|
---|
920 | end
|
---|
921 |
|
---|
922 | @OP.def_rule('$') do #'
|
---|
923 | identify_gvar
|
---|
924 | end
|
---|
925 |
|
---|
926 | @OP.def_rule('@') do
|
---|
927 | if peek(0) =~ /[@\w_]/
|
---|
928 | ungetc
|
---|
929 | identify_identifier
|
---|
930 | else
|
---|
931 | Token("@").set_text("@")
|
---|
932 | end
|
---|
933 | end
|
---|
934 |
|
---|
935 | # @OP.def_rule("def", proc{|op, io| /\s/ =~ io.peek(0)}) do
|
---|
936 | # |op, io|
|
---|
937 | # @indent += 1
|
---|
938 | # @lex_state = EXPR_FNAME
|
---|
939 | # # @lex_state = EXPR_END
|
---|
940 | # # until @rests[0] == "\n" or @rests[0] == ";"
|
---|
941 | # # rests.shift
|
---|
942 | # # end
|
---|
943 | # end
|
---|
944 |
|
---|
945 | @OP.def_rule("__END__", proc{@prev_char_no == 0 && peek(0) =~ /[\r\n]/}) do
|
---|
946 | throw :eof
|
---|
947 | end
|
---|
948 |
|
---|
949 | @OP.def_rule("") do
|
---|
950 | |op, io|
|
---|
951 | printf "MATCH: start %s: %s\n", op, io.inspect if RubyLex.debug?
|
---|
952 | if peek(0) =~ /[0-9]/
|
---|
953 | t = identify_number("")
|
---|
954 | elsif peek(0) =~ /[\w_]/
|
---|
955 | t = identify_identifier
|
---|
956 | end
|
---|
957 | printf "MATCH: end %s: %s\n", op, io.inspect if RubyLex.debug?
|
---|
958 | t
|
---|
959 | end
|
---|
960 |
|
---|
961 | p @OP if RubyLex.debug?
|
---|
962 | end
|
---|
963 |
|
---|
964 | def identify_gvar
|
---|
965 | @lex_state = EXPR_END
|
---|
966 | str = "$"
|
---|
967 |
|
---|
968 | tk = case ch = getc
|
---|
969 | when /[~_*$?!@\/\\;,=:<>".]/ #"
|
---|
970 | str << ch
|
---|
971 | Token(TkGVAR, str)
|
---|
972 |
|
---|
973 | when "-"
|
---|
974 | str << "-" << getc
|
---|
975 | Token(TkGVAR, str)
|
---|
976 |
|
---|
977 | when "&", "`", "'", "+"
|
---|
978 | str << ch
|
---|
979 | Token(TkBACK_REF, str)
|
---|
980 |
|
---|
981 | when /[1-9]/
|
---|
982 | str << ch
|
---|
983 | while (ch = getc) =~ /[0-9]/
|
---|
984 | str << ch
|
---|
985 | end
|
---|
986 | ungetc
|
---|
987 | Token(TkNTH_REF)
|
---|
988 | when /\w/
|
---|
989 | ungetc
|
---|
990 | ungetc
|
---|
991 | return identify_identifier
|
---|
992 | else
|
---|
993 | ungetc
|
---|
994 | Token("$")
|
---|
995 | end
|
---|
996 | tk.set_text(str)
|
---|
997 | end
|
---|
998 |
|
---|
999 | def identify_identifier
|
---|
1000 | token = ""
|
---|
1001 | token.concat getc if peek(0) =~ /[$@]/
|
---|
1002 | token.concat getc if peek(0) == "@"
|
---|
1003 |
|
---|
1004 | while (ch = getc) =~ /\w|_/
|
---|
1005 | print ":", ch, ":" if RubyLex.debug?
|
---|
1006 | token.concat ch
|
---|
1007 | end
|
---|
1008 | ungetc
|
---|
1009 |
|
---|
1010 | if ch == "!" or ch == "?"
|
---|
1011 | token.concat getc
|
---|
1012 | end
|
---|
1013 | # fix token
|
---|
1014 |
|
---|
1015 | # $stderr.puts "identifier - #{token}, state = #@lex_state"
|
---|
1016 |
|
---|
1017 | case token
|
---|
1018 | when /^\$/
|
---|
1019 | return Token(TkGVAR, token).set_text(token)
|
---|
1020 | when /^\@/
|
---|
1021 | @lex_state = EXPR_END
|
---|
1022 | return Token(TkIVAR, token).set_text(token)
|
---|
1023 | end
|
---|
1024 |
|
---|
1025 | if @lex_state != EXPR_DOT
|
---|
1026 | print token, "\n" if RubyLex.debug?
|
---|
1027 |
|
---|
1028 | token_c, *trans = TkReading2Token[token]
|
---|
1029 | if token_c
|
---|
1030 | # reserved word?
|
---|
1031 |
|
---|
1032 | if (@lex_state != EXPR_BEG &&
|
---|
1033 | @lex_state != EXPR_FNAME &&
|
---|
1034 | trans[1])
|
---|
1035 | # modifiers
|
---|
1036 | token_c = TkSymbol2Token[trans[1]]
|
---|
1037 | @lex_state = trans[0]
|
---|
1038 | else
|
---|
1039 | if @lex_state != EXPR_FNAME
|
---|
1040 | if ENINDENT_CLAUSE.include?(token)
|
---|
1041 | @indent += 1
|
---|
1042 | elsif DEINDENT_CLAUSE.include?(token)
|
---|
1043 | @indent -= 1
|
---|
1044 | end
|
---|
1045 | @lex_state = trans[0]
|
---|
1046 | else
|
---|
1047 | @lex_state = EXPR_END
|
---|
1048 | end
|
---|
1049 | end
|
---|
1050 | return Token(token_c, token).set_text(token)
|
---|
1051 | end
|
---|
1052 | end
|
---|
1053 |
|
---|
1054 | if @lex_state == EXPR_FNAME
|
---|
1055 | @lex_state = EXPR_END
|
---|
1056 | if peek(0) == '='
|
---|
1057 | token.concat getc
|
---|
1058 | end
|
---|
1059 | elsif @lex_state == EXPR_BEG || @lex_state == EXPR_DOT
|
---|
1060 | @lex_state = EXPR_ARG
|
---|
1061 | else
|
---|
1062 | @lex_state = EXPR_END
|
---|
1063 | end
|
---|
1064 |
|
---|
1065 | if token[0, 1] =~ /[A-Z]/
|
---|
1066 | return Token(TkCONSTANT, token).set_text(token)
|
---|
1067 | elsif token[token.size - 1, 1] =~ /[!?]/
|
---|
1068 | return Token(TkFID, token).set_text(token)
|
---|
1069 | else
|
---|
1070 | return Token(TkIDENTIFIER, token).set_text(token)
|
---|
1071 | end
|
---|
1072 | end
|
---|
1073 |
|
---|
1074 | def identify_here_document
|
---|
1075 | ch = getc
|
---|
1076 | if ch == "-"
|
---|
1077 | ch = getc
|
---|
1078 | indent = true
|
---|
1079 | end
|
---|
1080 | if /['"`]/ =~ ch # '
|
---|
1081 | lt = ch
|
---|
1082 | quoted = ""
|
---|
1083 | while (c = getc) && c != lt
|
---|
1084 | quoted.concat c
|
---|
1085 | end
|
---|
1086 | else
|
---|
1087 | lt = '"'
|
---|
1088 | quoted = ch.dup
|
---|
1089 | while (c = getc) && c =~ /\w/
|
---|
1090 | quoted.concat c
|
---|
1091 | end
|
---|
1092 | ungetc
|
---|
1093 | end
|
---|
1094 |
|
---|
1095 | ltback, @ltype = @ltype, lt
|
---|
1096 | reserve = ""
|
---|
1097 |
|
---|
1098 | while ch = getc
|
---|
1099 | reserve << ch
|
---|
1100 | if ch == "\\" #"
|
---|
1101 | ch = getc
|
---|
1102 | reserve << ch
|
---|
1103 | elsif ch == "\n"
|
---|
1104 | break
|
---|
1105 | end
|
---|
1106 | end
|
---|
1107 |
|
---|
1108 | str = ""
|
---|
1109 | while (l = gets)
|
---|
1110 | l.chomp!
|
---|
1111 | l.strip! if indent
|
---|
1112 | break if l == quoted
|
---|
1113 | str << l.chomp << "\n"
|
---|
1114 | end
|
---|
1115 |
|
---|
1116 | @reader.divert_read_from(reserve)
|
---|
1117 |
|
---|
1118 | @ltype = ltback
|
---|
1119 | @lex_state = EXPR_END
|
---|
1120 | Token(Ltype2Token[lt], str).set_text(str.dump)
|
---|
1121 | end
|
---|
1122 |
|
---|
1123 | def identify_quotation(initial_char)
|
---|
1124 | ch = getc
|
---|
1125 | if lt = PERCENT_LTYPE[ch]
|
---|
1126 | initial_char += ch
|
---|
1127 | ch = getc
|
---|
1128 | elsif ch =~ /\W/
|
---|
1129 | lt = "\""
|
---|
1130 | else
|
---|
1131 | RubyLex.fail SyntaxError, "unknown type of %string ('#{ch}')"
|
---|
1132 | end
|
---|
1133 | # if ch !~ /\W/
|
---|
1134 | # ungetc
|
---|
1135 | # next
|
---|
1136 | # end
|
---|
1137 | #@ltype = lt
|
---|
1138 | @quoted = ch unless @quoted = PERCENT_PAREN[ch]
|
---|
1139 | identify_string(lt, @quoted, ch, initial_char)
|
---|
1140 | end
|
---|
1141 |
|
---|
1142 | def identify_number(start)
|
---|
1143 | str = start.dup
|
---|
1144 |
|
---|
1145 | if start == "+" or start == "-" or start == ""
|
---|
1146 | start = getc
|
---|
1147 | str << start
|
---|
1148 | end
|
---|
1149 |
|
---|
1150 | @lex_state = EXPR_END
|
---|
1151 |
|
---|
1152 | if start == "0"
|
---|
1153 | if peek(0) == "x"
|
---|
1154 | ch = getc
|
---|
1155 | str << ch
|
---|
1156 | match = /[0-9a-f_]/
|
---|
1157 | else
|
---|
1158 | match = /[0-7_]/
|
---|
1159 | end
|
---|
1160 | while ch = getc
|
---|
1161 | if ch !~ match
|
---|
1162 | ungetc
|
---|
1163 | break
|
---|
1164 | else
|
---|
1165 | str << ch
|
---|
1166 | end
|
---|
1167 | end
|
---|
1168 | return Token(TkINTEGER).set_text(str)
|
---|
1169 | end
|
---|
1170 |
|
---|
1171 | type = TkINTEGER
|
---|
1172 | allow_point = TRUE
|
---|
1173 | allow_e = TRUE
|
---|
1174 | while ch = getc
|
---|
1175 | case ch
|
---|
1176 | when /[0-9_]/
|
---|
1177 | str << ch
|
---|
1178 |
|
---|
1179 | when allow_point && "."
|
---|
1180 | type = TkFLOAT
|
---|
1181 | if peek(0) !~ /[0-9]/
|
---|
1182 | ungetc
|
---|
1183 | break
|
---|
1184 | end
|
---|
1185 | str << ch
|
---|
1186 | allow_point = false
|
---|
1187 |
|
---|
1188 | when allow_e && "e", allow_e && "E"
|
---|
1189 | str << ch
|
---|
1190 | type = TkFLOAT
|
---|
1191 | if peek(0) =~ /[+-]/
|
---|
1192 | str << getc
|
---|
1193 | end
|
---|
1194 | allow_e = false
|
---|
1195 | allow_point = false
|
---|
1196 | else
|
---|
1197 | ungetc
|
---|
1198 | break
|
---|
1199 | end
|
---|
1200 | end
|
---|
1201 | Token(type).set_text(str)
|
---|
1202 | end
|
---|
1203 |
|
---|
1204 | def identify_string(ltype, quoted = ltype, opener=nil, initial_char = nil)
|
---|
1205 | @ltype = ltype
|
---|
1206 | @quoted = quoted
|
---|
1207 | subtype = nil
|
---|
1208 |
|
---|
1209 | str = ""
|
---|
1210 | str << initial_char if initial_char
|
---|
1211 | str << (opener||quoted)
|
---|
1212 |
|
---|
1213 | nest = 0
|
---|
1214 | begin
|
---|
1215 | while ch = getc
|
---|
1216 | str << ch
|
---|
1217 | if @quoted == ch
|
---|
1218 | if nest == 0
|
---|
1219 | break
|
---|
1220 | else
|
---|
1221 | nest -= 1
|
---|
1222 | end
|
---|
1223 | elsif opener == ch
|
---|
1224 | nest += 1
|
---|
1225 | elsif @ltype != "'" && @ltype != "]" and ch == "#"
|
---|
1226 | ch = getc
|
---|
1227 | if ch == "{"
|
---|
1228 | subtype = true
|
---|
1229 | str << ch << skip_inner_expression
|
---|
1230 | else
|
---|
1231 | ungetc(ch)
|
---|
1232 | end
|
---|
1233 | elsif ch == '\\' #'
|
---|
1234 | str << read_escape
|
---|
1235 | end
|
---|
1236 | end
|
---|
1237 | if @ltype == "/"
|
---|
1238 | if peek(0) =~ /i|o|n|e|s/
|
---|
1239 | str << getc
|
---|
1240 | end
|
---|
1241 | end
|
---|
1242 | if subtype
|
---|
1243 | Token(DLtype2Token[ltype], str)
|
---|
1244 | else
|
---|
1245 | Token(Ltype2Token[ltype], str)
|
---|
1246 | end.set_text(str)
|
---|
1247 | ensure
|
---|
1248 | @ltype = nil
|
---|
1249 | @quoted = nil
|
---|
1250 | @lex_state = EXPR_END
|
---|
1251 | end
|
---|
1252 | end
|
---|
1253 |
|
---|
1254 | def skip_inner_expression
|
---|
1255 | res = ""
|
---|
1256 | nest = 0
|
---|
1257 | while (ch = getc)
|
---|
1258 | res << ch
|
---|
1259 | if ch == '}'
|
---|
1260 | break if nest.zero?
|
---|
1261 | nest -= 1
|
---|
1262 | elsif ch == '{'
|
---|
1263 | nest += 1
|
---|
1264 | end
|
---|
1265 | end
|
---|
1266 | res
|
---|
1267 | end
|
---|
1268 |
|
---|
1269 | def identify_comment
|
---|
1270 | @ltype = "#"
|
---|
1271 | comment = "#"
|
---|
1272 | while ch = getc
|
---|
1273 | if ch == "\\"
|
---|
1274 | ch = getc
|
---|
1275 | if ch == "\n"
|
---|
1276 | ch = " "
|
---|
1277 | else
|
---|
1278 | comment << "\\"
|
---|
1279 | end
|
---|
1280 | else
|
---|
1281 | if ch == "\n"
|
---|
1282 | @ltype = nil
|
---|
1283 | ungetc
|
---|
1284 | break
|
---|
1285 | end
|
---|
1286 | end
|
---|
1287 | comment << ch
|
---|
1288 | end
|
---|
1289 | return Token(TkCOMMENT).set_text(comment)
|
---|
1290 | end
|
---|
1291 |
|
---|
1292 | def read_escape
|
---|
1293 | res = ""
|
---|
1294 | case ch = getc
|
---|
1295 | when /[0-7]/
|
---|
1296 | ungetc ch
|
---|
1297 | 3.times do
|
---|
1298 | case ch = getc
|
---|
1299 | when /[0-7]/
|
---|
1300 | when nil
|
---|
1301 | break
|
---|
1302 | else
|
---|
1303 | ungetc
|
---|
1304 | break
|
---|
1305 | end
|
---|
1306 | res << ch
|
---|
1307 | end
|
---|
1308 |
|
---|
1309 | when "x"
|
---|
1310 | res << ch
|
---|
1311 | 2.times do
|
---|
1312 | case ch = getc
|
---|
1313 | when /[0-9a-fA-F]/
|
---|
1314 | when nil
|
---|
1315 | break
|
---|
1316 | else
|
---|
1317 | ungetc
|
---|
1318 | break
|
---|
1319 | end
|
---|
1320 | res << ch
|
---|
1321 | end
|
---|
1322 |
|
---|
1323 | when "M"
|
---|
1324 | res << ch
|
---|
1325 | if (ch = getc) != '-'
|
---|
1326 | ungetc
|
---|
1327 | else
|
---|
1328 | res << ch
|
---|
1329 | if (ch = getc) == "\\" #"
|
---|
1330 | res << ch
|
---|
1331 | res << read_escape
|
---|
1332 | else
|
---|
1333 | res << ch
|
---|
1334 | end
|
---|
1335 | end
|
---|
1336 |
|
---|
1337 | when "C", "c" #, "^"
|
---|
1338 | res << ch
|
---|
1339 | if ch == "C" and (ch = getc) != "-"
|
---|
1340 | ungetc
|
---|
1341 | else
|
---|
1342 | res << ch
|
---|
1343 | if (ch = getc) == "\\" #"
|
---|
1344 | res << ch
|
---|
1345 | res << read_escape
|
---|
1346 | else
|
---|
1347 | res << ch
|
---|
1348 | end
|
---|
1349 | end
|
---|
1350 | else
|
---|
1351 | res << ch
|
---|
1352 | end
|
---|
1353 | res
|
---|
1354 | end
|
---|
1355 | end
|
---|
1356 |
|
---|
1357 |
|
---|
1358 |
|
---|
1359 | # Extract code elements from a source file, returning a TopLevel
|
---|
1360 | # object containing the constituent file elements.
|
---|
1361 | #
|
---|
1362 | # This file is based on rtags
|
---|
1363 |
|
---|
1364 | module RDoc
|
---|
1365 |
|
---|
1366 | GENERAL_MODIFIERS = [ 'nodoc' ].freeze
|
---|
1367 |
|
---|
1368 | CLASS_MODIFIERS = GENERAL_MODIFIERS
|
---|
1369 |
|
---|
1370 | ATTR_MODIFIERS = GENERAL_MODIFIERS
|
---|
1371 |
|
---|
1372 | CONSTANT_MODIFIERS = GENERAL_MODIFIERS
|
---|
1373 |
|
---|
1374 | METHOD_MODIFIERS = GENERAL_MODIFIERS +
|
---|
1375 | [ 'arg', 'args', 'yield', 'yields', 'notnew', 'not-new', 'not_new', 'doc' ]
|
---|
1376 |
|
---|
1377 |
|
---|
1378 | class RubyParser
|
---|
1379 | include RubyToken
|
---|
1380 | include TokenStream
|
---|
1381 |
|
---|
1382 | extend ParserFactory
|
---|
1383 |
|
---|
1384 | parse_files_matching(/\.rbw?$/)
|
---|
1385 |
|
---|
1386 |
|
---|
1387 | def initialize(top_level, file_name, content, options, stats)
|
---|
1388 | @options = options
|
---|
1389 | @stats = stats
|
---|
1390 | @size = 0
|
---|
1391 | @token_listeners = nil
|
---|
1392 | @input_file_name = file_name
|
---|
1393 | @scanner = RubyLex.new(content)
|
---|
1394 | @scanner.exception_on_syntax_error = false
|
---|
1395 | @top_level = top_level
|
---|
1396 | @progress = $stderr unless options.quiet
|
---|
1397 | end
|
---|
1398 |
|
---|
1399 | def scan
|
---|
1400 | @tokens = []
|
---|
1401 | @unget_read = []
|
---|
1402 | @read = []
|
---|
1403 | catch(:eof) do
|
---|
1404 | catch(:enddoc) do
|
---|
1405 | begin
|
---|
1406 | parse_toplevel_statements(@top_level)
|
---|
1407 | rescue Exception => e
|
---|
1408 | $stderr.puts "\n\n"
|
---|
1409 | $stderr.puts "RDoc failure in #@input_file_name at or around " +
|
---|
1410 | "line #{@scanner.line_no} column #{@scanner.char_no}"
|
---|
1411 | $stderr.puts
|
---|
1412 | $stderr.puts "Before reporting this, could you check that the file"
|
---|
1413 | $stderr.puts "you're documenting compiles cleanly--RDoc is not a"
|
---|
1414 | $stderr.puts "full Ruby parser, and gets confused easily if fed"
|
---|
1415 | $stderr.puts "invalid programs."
|
---|
1416 | $stderr.puts
|
---|
1417 | $stderr.puts "The internal error was:\n\n"
|
---|
1418 |
|
---|
1419 | e.set_backtrace(e.backtrace[0,4])
|
---|
1420 | raise
|
---|
1421 | end
|
---|
1422 | end
|
---|
1423 | end
|
---|
1424 | @top_level
|
---|
1425 | end
|
---|
1426 |
|
---|
1427 | private
|
---|
1428 |
|
---|
1429 | def make_message(msg)
|
---|
1430 | prefix = "\n" + @input_file_name + ":"
|
---|
1431 | if @scanner
|
---|
1432 | prefix << "#{@scanner.line_no}:#{@scanner.char_no}: "
|
---|
1433 | end
|
---|
1434 | return prefix + msg
|
---|
1435 | end
|
---|
1436 |
|
---|
1437 | def warn(msg)
|
---|
1438 | return if @options.quiet
|
---|
1439 | msg = make_message msg
|
---|
1440 | $stderr.puts msg
|
---|
1441 | end
|
---|
1442 |
|
---|
1443 | def error(msg)
|
---|
1444 | msg = make_message msg
|
---|
1445 | $stderr.puts msg
|
---|
1446 | exit(1)
|
---|
1447 | end
|
---|
1448 |
|
---|
1449 | def progress(char)
|
---|
1450 | unless @options.quiet
|
---|
1451 | @progress.print(char)
|
---|
1452 | @progress.flush
|
---|
1453 | end
|
---|
1454 | end
|
---|
1455 |
|
---|
1456 | def add_token_listener(obj)
|
---|
1457 | @token_listeners ||= []
|
---|
1458 | @token_listeners << obj
|
---|
1459 | end
|
---|
1460 |
|
---|
1461 | def remove_token_listener(obj)
|
---|
1462 | @token_listeners.delete(obj)
|
---|
1463 | end
|
---|
1464 |
|
---|
1465 | def get_tk
|
---|
1466 | tk = nil
|
---|
1467 | if @tokens.empty?
|
---|
1468 | tk = @scanner.token
|
---|
1469 | @read.push @scanner.get_read
|
---|
1470 | puts "get_tk1 => #{tk.inspect}" if $TOKEN_DEBUG
|
---|
1471 | else
|
---|
1472 | @read.push @unget_read.shift
|
---|
1473 | tk = @tokens.shift
|
---|
1474 | puts "get_tk2 => #{tk.inspect}" if $TOKEN_DEBUG
|
---|
1475 | end
|
---|
1476 |
|
---|
1477 | if tk.kind_of?(TkSYMBEG)
|
---|
1478 | set_token_position(tk.line_no, tk.char_no)
|
---|
1479 | tk1 = get_tk
|
---|
1480 | if tk1.kind_of?(TkId) || tk1.kind_of?(TkOp)
|
---|
1481 | tk = Token(TkSYMBOL).set_text(":" + tk1.name)
|
---|
1482 | # remove the identifier we just read (we're about to
|
---|
1483 | # replace it with a symbol)
|
---|
1484 | @token_listeners.each do |obj|
|
---|
1485 | obj.pop_token
|
---|
1486 | end if @token_listeners
|
---|
1487 | else
|
---|
1488 | warn("':' not followed by identifier or operator")
|
---|
1489 | tk = tk1
|
---|
1490 | end
|
---|
1491 | end
|
---|
1492 |
|
---|
1493 | # inform any listeners of our shiny new token
|
---|
1494 | @token_listeners.each do |obj|
|
---|
1495 | obj.add_token(tk)
|
---|
1496 | end if @token_listeners
|
---|
1497 |
|
---|
1498 | tk
|
---|
1499 | end
|
---|
1500 |
|
---|
1501 | def peek_tk
|
---|
1502 | unget_tk(tk = get_tk)
|
---|
1503 | tk
|
---|
1504 | end
|
---|
1505 |
|
---|
1506 | def unget_tk(tk)
|
---|
1507 | @tokens.unshift tk
|
---|
1508 | @unget_read.unshift @read.pop
|
---|
1509 |
|
---|
1510 | # Remove this token from any listeners
|
---|
1511 | @token_listeners.each do |obj|
|
---|
1512 | obj.pop_token
|
---|
1513 | end if @token_listeners
|
---|
1514 | end
|
---|
1515 |
|
---|
1516 | def skip_tkspace(skip_nl = true)
|
---|
1517 | tokens = []
|
---|
1518 | while ((tk = get_tk).kind_of?(TkSPACE) ||
|
---|
1519 | (skip_nl && tk.kind_of?(TkNL)))
|
---|
1520 | tokens.push tk
|
---|
1521 | end
|
---|
1522 | unget_tk(tk)
|
---|
1523 | tokens
|
---|
1524 | end
|
---|
1525 |
|
---|
1526 | def get_tkread
|
---|
1527 | read = @read.join("")
|
---|
1528 | @read = []
|
---|
1529 | read
|
---|
1530 | end
|
---|
1531 |
|
---|
1532 | def peek_read
|
---|
1533 | @read.join('')
|
---|
1534 | end
|
---|
1535 |
|
---|
1536 | NORMAL = "::"
|
---|
1537 | SINGLE = "<<"
|
---|
1538 |
|
---|
1539 | # Look for the first comment in a file that isn't
|
---|
1540 | # a shebang line.
|
---|
1541 |
|
---|
1542 | def collect_first_comment
|
---|
1543 | skip_tkspace
|
---|
1544 | res = ''
|
---|
1545 | first_line = true
|
---|
1546 |
|
---|
1547 | tk = get_tk
|
---|
1548 | while tk.kind_of?(TkCOMMENT)
|
---|
1549 | if first_line && tk.text[0,2] == "#!"
|
---|
1550 | skip_tkspace
|
---|
1551 | tk = get_tk
|
---|
1552 | else
|
---|
1553 | res << tk.text << "\n"
|
---|
1554 | tk = get_tk
|
---|
1555 | if tk.kind_of? TkNL
|
---|
1556 | skip_tkspace(false)
|
---|
1557 | tk = get_tk
|
---|
1558 | end
|
---|
1559 | end
|
---|
1560 | first_line = false
|
---|
1561 | end
|
---|
1562 | unget_tk(tk)
|
---|
1563 | res
|
---|
1564 | end
|
---|
1565 |
|
---|
1566 | def parse_toplevel_statements(container)
|
---|
1567 | comment = collect_first_comment
|
---|
1568 | look_for_directives_in(container, comment)
|
---|
1569 | container.comment = comment unless comment.empty?
|
---|
1570 | parse_statements(container, NORMAL, nil, comment)
|
---|
1571 | end
|
---|
1572 |
|
---|
1573 | def parse_statements(container, single=NORMAL, current_method=nil, comment='')
|
---|
1574 | nest = 1
|
---|
1575 | save_visibility = container.visibility
|
---|
1576 |
|
---|
1577 | # if container.kind_of?(TopLevel)
|
---|
1578 | # else
|
---|
1579 | # comment = ''
|
---|
1580 | # end
|
---|
1581 |
|
---|
1582 | non_comment_seen = true
|
---|
1583 |
|
---|
1584 | while tk = get_tk
|
---|
1585 |
|
---|
1586 | keep_comment = false
|
---|
1587 |
|
---|
1588 | non_comment_seen = true unless tk.kind_of?(TkCOMMENT)
|
---|
1589 |
|
---|
1590 | case tk
|
---|
1591 |
|
---|
1592 | when TkNL
|
---|
1593 | skip_tkspace(true) # Skip blanks and newlines
|
---|
1594 | tk = get_tk
|
---|
1595 | if tk.kind_of?(TkCOMMENT)
|
---|
1596 | if non_comment_seen
|
---|
1597 | comment = ''
|
---|
1598 | non_comment_seen = false
|
---|
1599 | end
|
---|
1600 | while tk.kind_of?(TkCOMMENT)
|
---|
1601 | comment << tk.text << "\n"
|
---|
1602 | tk = get_tk # this is the newline
|
---|
1603 | skip_tkspace(false) # leading spaces
|
---|
1604 | tk = get_tk
|
---|
1605 | end
|
---|
1606 | unless comment.empty?
|
---|
1607 | look_for_directives_in(container, comment)
|
---|
1608 | if container.done_documenting
|
---|
1609 | container.ongoing_visibility = save_visibility
|
---|
1610 | # return
|
---|
1611 | end
|
---|
1612 | end
|
---|
1613 | keep_comment = true
|
---|
1614 | else
|
---|
1615 | non_comment_seen = true
|
---|
1616 | end
|
---|
1617 | unget_tk(tk)
|
---|
1618 | keep_comment = true
|
---|
1619 |
|
---|
1620 |
|
---|
1621 | when TkCLASS
|
---|
1622 | if container.document_children
|
---|
1623 | parse_class(container, single, tk, comment)
|
---|
1624 | else
|
---|
1625 | nest += 1
|
---|
1626 | end
|
---|
1627 |
|
---|
1628 | when TkMODULE
|
---|
1629 | if container.document_children
|
---|
1630 | parse_module(container, single, tk, comment)
|
---|
1631 | else
|
---|
1632 | nest += 1
|
---|
1633 | end
|
---|
1634 |
|
---|
1635 | when TkDEF
|
---|
1636 | if container.document_self
|
---|
1637 | parse_method(container, single, tk, comment)
|
---|
1638 | else
|
---|
1639 | nest += 1
|
---|
1640 | end
|
---|
1641 |
|
---|
1642 | when TkCONSTANT
|
---|
1643 | if container.document_self
|
---|
1644 | parse_constant(container, single, tk, comment)
|
---|
1645 | end
|
---|
1646 |
|
---|
1647 | when TkALIAS
|
---|
1648 | if container.document_self
|
---|
1649 | parse_alias(container, single, tk, comment)
|
---|
1650 | end
|
---|
1651 |
|
---|
1652 | when TkYIELD
|
---|
1653 | if current_method.nil?
|
---|
1654 | warn("Warning: yield outside of method") if container.document_self
|
---|
1655 | else
|
---|
1656 | parse_yield(container, single, tk, current_method)
|
---|
1657 | end
|
---|
1658 |
|
---|
1659 | # Until and While can have a 'do', which shouldn't increas
|
---|
1660 | # the nesting. We can't solve the general case, but we can
|
---|
1661 | # handle most occurrences by ignoring a do at the end of a line
|
---|
1662 |
|
---|
1663 | when TkUNTIL, TkWHILE
|
---|
1664 | nest += 1
|
---|
1665 | puts "FOUND #{tk.class} in #{container.name}, nest = #{nest}, " +
|
---|
1666 | "line #{tk.line_no}" if $DEBUG
|
---|
1667 | skip_optional_do_after_expression
|
---|
1668 |
|
---|
1669 | # 'for' is trickier
|
---|
1670 | when TkFOR
|
---|
1671 | nest += 1
|
---|
1672 | puts "FOUND #{tk.class} in #{container.name}, nest = #{nest}, " +
|
---|
1673 | "line #{tk.line_no}" if $DEBUG
|
---|
1674 | skip_for_variable
|
---|
1675 | skip_optional_do_after_expression
|
---|
1676 |
|
---|
1677 | when TkCASE, TkDO, TkIF, TkUNLESS, TkBEGIN
|
---|
1678 | nest += 1
|
---|
1679 | puts "Found #{tk.class} in #{container.name}, nest = #{nest}, " +
|
---|
1680 | "line #{tk.line_no}" if $DEBUG
|
---|
1681 |
|
---|
1682 | when TkIDENTIFIER
|
---|
1683 | if nest == 1 and current_method.nil?
|
---|
1684 | case tk.name
|
---|
1685 | when "private", "protected", "public",
|
---|
1686 | "private_class_method", "public_class_method"
|
---|
1687 | parse_visibility(container, single, tk)
|
---|
1688 | keep_comment = true
|
---|
1689 | when "attr"
|
---|
1690 | parse_attr(container, single, tk, comment)
|
---|
1691 | when /^attr_(reader|writer|accessor)$/, @options.extra_accessors
|
---|
1692 | parse_attr_accessor(container, single, tk, comment)
|
---|
1693 | when "alias_method"
|
---|
1694 | if container.document_self
|
---|
1695 | parse_alias(container, single, tk, comment)
|
---|
1696 | end
|
---|
1697 | end
|
---|
1698 | end
|
---|
1699 |
|
---|
1700 | case tk.name
|
---|
1701 | when "require"
|
---|
1702 | parse_require(container, comment)
|
---|
1703 | when "include"
|
---|
1704 | parse_include(container, comment)
|
---|
1705 | end
|
---|
1706 |
|
---|
1707 |
|
---|
1708 | when TkEND
|
---|
1709 | nest -= 1
|
---|
1710 | puts "Found 'end' in #{container.name}, nest = #{nest}, line #{tk.line_no}" if $DEBUG
|
---|
1711 | puts "Method = #{current_method.name}" if $DEBUG and current_method
|
---|
1712 | if nest == 0
|
---|
1713 | read_documentation_modifiers(container, CLASS_MODIFIERS)
|
---|
1714 | container.ongoing_visibility = save_visibility
|
---|
1715 | return
|
---|
1716 | end
|
---|
1717 |
|
---|
1718 | end
|
---|
1719 |
|
---|
1720 | comment = '' unless keep_comment
|
---|
1721 | begin
|
---|
1722 | get_tkread
|
---|
1723 | skip_tkspace(false)
|
---|
1724 | end while peek_tk == TkNL
|
---|
1725 |
|
---|
1726 | end
|
---|
1727 | end
|
---|
1728 |
|
---|
1729 | def parse_class(container, single, tk, comment, &block)
|
---|
1730 | progress("c")
|
---|
1731 |
|
---|
1732 | @stats.num_classes += 1
|
---|
1733 |
|
---|
1734 | container, name_t = get_class_or_module(container)
|
---|
1735 |
|
---|
1736 | case name_t
|
---|
1737 | when TkCONSTANT
|
---|
1738 | name = name_t.name
|
---|
1739 | superclass = "Object"
|
---|
1740 |
|
---|
1741 | if peek_tk.kind_of?(TkLT)
|
---|
1742 | get_tk
|
---|
1743 | skip_tkspace(true)
|
---|
1744 | superclass = get_class_specification
|
---|
1745 | superclass = "<unknown>" if superclass.empty?
|
---|
1746 | end
|
---|
1747 |
|
---|
1748 | if single == SINGLE
|
---|
1749 | cls_type = SingleClass
|
---|
1750 | else
|
---|
1751 | cls_type = NormalClass
|
---|
1752 | end
|
---|
1753 |
|
---|
1754 | cls = container.add_class(cls_type, name, superclass)
|
---|
1755 | read_documentation_modifiers(cls, CLASS_MODIFIERS)
|
---|
1756 | cls.record_location(@top_level)
|
---|
1757 | parse_statements(cls)
|
---|
1758 | cls.comment = comment
|
---|
1759 |
|
---|
1760 | when TkLSHFT
|
---|
1761 | case name = get_class_specification
|
---|
1762 | when "self", container.name
|
---|
1763 | parse_statements(container, SINGLE, &block)
|
---|
1764 | else
|
---|
1765 | other = TopLevel.find_class_named(name)
|
---|
1766 | unless other
|
---|
1767 | # other = @top_level.add_class(NormalClass, name, nil)
|
---|
1768 | # other.record_location(@top_level)
|
---|
1769 | # other.comment = comment
|
---|
1770 | other = NormalClass.new("Dummy", nil)
|
---|
1771 | end
|
---|
1772 | read_documentation_modifiers(other, CLASS_MODIFIERS)
|
---|
1773 | parse_statements(other, SINGLE, &block)
|
---|
1774 | end
|
---|
1775 |
|
---|
1776 | else
|
---|
1777 | warn("Expected class name or '<<'. Got #{name_t.class}: #{name_t.text.inspect}")
|
---|
1778 | end
|
---|
1779 | end
|
---|
1780 |
|
---|
1781 | def parse_module(container, single, tk, comment)
|
---|
1782 | progress("m")
|
---|
1783 | @stats.num_modules += 1
|
---|
1784 | container, name_t = get_class_or_module(container)
|
---|
1785 | # skip_tkspace
|
---|
1786 | name = name_t.name
|
---|
1787 | mod = container.add_module(NormalModule, name)
|
---|
1788 | mod.record_location(@top_level)
|
---|
1789 | read_documentation_modifiers(mod, CLASS_MODIFIERS)
|
---|
1790 | parse_statements(mod)
|
---|
1791 | mod.comment = comment
|
---|
1792 | end
|
---|
1793 |
|
---|
1794 | # Look for the name of a class of module (optionally with a leading :: or
|
---|
1795 | # with :: separated named) and return the ultimate name and container
|
---|
1796 |
|
---|
1797 | def get_class_or_module(container)
|
---|
1798 | skip_tkspace
|
---|
1799 | name_t = get_tk
|
---|
1800 |
|
---|
1801 | # class ::A -> A is in the top level
|
---|
1802 | if name_t.kind_of?(TkCOLON2)
|
---|
1803 | name_t = get_tk
|
---|
1804 | container = @top_level
|
---|
1805 | end
|
---|
1806 |
|
---|
1807 | skip_tkspace(false)
|
---|
1808 |
|
---|
1809 | while peek_tk.kind_of?(TkCOLON2)
|
---|
1810 | prev_container = container
|
---|
1811 | container = container.find_module_named(name_t.name)
|
---|
1812 | if !container
|
---|
1813 | # warn("Couldn't find module #{name_t.name}")
|
---|
1814 | container = prev_container.add_module(NormalModule, name_t.name)
|
---|
1815 | end
|
---|
1816 | get_tk
|
---|
1817 | name_t = get_tk
|
---|
1818 | end
|
---|
1819 | skip_tkspace(false)
|
---|
1820 | return [container, name_t]
|
---|
1821 | end
|
---|
1822 |
|
---|
1823 | def parse_constant(container, single, tk, comment)
|
---|
1824 | name = tk.name
|
---|
1825 | skip_tkspace(false)
|
---|
1826 | eq_tk = get_tk
|
---|
1827 |
|
---|
1828 | unless eq_tk.kind_of?(TkASSIGN)
|
---|
1829 | unget_tk(eq_tk)
|
---|
1830 | return
|
---|
1831 | end
|
---|
1832 |
|
---|
1833 |
|
---|
1834 | nest = 0
|
---|
1835 | get_tkread
|
---|
1836 |
|
---|
1837 | tk = get_tk
|
---|
1838 | if tk.kind_of? TkGT
|
---|
1839 | unget_tk(tk)
|
---|
1840 | unget_tk(eq_tk)
|
---|
1841 | return
|
---|
1842 | end
|
---|
1843 |
|
---|
1844 | loop do
|
---|
1845 | puts("Param: #{tk}, #{@scanner.continue} " +
|
---|
1846 | "#{@scanner.lex_state} #{nest}") if $DEBUG
|
---|
1847 |
|
---|
1848 | case tk
|
---|
1849 | when TkSEMICOLON
|
---|
1850 | break
|
---|
1851 | when TkLPAREN, TkfLPAREN
|
---|
1852 | nest += 1
|
---|
1853 | when TkRPAREN
|
---|
1854 | nest -= 1
|
---|
1855 | when TkCOMMENT
|
---|
1856 | if nest <= 0 && @scanner.lex_state == EXPR_END
|
---|
1857 | unget_tk(tk)
|
---|
1858 | break
|
---|
1859 | end
|
---|
1860 | when TkNL
|
---|
1861 | if (@scanner.lex_state == EXPR_END and nest <= 0) || [email protected]
|
---|
1862 | unget_tk(tk)
|
---|
1863 | break
|
---|
1864 | end
|
---|
1865 | end
|
---|
1866 | tk = get_tk
|
---|
1867 | end
|
---|
1868 |
|
---|
1869 | res = get_tkread.tr("\n", " ").strip
|
---|
1870 | res = "" if res == ";"
|
---|
1871 | con = Constant.new(name, res, comment)
|
---|
1872 | read_documentation_modifiers(con, CONSTANT_MODIFIERS)
|
---|
1873 | if con.document_self
|
---|
1874 | container.add_constant(con)
|
---|
1875 | end
|
---|
1876 | end
|
---|
1877 |
|
---|
1878 | def parse_method(container, single, tk, comment)
|
---|
1879 | progress(".")
|
---|
1880 | @stats.num_methods += 1
|
---|
1881 | line_no = tk.line_no
|
---|
1882 | column = tk.char_no
|
---|
1883 |
|
---|
1884 | start_collecting_tokens
|
---|
1885 | add_token(tk)
|
---|
1886 | add_token_listener(self)
|
---|
1887 |
|
---|
1888 | @scanner.instance_eval{@lex_state = EXPR_FNAME}
|
---|
1889 | skip_tkspace(false)
|
---|
1890 | name_t = get_tk
|
---|
1891 | back_tk = skip_tkspace
|
---|
1892 | meth = nil
|
---|
1893 | added_container = false
|
---|
1894 |
|
---|
1895 | dot = get_tk
|
---|
1896 | if dot.kind_of?(TkDOT) or dot.kind_of?(TkCOLON2)
|
---|
1897 | @scanner.instance_eval{@lex_state = EXPR_FNAME}
|
---|
1898 | skip_tkspace
|
---|
1899 | name_t2 = get_tk
|
---|
1900 | case name_t
|
---|
1901 | when TkSELF
|
---|
1902 | name = name_t2.name
|
---|
1903 | when TkCONSTANT
|
---|
1904 | name = name_t2.name
|
---|
1905 | prev_container = container
|
---|
1906 | container = container.find_module_named(name_t.name)
|
---|
1907 | if !container
|
---|
1908 | added_container = true
|
---|
1909 | obj = name_t.name.split("::").inject(Object) do |state, item|
|
---|
1910 | state.const_get(item)
|
---|
1911 | end rescue nil
|
---|
1912 |
|
---|
1913 | type = obj.class == Class ? NormalClass : NormalModule
|
---|
1914 | if not [Class, Module].include?(obj.class)
|
---|
1915 | warn("Couldn't find #{name_t.name}. Assuming it's a module")
|
---|
1916 | end
|
---|
1917 |
|
---|
1918 | if type == NormalClass then
|
---|
1919 | container = prev_container.add_class(type, name_t.name, obj.superclass.name)
|
---|
1920 | else
|
---|
1921 | container = prev_container.add_module(type, name_t.name)
|
---|
1922 | end
|
---|
1923 | end
|
---|
1924 | else
|
---|
1925 | # warn("Unexpected token '#{name_t2.inspect}'")
|
---|
1926 | # break
|
---|
1927 | skip_method(container)
|
---|
1928 | return
|
---|
1929 | end
|
---|
1930 | meth = AnyMethod.new(get_tkread, name)
|
---|
1931 | meth.singleton = true
|
---|
1932 | else
|
---|
1933 | unget_tk dot
|
---|
1934 | back_tk.reverse_each do
|
---|
1935 | |tk|
|
---|
1936 | unget_tk tk
|
---|
1937 | end
|
---|
1938 | name = name_t.name
|
---|
1939 |
|
---|
1940 | meth = AnyMethod.new(get_tkread, name)
|
---|
1941 | meth.singleton = (single == SINGLE)
|
---|
1942 | end
|
---|
1943 |
|
---|
1944 | remove_token_listener(self)
|
---|
1945 |
|
---|
1946 | meth.start_collecting_tokens
|
---|
1947 | indent = TkSPACE.new(1,1)
|
---|
1948 | indent.set_text(" " * column)
|
---|
1949 |
|
---|
1950 | meth.add_tokens([TkCOMMENT.new(line_no,
|
---|
1951 | 1,
|
---|
1952 | "# File #{@top_level.file_absolute_name}, line #{line_no}"),
|
---|
1953 | NEWLINE_TOKEN,
|
---|
1954 | indent])
|
---|
1955 |
|
---|
1956 | meth.add_tokens(@token_stream)
|
---|
1957 |
|
---|
1958 | add_token_listener(meth)
|
---|
1959 |
|
---|
1960 | @scanner.instance_eval{@continue = false}
|
---|
1961 | parse_method_parameters(meth)
|
---|
1962 |
|
---|
1963 | if meth.document_self
|
---|
1964 | container.add_method(meth)
|
---|
1965 | elsif added_container
|
---|
1966 | container.document_self = false
|
---|
1967 | end
|
---|
1968 |
|
---|
1969 | # Having now read the method parameters and documentation modifiers, we
|
---|
1970 | # now know whether we have to rename #initialize to ::new
|
---|
1971 |
|
---|
1972 | if name == "initialize" && !meth.singleton
|
---|
1973 | if meth.dont_rename_initialize
|
---|
1974 | meth.visibility = :protected
|
---|
1975 | else
|
---|
1976 | meth.singleton = true
|
---|
1977 | meth.name = "new"
|
---|
1978 | meth.visibility = :public
|
---|
1979 | end
|
---|
1980 | end
|
---|
1981 |
|
---|
1982 | parse_statements(container, single, meth)
|
---|
1983 |
|
---|
1984 | remove_token_listener(meth)
|
---|
1985 |
|
---|
1986 | # Look for a 'call-seq' in the comment, and override the
|
---|
1987 | # normal parameter stuff
|
---|
1988 |
|
---|
1989 | if comment.sub!(/:?call-seq:(.*?)^\s*\#?\s*$/m, '')
|
---|
1990 | seq = $1
|
---|
1991 | seq.gsub!(/^\s*\#\s*/, '')
|
---|
1992 | meth.call_seq = seq
|
---|
1993 | end
|
---|
1994 |
|
---|
1995 | meth.comment = comment
|
---|
1996 |
|
---|
1997 | end
|
---|
1998 |
|
---|
1999 | def skip_method(container)
|
---|
2000 | meth = AnyMethod.new("", "anon")
|
---|
2001 | parse_method_parameters(meth)
|
---|
2002 | parse_statements(container, false, meth)
|
---|
2003 | end
|
---|
2004 |
|
---|
2005 | # Capture the method's parameters. Along the way,
|
---|
2006 | # look for a comment containing
|
---|
2007 | #
|
---|
2008 | # # yields: ....
|
---|
2009 | #
|
---|
2010 | # and add this as the block_params for the method
|
---|
2011 |
|
---|
2012 | def parse_method_parameters(method)
|
---|
2013 | res = parse_method_or_yield_parameters(method)
|
---|
2014 | res = "(" + res + ")" unless res[0] == ?(
|
---|
2015 | method.params = res unless method.params
|
---|
2016 | if method.block_params.nil?
|
---|
2017 | skip_tkspace(false)
|
---|
2018 | read_documentation_modifiers(method, METHOD_MODIFIERS)
|
---|
2019 | end
|
---|
2020 | end
|
---|
2021 |
|
---|
2022 | def parse_method_or_yield_parameters(method=nil, modifiers=METHOD_MODIFIERS)
|
---|
2023 | skip_tkspace(false)
|
---|
2024 | tk = get_tk
|
---|
2025 |
|
---|
2026 | # Little hack going on here. In the statement
|
---|
2027 | # f = 2*(1+yield)
|
---|
2028 | # We see the RPAREN as the next token, so we need
|
---|
2029 | # to exit early. This still won't catch all cases
|
---|
2030 | # (such as "a = yield + 1"
|
---|
2031 | end_token = case tk
|
---|
2032 | when TkLPAREN, TkfLPAREN
|
---|
2033 | TkRPAREN
|
---|
2034 | when TkRPAREN
|
---|
2035 | return ""
|
---|
2036 | else
|
---|
2037 | TkNL
|
---|
2038 | end
|
---|
2039 | nest = 0
|
---|
2040 |
|
---|
2041 | loop do
|
---|
2042 | puts("Param: #{tk.inspect}, #{@scanner.continue} " +
|
---|
2043 | "#{@scanner.lex_state} #{nest}") if $DEBUG
|
---|
2044 | case tk
|
---|
2045 | when TkSEMICOLON
|
---|
2046 | break
|
---|
2047 | when TkLBRACE
|
---|
2048 | nest += 1
|
---|
2049 | when TkRBRACE
|
---|
2050 | # we might have a.each {|i| yield i }
|
---|
2051 | unget_tk(tk) if nest.zero?
|
---|
2052 | nest -= 1
|
---|
2053 | break if nest <= 0
|
---|
2054 | when TkLPAREN, TkfLPAREN
|
---|
2055 | nest += 1
|
---|
2056 | when end_token
|
---|
2057 | if end_token == TkRPAREN
|
---|
2058 | nest -= 1
|
---|
2059 | break if @scanner.lex_state == EXPR_END and nest <= 0
|
---|
2060 | else
|
---|
2061 | break unless @scanner.continue
|
---|
2062 | end
|
---|
2063 | when method && method.block_params.nil? && TkCOMMENT
|
---|
2064 | unget_tk(tk)
|
---|
2065 | read_documentation_modifiers(method, modifiers)
|
---|
2066 | end
|
---|
2067 | tk = get_tk
|
---|
2068 | end
|
---|
2069 | res = get_tkread.tr("\n", " ").strip
|
---|
2070 | res = "" if res == ";"
|
---|
2071 | res
|
---|
2072 | end
|
---|
2073 |
|
---|
2074 | # skip the var [in] part of a 'for' statement
|
---|
2075 | def skip_for_variable
|
---|
2076 | skip_tkspace(false)
|
---|
2077 | tk = get_tk
|
---|
2078 | skip_tkspace(false)
|
---|
2079 | tk = get_tk
|
---|
2080 | unget_tk(tk) unless tk.kind_of?(TkIN)
|
---|
2081 | end
|
---|
2082 |
|
---|
2083 | # while, until, and for have an optional
|
---|
2084 | def skip_optional_do_after_expression
|
---|
2085 | skip_tkspace(false)
|
---|
2086 | tk = get_tk
|
---|
2087 | case tk
|
---|
2088 | when TkLPAREN, TkfLPAREN
|
---|
2089 | end_token = TkRPAREN
|
---|
2090 | else
|
---|
2091 | end_token = TkNL
|
---|
2092 | end
|
---|
2093 |
|
---|
2094 | nest = 0
|
---|
2095 | @scanner.instance_eval{@continue = false}
|
---|
2096 |
|
---|
2097 | loop do
|
---|
2098 | puts("\nWhile: #{tk}, #{@scanner.continue} " +
|
---|
2099 | "#{@scanner.lex_state} #{nest}") if $DEBUG
|
---|
2100 | case tk
|
---|
2101 | when TkSEMICOLON
|
---|
2102 | break
|
---|
2103 | when TkLPAREN, TkfLPAREN
|
---|
2104 | nest += 1
|
---|
2105 | when TkDO
|
---|
2106 | break if nest.zero?
|
---|
2107 | when end_token
|
---|
2108 | if end_token == TkRPAREN
|
---|
2109 | nest -= 1
|
---|
2110 | break if @scanner.lex_state == EXPR_END and nest.zero?
|
---|
2111 | else
|
---|
2112 | break unless @scanner.continue
|
---|
2113 | end
|
---|
2114 | end
|
---|
2115 | tk = get_tk
|
---|
2116 | end
|
---|
2117 | skip_tkspace(false)
|
---|
2118 | if peek_tk.kind_of? TkDO
|
---|
2119 | get_tk
|
---|
2120 | end
|
---|
2121 | end
|
---|
2122 |
|
---|
2123 | # Return a superclass, which can be either a constant
|
---|
2124 | # of an expression
|
---|
2125 |
|
---|
2126 | def get_class_specification
|
---|
2127 | tk = get_tk
|
---|
2128 | return "self" if tk.kind_of?(TkSELF)
|
---|
2129 |
|
---|
2130 | res = ""
|
---|
2131 | while tk.kind_of?(TkCOLON2) ||
|
---|
2132 | tk.kind_of?(TkCOLON3) ||
|
---|
2133 | tk.kind_of?(TkCONSTANT)
|
---|
2134 |
|
---|
2135 | res += tk.text
|
---|
2136 | tk = get_tk
|
---|
2137 | end
|
---|
2138 |
|
---|
2139 | unget_tk(tk)
|
---|
2140 | skip_tkspace(false)
|
---|
2141 |
|
---|
2142 | get_tkread # empty out read buffer
|
---|
2143 |
|
---|
2144 | tk = get_tk
|
---|
2145 |
|
---|
2146 | case tk
|
---|
2147 | when TkNL, TkCOMMENT, TkSEMICOLON
|
---|
2148 | unget_tk(tk)
|
---|
2149 | return res
|
---|
2150 | end
|
---|
2151 |
|
---|
2152 | res += parse_call_parameters(tk)
|
---|
2153 | res
|
---|
2154 | end
|
---|
2155 |
|
---|
2156 | def parse_call_parameters(tk)
|
---|
2157 |
|
---|
2158 | end_token = case tk
|
---|
2159 | when TkLPAREN, TkfLPAREN
|
---|
2160 | TkRPAREN
|
---|
2161 | when TkRPAREN
|
---|
2162 | return ""
|
---|
2163 | else
|
---|
2164 | TkNL
|
---|
2165 | end
|
---|
2166 | nest = 0
|
---|
2167 |
|
---|
2168 | loop do
|
---|
2169 | puts("Call param: #{tk}, #{@scanner.continue} " +
|
---|
2170 | "#{@scanner.lex_state} #{nest}") if $DEBUG
|
---|
2171 | case tk
|
---|
2172 | when TkSEMICOLON
|
---|
2173 | break
|
---|
2174 | when TkLPAREN, TkfLPAREN
|
---|
2175 | nest += 1
|
---|
2176 | when end_token
|
---|
2177 | if end_token == TkRPAREN
|
---|
2178 | nest -= 1
|
---|
2179 | break if @scanner.lex_state == EXPR_END and nest <= 0
|
---|
2180 | else
|
---|
2181 | break unless @scanner.continue
|
---|
2182 | end
|
---|
2183 | when TkCOMMENT
|
---|
2184 | unget_tk(tk)
|
---|
2185 | break
|
---|
2186 | end
|
---|
2187 | tk = get_tk
|
---|
2188 | end
|
---|
2189 | res = get_tkread.tr("\n", " ").strip
|
---|
2190 | res = "" if res == ";"
|
---|
2191 | res
|
---|
2192 | end
|
---|
2193 |
|
---|
2194 |
|
---|
2195 | # Parse a constant, which might be qualified by
|
---|
2196 | # one or more class or module names
|
---|
2197 |
|
---|
2198 | def get_constant
|
---|
2199 | res = ""
|
---|
2200 | skip_tkspace(false)
|
---|
2201 | tk = get_tk
|
---|
2202 |
|
---|
2203 | while tk.kind_of?(TkCOLON2) ||
|
---|
2204 | tk.kind_of?(TkCOLON3) ||
|
---|
2205 | tk.kind_of?(TkCONSTANT)
|
---|
2206 |
|
---|
2207 | res += tk.text
|
---|
2208 | tk = get_tk
|
---|
2209 | end
|
---|
2210 |
|
---|
2211 | # if res.empty?
|
---|
2212 | # warn("Unexpected token #{tk} in constant")
|
---|
2213 | # end
|
---|
2214 | unget_tk(tk)
|
---|
2215 | res
|
---|
2216 | end
|
---|
2217 |
|
---|
2218 | # Get a constant that may be surrounded by parens
|
---|
2219 |
|
---|
2220 | def get_constant_with_optional_parens
|
---|
2221 | skip_tkspace(false)
|
---|
2222 | nest = 0
|
---|
2223 | while (tk = peek_tk).kind_of?(TkLPAREN) || tk.kind_of?(TkfLPAREN)
|
---|
2224 | get_tk
|
---|
2225 | skip_tkspace(true)
|
---|
2226 | nest += 1
|
---|
2227 | end
|
---|
2228 |
|
---|
2229 | name = get_constant
|
---|
2230 |
|
---|
2231 | while nest > 0
|
---|
2232 | skip_tkspace(true)
|
---|
2233 | tk = get_tk
|
---|
2234 | nest -= 1 if tk.kind_of?(TkRPAREN)
|
---|
2235 | end
|
---|
2236 | name
|
---|
2237 | end
|
---|
2238 |
|
---|
2239 | # Directives are modifier comments that can appear after class, module,
|
---|
2240 | # or method names. For example
|
---|
2241 | #
|
---|
2242 | # def fred # :yields: a, b
|
---|
2243 | #
|
---|
2244 | # or
|
---|
2245 | #
|
---|
2246 | # class SM # :nodoc:
|
---|
2247 | #
|
---|
2248 | # we return the directive name and any parameters as a two element array
|
---|
2249 |
|
---|
2250 | def read_directive(allowed)
|
---|
2251 | tk = get_tk
|
---|
2252 | puts "directive: #{tk.inspect}" if $DEBUG
|
---|
2253 | result = nil
|
---|
2254 | if tk.kind_of?(TkCOMMENT)
|
---|
2255 | if tk.text =~ /\s*:?(\w+):\s*(.*)/
|
---|
2256 | directive = $1.downcase
|
---|
2257 | if allowed.include?(directive)
|
---|
2258 | result = [directive, $2]
|
---|
2259 | end
|
---|
2260 | end
|
---|
2261 | else
|
---|
2262 | unget_tk(tk)
|
---|
2263 | end
|
---|
2264 | result
|
---|
2265 | end
|
---|
2266 |
|
---|
2267 |
|
---|
2268 | def read_documentation_modifiers(context, allow)
|
---|
2269 | dir = read_directive(allow)
|
---|
2270 |
|
---|
2271 | case dir[0]
|
---|
2272 |
|
---|
2273 | when "notnew", "not_new", "not-new"
|
---|
2274 | context.dont_rename_initialize = true
|
---|
2275 |
|
---|
2276 | when "nodoc"
|
---|
2277 | context.document_self = false
|
---|
2278 | if dir[1].downcase == "all"
|
---|
2279 | context.document_children = false
|
---|
2280 | end
|
---|
2281 |
|
---|
2282 | when "doc"
|
---|
2283 | context.document_self = true
|
---|
2284 | context.force_documentation = true
|
---|
2285 |
|
---|
2286 | when "yield", "yields"
|
---|
2287 | unless context.params.nil?
|
---|
2288 | context.params.sub!(/(,|)\s*&\w+/,'') # remove parameter &proc
|
---|
2289 | end
|
---|
2290 | context.block_params = dir[1]
|
---|
2291 |
|
---|
2292 | when "arg", "args"
|
---|
2293 | context.params = dir[1]
|
---|
2294 | end if dir
|
---|
2295 | end
|
---|
2296 |
|
---|
2297 |
|
---|
2298 | # Look for directives in a normal comment block:
|
---|
2299 | #
|
---|
2300 | # #-- - don't display comment from this point forward
|
---|
2301 | #
|
---|
2302 | #
|
---|
2303 | # This routine modifies it's parameter
|
---|
2304 |
|
---|
2305 | def look_for_directives_in(context, comment)
|
---|
2306 |
|
---|
2307 | preprocess = SM::PreProcess.new(@input_file_name,
|
---|
2308 | @options.rdoc_include)
|
---|
2309 |
|
---|
2310 | preprocess.handle(comment) do |directive, param|
|
---|
2311 | case directive
|
---|
2312 | when "stopdoc"
|
---|
2313 | context.stop_doc
|
---|
2314 | ""
|
---|
2315 | when "startdoc"
|
---|
2316 | context.start_doc
|
---|
2317 | context.force_documentation = true
|
---|
2318 | ""
|
---|
2319 |
|
---|
2320 | when "enddoc"
|
---|
2321 | #context.done_documenting = true
|
---|
2322 | #""
|
---|
2323 | throw :enddoc
|
---|
2324 |
|
---|
2325 | when "main"
|
---|
2326 | options = Options.instance
|
---|
2327 | options.main_page = param
|
---|
2328 | ""
|
---|
2329 |
|
---|
2330 | when "title"
|
---|
2331 | options = Options.instance
|
---|
2332 | options.title = param
|
---|
2333 | ""
|
---|
2334 |
|
---|
2335 | when "section"
|
---|
2336 | context.set_current_section(param, comment)
|
---|
2337 | comment.replace("") # 1.8 doesn't support #clear
|
---|
2338 | break
|
---|
2339 | else
|
---|
2340 | warn "Unrecognized directive '#{directive}'"
|
---|
2341 | break
|
---|
2342 | end
|
---|
2343 | end
|
---|
2344 |
|
---|
2345 | remove_private_comments(comment)
|
---|
2346 | end
|
---|
2347 |
|
---|
2348 | def remove_private_comments(comment)
|
---|
2349 | comment.gsub!(/^#--.*?^#\+\+/m, '')
|
---|
2350 | comment.sub!(/^#--.*/m, '')
|
---|
2351 | end
|
---|
2352 |
|
---|
2353 |
|
---|
2354 |
|
---|
2355 | def get_symbol_or_name
|
---|
2356 | tk = get_tk
|
---|
2357 | case tk
|
---|
2358 | when TkSYMBOL
|
---|
2359 | tk.text.sub(/^:/, '')
|
---|
2360 | when TkId, TkOp
|
---|
2361 | tk.name
|
---|
2362 | when TkSTRING
|
---|
2363 | tk.text
|
---|
2364 | else
|
---|
2365 | raise "Name or symbol expected (got #{tk})"
|
---|
2366 | end
|
---|
2367 | end
|
---|
2368 |
|
---|
2369 | def parse_alias(context, single, tk, comment)
|
---|
2370 | skip_tkspace
|
---|
2371 | if (peek_tk.kind_of? TkLPAREN)
|
---|
2372 | get_tk
|
---|
2373 | skip_tkspace
|
---|
2374 | end
|
---|
2375 | new_name = get_symbol_or_name
|
---|
2376 | @scanner.instance_eval{@lex_state = EXPR_FNAME}
|
---|
2377 | skip_tkspace
|
---|
2378 | if (peek_tk.kind_of? TkCOMMA)
|
---|
2379 | get_tk
|
---|
2380 | skip_tkspace
|
---|
2381 | end
|
---|
2382 | old_name = get_symbol_or_name
|
---|
2383 |
|
---|
2384 | al = Alias.new(get_tkread, old_name, new_name, comment)
|
---|
2385 | read_documentation_modifiers(al, ATTR_MODIFIERS)
|
---|
2386 | if al.document_self
|
---|
2387 | context.add_alias(al)
|
---|
2388 | end
|
---|
2389 | end
|
---|
2390 |
|
---|
2391 | def parse_yield_parameters
|
---|
2392 | parse_method_or_yield_parameters
|
---|
2393 | end
|
---|
2394 |
|
---|
2395 | def parse_yield(context, single, tk, method)
|
---|
2396 | if method.block_params.nil?
|
---|
2397 | get_tkread
|
---|
2398 | @scanner.instance_eval{@continue = false}
|
---|
2399 | method.block_params = parse_yield_parameters
|
---|
2400 | end
|
---|
2401 | end
|
---|
2402 |
|
---|
2403 | def parse_require(context, comment)
|
---|
2404 | skip_tkspace_comment
|
---|
2405 | tk = get_tk
|
---|
2406 | if tk.kind_of? TkLPAREN
|
---|
2407 | skip_tkspace_comment
|
---|
2408 | tk = get_tk
|
---|
2409 | end
|
---|
2410 |
|
---|
2411 | name = nil
|
---|
2412 | case tk
|
---|
2413 | when TkSTRING
|
---|
2414 | name = tk.text
|
---|
2415 | # when TkCONSTANT, TkIDENTIFIER, TkIVAR, TkGVAR
|
---|
2416 | # name = tk.name
|
---|
2417 | when TkDSTRING
|
---|
2418 | warn "Skipping require of dynamic string: #{tk.text}"
|
---|
2419 | # else
|
---|
2420 | # warn "'require' used as variable"
|
---|
2421 | end
|
---|
2422 | if name
|
---|
2423 | context.add_require(Require.new(name, comment))
|
---|
2424 | else
|
---|
2425 | unget_tk(tk)
|
---|
2426 | end
|
---|
2427 | end
|
---|
2428 |
|
---|
2429 | def parse_include(context, comment)
|
---|
2430 | loop do
|
---|
2431 | skip_tkspace_comment
|
---|
2432 | name = get_constant_with_optional_parens
|
---|
2433 | unless name.empty?
|
---|
2434 | context.add_include(Include.new(name, comment))
|
---|
2435 | end
|
---|
2436 | return unless peek_tk.kind_of?(TkCOMMA)
|
---|
2437 | get_tk
|
---|
2438 | end
|
---|
2439 | end
|
---|
2440 |
|
---|
2441 | def get_bool
|
---|
2442 | skip_tkspace
|
---|
2443 | tk = get_tk
|
---|
2444 | case tk
|
---|
2445 | when TkTRUE
|
---|
2446 | true
|
---|
2447 | when TkFALSE, TkNIL
|
---|
2448 | false
|
---|
2449 | else
|
---|
2450 | unget_tk tk
|
---|
2451 | true
|
---|
2452 | end
|
---|
2453 | end
|
---|
2454 |
|
---|
2455 | def parse_attr(context, single, tk, comment)
|
---|
2456 | args = parse_symbol_arg(1)
|
---|
2457 | if args.size > 0
|
---|
2458 | name = args[0]
|
---|
2459 | rw = "R"
|
---|
2460 | skip_tkspace(false)
|
---|
2461 | tk = get_tk
|
---|
2462 | if tk.kind_of? TkCOMMA
|
---|
2463 | rw = "RW" if get_bool
|
---|
2464 | else
|
---|
2465 | unget_tk tk
|
---|
2466 | end
|
---|
2467 | att = Attr.new(get_tkread, name, rw, comment)
|
---|
2468 | read_documentation_modifiers(att, ATTR_MODIFIERS)
|
---|
2469 | if att.document_self
|
---|
2470 | context.add_attribute(att)
|
---|
2471 | end
|
---|
2472 | else
|
---|
2473 | warn("'attr' ignored - looks like a variable")
|
---|
2474 | end
|
---|
2475 |
|
---|
2476 | end
|
---|
2477 |
|
---|
2478 | def parse_visibility(container, single, tk)
|
---|
2479 | singleton = (single == SINGLE)
|
---|
2480 | vis = case tk.name
|
---|
2481 | when "private" then :private
|
---|
2482 | when "protected" then :protected
|
---|
2483 | when "public" then :public
|
---|
2484 | when "private_class_method"
|
---|
2485 | singleton = true
|
---|
2486 | :private
|
---|
2487 | when "public_class_method"
|
---|
2488 | singleton = true
|
---|
2489 | :public
|
---|
2490 | else raise "Invalid visibility: #{tk.name}"
|
---|
2491 | end
|
---|
2492 |
|
---|
2493 | skip_tkspace_comment(false)
|
---|
2494 | case peek_tk
|
---|
2495 | # Ryan Davis suggested the extension to ignore modifiers, because he
|
---|
2496 | # often writes
|
---|
2497 | #
|
---|
2498 | # protected unless $TESTING
|
---|
2499 | #
|
---|
2500 | when TkNL, TkUNLESS_MOD, TkIF_MOD
|
---|
2501 | # error("Missing argument") if singleton
|
---|
2502 | container.ongoing_visibility = vis
|
---|
2503 | else
|
---|
2504 | args = parse_symbol_arg
|
---|
2505 | container.set_visibility_for(args, vis, singleton)
|
---|
2506 | end
|
---|
2507 | end
|
---|
2508 |
|
---|
2509 | def parse_attr_accessor(context, single, tk, comment)
|
---|
2510 | args = parse_symbol_arg
|
---|
2511 | read = get_tkread
|
---|
2512 | rw = "?"
|
---|
2513 |
|
---|
2514 | # If nodoc is given, don't document any of them
|
---|
2515 |
|
---|
2516 | tmp = CodeObject.new
|
---|
2517 | read_documentation_modifiers(tmp, ATTR_MODIFIERS)
|
---|
2518 | return unless tmp.document_self
|
---|
2519 |
|
---|
2520 | case tk.name
|
---|
2521 | when "attr_reader" then rw = "R"
|
---|
2522 | when "attr_writer" then rw = "W"
|
---|
2523 | when "attr_accessor" then rw = "RW"
|
---|
2524 | else
|
---|
2525 | rw = @options.extra_accessor_flags[tk.name]
|
---|
2526 | end
|
---|
2527 |
|
---|
2528 | for name in args
|
---|
2529 | att = Attr.new(get_tkread, name, rw, comment)
|
---|
2530 | context.add_attribute(att)
|
---|
2531 | end
|
---|
2532 | end
|
---|
2533 |
|
---|
2534 | def skip_tkspace_comment(skip_nl = true)
|
---|
2535 | loop do
|
---|
2536 | skip_tkspace(skip_nl)
|
---|
2537 | return unless peek_tk.kind_of? TkCOMMENT
|
---|
2538 | get_tk
|
---|
2539 | end
|
---|
2540 | end
|
---|
2541 |
|
---|
2542 | def parse_symbol_arg(no = nil)
|
---|
2543 |
|
---|
2544 | args = []
|
---|
2545 | skip_tkspace_comment
|
---|
2546 | case tk = get_tk
|
---|
2547 | when TkLPAREN
|
---|
2548 | loop do
|
---|
2549 | skip_tkspace_comment
|
---|
2550 | if tk1 = parse_symbol_in_arg
|
---|
2551 | args.push tk1
|
---|
2552 | break if no and args.size >= no
|
---|
2553 | end
|
---|
2554 |
|
---|
2555 | skip_tkspace_comment
|
---|
2556 | case tk2 = get_tk
|
---|
2557 | when TkRPAREN
|
---|
2558 | break
|
---|
2559 | when TkCOMMA
|
---|
2560 | else
|
---|
2561 | warn("unexpected token: '#{tk2.inspect}'") if $DEBUG
|
---|
2562 | break
|
---|
2563 | end
|
---|
2564 | end
|
---|
2565 | else
|
---|
2566 | unget_tk tk
|
---|
2567 | if tk = parse_symbol_in_arg
|
---|
2568 | args.push tk
|
---|
2569 | return args if no and args.size >= no
|
---|
2570 | end
|
---|
2571 |
|
---|
2572 | loop do
|
---|
2573 | # skip_tkspace_comment(false)
|
---|
2574 | skip_tkspace(false)
|
---|
2575 |
|
---|
2576 | tk1 = get_tk
|
---|
2577 | unless tk1.kind_of?(TkCOMMA)
|
---|
2578 | unget_tk tk1
|
---|
2579 | break
|
---|
2580 | end
|
---|
2581 |
|
---|
2582 | skip_tkspace_comment
|
---|
2583 | if tk = parse_symbol_in_arg
|
---|
2584 | args.push tk
|
---|
2585 | break if no and args.size >= no
|
---|
2586 | end
|
---|
2587 | end
|
---|
2588 | end
|
---|
2589 | args
|
---|
2590 | end
|
---|
2591 |
|
---|
2592 | def parse_symbol_in_arg
|
---|
2593 | case tk = get_tk
|
---|
2594 | when TkSYMBOL
|
---|
2595 | tk.text.sub(/^:/, '')
|
---|
2596 | when TkSTRING
|
---|
2597 | eval @read[-1]
|
---|
2598 | else
|
---|
2599 | warn("Expected symbol or string, got #{tk.inspect}") if $DEBUG
|
---|
2600 | nil
|
---|
2601 | end
|
---|
2602 | end
|
---|
2603 | end
|
---|
2604 |
|
---|
2605 | end
|
---|