Fix lexing of a bb <<-HEREDOC. Issue #75

Author: drbrain

History.rdoc

@@ -86,6 +86,7 @@
   * The C parser now understands classes defined with
     +rb_struct_define_without_accessor+ (like Range).  Pull Request #73 by Dan
     Bernier
+  * Fixed lexing of <code>a b <<-HEREDOC</code>.  Issue #75 by John Mair.
 
 === 3.9.2 / 2011-08-11
 

TODO.rdoc

@@ -11,6 +11,9 @@ Some file contains some things that might happen in RDoc, or might not
 * Add direct accessor to RDoc::Options to RDoc::Task
 * Remove Public in HTML output if there are only public methods
 * Method markup support for rd documentation
+* Improve SIGINFO handling
+* Global variable support
+* Page support for ri
 
 === 4
 

lib/rdoc/ruby_lex.rb

@@ -55,7 +55,7 @@ def self.debug?
   end
 
   self.debug_level = 0
-  
+
   # :startdoc:
 
   ##
@@ -69,7 +69,7 @@ def self.tokenize ruby, options
     scanner.exception_on_syntax_error = true
 
     while token = scanner.token do
-      tokens << token 
+      tokens << token
     end
 
     tokens
@@ -960,7 +960,8 @@ def identify_identifier
       if peek(0) == '='
         token.concat getc
       end
-    elsif @lex_state == EXPR_BEG || @lex_state == EXPR_DOT
+    elsif @lex_state == EXPR_BEG || @lex_state == EXPR_DOT ||
+          @lex_state == EXPR_ARG
       @lex_state = EXPR_ARG
     else
       @lex_state = EXPR_END

test/test_rdoc_ruby_lex.rb

@@ -50,6 +50,25 @@ def test_class_tokenize_heredoc
     assert_equal expected, tokens
   end
 
+  def test_class_tokenize_heredoc_percent_N
+    tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
+a b <<-U
+%N
+U
+    RUBY
+
+    expected = [
+      @TK::TkIDENTIFIER.new( 0, 1,  0, 'a'),
+      @TK::TkSPACE     .new( 1, 1,  1, ' '),
+      @TK::TkIDENTIFIER.new( 2, 1,  2, 'b'),
+      @TK::TkSPACE     .new( 3, 1,  3, ' '),
+      @TK::TkSTRING    .new( 4, 1,  4, %Q{"%N\n"}),
+      @TK::TkNL        .new(13, 3, 14, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
   def test_unary_minus
     ruby_lex = RDoc::RubyLex.new("-1", nil)
     assert_equal("-1", ruby_lex.token.value)