Fix lexing of comments at the EOF (#1371)

A number of lexers included a newline at the end of the pattern for
matching comments. This would result in a comment not being lexed
correctly if it was at the end of text. This commit fixes numerous
instances of this.

lib/rouge/lexers/d.rb

@@ -48,7 +48,7 @@ module new nothrow out override package pragma
       state :root do
         mixin :whitespace
         # Comments
-        rule %r(//(.*?)\n), Comment::Single
+        rule %r(//.*), Comment::Single
         rule %r(/(\\\n)?[*](.|\n)*?[*](\\\n)?/), Comment::Multiline
         rule %r(/\+), Comment::Multiline, :nested_comment
         # Keywords

lib/rouge/lexers/dot.rb

@@ -17,8 +17,8 @@ class Dot < RegexLexer
 
       state :comments_and_whitespace do
         rule %r/\s+/, Text
-        rule %r(#.*?\n), Comment::Single
-        rule %r(//.*?\n), Comment::Single
+        rule %r(#.*), Comment::Single
+        rule %r(//.*?$), Comment::Single
         rule %r(/(\\\n)?[*].*?[*](\\\n)?/)m, Comment::Multiline
       end
 

lib/rouge/lexers/io.rb

@@ -27,8 +27,8 @@ def self.builtins
 
       state :root do
         rule %r/\s+/m, Text
-        rule %r(//.*?\n), Comment::Single
-        rule %r(#.*?\n), Comment::Single
+        rule %r(//.*), Comment::Single
+        rule %r(#.*), Comment::Single
         rule %r(/(\\\n)?[*].*?[*](\\\n)?/)m, Comment::Multiline
         rule %r(/[+]), Comment::Multiline, :nested_comment
 

lib/rouge/lexers/m68k.rb

@@ -87,8 +87,8 @@ def self.builtins
 
       state :whitespace do
         rule %r/\n+/m, Text, :expr_bol
-        rule %r(^\*(\\.|.)*?\n), Comment::Single, :expr_bol
-        rule %r(;(\\.|.)*?\n), Comment::Single, :expr_bol
+        rule %r(^\*(\\.|.)*?$), Comment::Single, :expr_bol
+        rule %r(;(\\.|.)*?$), Comment::Single, :expr_bol
         mixin :inline_whitespace
       end
 

lib/rouge/lexers/nginx.rb

@@ -41,7 +41,7 @@ class Nginx < RegexLexer
       state :base do
         rule %r/\s+/, Text
 
-        rule %r/#.*?\n/, Comment::Single
+        rule %r/#.*/, Comment::Single
         rule %r/(?:on|off)\b/, Name::Constant
         rule %r/[$][\w-]+/, Name::Variable
 

lib/rouge/lexers/pony.rb

@@ -46,7 +46,7 @@ class compiler_intrinsic consume continue
       state :root do
         mixin :whitespace
         rule %r/"""/, Str::Doc, :docstring
-        rule %r{//(.*?)\n}, Comment::Single
+        rule %r{//.*}, Comment::Single
         rule %r{/(\\\n)?[*](.|\n)*?[*](\\\n)?/}, Comment::Multiline
         rule %r/"/, Str, :string
         rule %r([~!%^&*+=\|?:<>/-]), Operator

lib/rouge/lexers/protobuf.rb

@@ -17,7 +17,7 @@ class Protobuf < RegexLexer
       state :root do
         rule %r/[\s]+/, Text
         rule %r/[,;{}\[\]()]/, Punctuation
-        rule %r/\/(\\\n)?\/(\n|(.|\n)*?[^\\]\n)/, Comment::Single
+        rule %r/\/(\\\n)?\/($|(.|\n)*?[^\\]$)/, Comment::Single
         rule %r/\/(\\\n)?\*(.|\n)*?\*(\\\n)?\//, Comment::Multiline
         rule kw, Keyword
         rule datatype, Keyword::Type

lib/rouge/lexers/scala.rb

@@ -43,7 +43,7 @@ class Scala < RegexLexer
         rule %r/'#{idrest}[^']/, Str::Symbol
         rule %r/[^\S\n]+/, Text
 
-        rule %r(//.*?\n), Comment::Single
+        rule %r(//.*), Comment::Single
         rule %r(/\*), Comment::Multiline, :comment
 
         rule %r/@#{idrest}/, Name::Decorator
@@ -117,7 +117,7 @@ class Scala < RegexLexer
         rule %r/\s+/, Text
         rule %r/{/, Operator, :pop!
         rule %r/\(/, Operator, :pop!
-        rule %r(//.*?\n), Comment::Single, :pop!
+        rule %r(//.*), Comment::Single, :pop!
         rule %r(#{idrest}|#{op}+|`[^`]+`), Name::Class, :pop!
       end
 
@@ -142,7 +142,7 @@ class Scala < RegexLexer
           pop!
         end
 
-        rule %r(//.*?\n), Comment::Single, :pop!
+        rule %r(//.*), Comment::Single, :pop!
         rule %r/\.|#{idrest}|#{op}+|`[^`]+`/, Keyword::Type
       end
 

lib/rouge/lexers/sieve.rb

@@ -58,7 +58,7 @@ def self.tests
 
       state :comments_and_whitespace do
         rule %r/\s+/, Text
-        rule %r(#.*?\n), Comment::Single
+        rule %r(#.*), Comment::Single
         rule %r(/(\\\n)?[*].*?[*](\\\n)?/)m, Comment::Multiline
       end
 

lib/rouge/lexers/sqf.rb

@@ -65,7 +65,7 @@ def self.commands
 
         # Preprocessor instructions
         rule %r"/\*.*?\*/"m, Comment::Multiline
-        rule %r"//.*\n", Comment::Single
+        rule %r"//.*", Comment::Single
         rule %r"#(define|undef|if(n)?def|else|endif|include)", Comment::Preproc
         rule %r"\\\r?\n", Comment::Preproc
         rule %r"__(EVAL|EXEC|LINE__|FILE__)", Name::Builtin

lib/rouge/lexers/verilog.rb

@@ -100,7 +100,7 @@ def self.keywords_system_task
 
       state :whitespace do
         rule %r/\n+/m, Text, :bol
-        rule %r(//(\\.|.)*?\n), Comment::Single, :bol
+        rule %r(//(\\.|.)*?$), Comment::Single, :bol
         mixin :inline_whitespace
       end
 

spec/lexers/d_spec.rb

@@ -17,4 +17,12 @@
       assert_guess :mimetype => 'application/x-dsrc'
     end
   end
+
+  describe 'lexing' do
+    include Support::Lexing
+
+    it 'recognizes one-line comments not followed by a newline' do
+      assert_tokens_equal '// comment', ['Comment.Single', '// comment']
+    end
+  end
 end

spec/lexers/dot_spec.rb

@@ -15,4 +15,16 @@
       assert_guess :mimetype => 'text/vnd.graphviz'
     end
   end
+
+  describe 'lexing' do
+    include Support::Lexing
+
+    it 'recognizes one-line "//" comments not followed by a newline' do
+      assert_tokens_equal '// comment', ['Comment.Single', '// comment']
+    end
+
+    it 'recognizes one-line "#" comments not followed by a newline' do
+      assert_tokens_equal '# comment', ['Comment.Single', '# comment']
+    end
+  end
 end

spec/lexers/io_spec.rb

@@ -19,4 +19,16 @@
       assert_guess :source => '#!/usr/local/bin/io'
     end
   end
+
+  describe 'lexing' do
+    include Support::Lexing
+
+    it 'recognizes one-line "//" comments not followed by a newline' do
+      assert_tokens_equal '// comment', ['Comment.Single', '// comment']
+    end
+
+    it 'recognizes one-line "#" comments not followed by a newline' do
+      assert_tokens_equal '# comment', ['Comment.Single', '# comment']
+    end
+  end
 end

spec/lexers/m68k_spec.rb

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+describe Rouge::Lexers::M68k do
+  let(:subject) { Rouge::Lexers::M68k.new }
+
+  describe 'lexing' do
+    include Support::Lexing
+
+    it 'recognizes ";" comments not followed by a newline' do
+      assert_tokens_equal '; comment', ['Comment.Single', '; comment']
+    end
+
+    it 'recognizes "*" comments not followed by a newline' do
+      assert_tokens_equal '* comment', ['Comment.Single', '* comment']
+    end
+  end
+end

spec/lexers/nginx_spec.rb

@@ -16,4 +16,12 @@
       assert_guess :mimetype => 'text/x-nginx-conf'
     end
   end
+
+  describe 'lexing' do
+    include Support::Lexing
+
+    it 'recognizes one-line comments not followed by a newline' do
+      assert_tokens_equal '# comment', ['Comment.Single', '# comment']
+    end
+  end
 end

spec/lexers/pony_spec.rb

@@ -11,4 +11,12 @@
       assert_guess :filename => 'foo.pony'
     end
   end
+
+  describe 'lexing' do
+    include Support::Lexing
+
+    it 'recognizes one-line comments not followed by a newline' do
+      assert_tokens_equal '// comment', ['Comment.Single', '// comment']
+    end
+  end
 end

spec/lexers/protobuf_spec.rb

@@ -15,4 +15,12 @@
       assert_guess :mimetype => 'text/x-proto'
     end
   end
+
+  describe 'lexing' do
+    include Support::Lexing
+
+    it 'recognizes one-line comments not followed by a newline' do
+      assert_tokens_equal '// comment', ['Comment.Single', '// comment']
+    end
+  end
 end

spec/lexers/scala_spec.rb

@@ -17,4 +17,12 @@
       assert_guess :mimetype => 'application/x-scala'
     end
   end
+
+  describe 'lexing' do
+    include Support::Lexing
+
+    it 'recognizes one-line comments not followed by a newline' do
+      assert_tokens_equal '// comment', ['Comment.Single', '// comment']
+    end
+  end
 end

spec/lexers/sieve_spec.rb

@@ -11,4 +11,12 @@
       assert_guess :filename => 'foo.sieve'
     end
   end
+
+  describe 'lexing' do
+    include Support::Lexing
+
+    it 'recognizes one-line comments not followed by a newline' do
+      assert_tokens_equal '# comment', ['Comment.Single', '# comment']
+    end
+  end
 end

spec/lexers/sqf_spec.rb

@@ -11,4 +11,12 @@
       assert_guess :filename => 'foo.sqf'
     end
   end
+
+  describe 'lexing' do
+    include Support::Lexing
+
+    it 'recognizes one-line comments not followed by a newline' do
+      assert_tokens_equal '// comment', ['Comment.Single', '// comment']
+    end
+  end
 end

spec/lexers/verilog_spec.rb

@@ -18,4 +18,12 @@
       assert_guess :mimetype => 'text/x-systemverilog'
     end
   end
+
+  describe 'lexing' do
+    include Support::Lexing
+
+    it 'recognizes one-line comments not followed by a newline' do
+      assert_tokens_equal '// comment', ['Comment.Single', '// comment']
+    end
+  end
 end

spec/visual/samples/d

@@ -1584,3 +1584,5 @@ pure nothrow @safe @nogc unittest
     testVal = null;
     assert(testVal.isNull);
 }
+
+// comment at eof

spec/visual/samples/dot

@@ -16,3 +16,5 @@ digraph graphname {
 
  testing [some=<pretty <i>nasty<font style="red">HTML</font></i>>];
 }
+
+// comment at eof

spec/visual/samples/io

@@ -37,3 +37,5 @@ n := System args at(1) asNumber
 nsieve( (2^n)*10000 )
 nsieve( (2^(n-1))*10000 )
 nsieve( (2^(n-2))*10000 )
+
+// comment at eof

spec/visual/samples/m68k

@@ -57,3 +57,5 @@ restore:                        ; go back into user mode
    rts
 
 oldstack dc.l 0
+
+; comment at eof

spec/visual/samples/nginx

@@ -115,3 +115,5 @@ http {
     }
 
 }
+
+# comment at eof

spec/visual/samples/pony

@@ -183,3 +183,5 @@ actor Main
       --output, -o      File to write the output to.
       """
       )
+
+// Comment at eof

spec/visual/samples/protobuf

@@ -28,3 +28,5 @@ message Person {
 message AddressBook {
   repeated Person person = 1;
 }
+
+// 

spec/visual/samples/scala

@@ -5,6 +5,8 @@ import foo.bar.{Foo, Bar => Baz}
 /* This file /* which is totally legal scala */ should be highlighted
    correcty by rouge */
 
+// Single-line comments too
+
 object ⌘ {
   val `interface` = """
 A
@@ -46,6 +48,12 @@ abstract case class Foo[+A, B <: List[A]](a: A) {
   def apply[C >: A](c: Foo[C]): Foo[C]
 }
 
+class // comment 1
+  Test(a: // comment 2
+    Int) {
+
+}
+
 class why_would_you_name_a_class_this_way_oh_well_we_need_to_highlight_it(a: Int) extends Foo(a) {
   def this(b: Float) = this(b.toInt)
 
@@ -55,3 +63,5 @@ class why_would_you_name_a_class_this_way_oh_well_we_need_to_highlight_it(a: Int
   def longs = 4L
   def hex = 0x123
 }
+
+// Comment at EOF

spec/visual/samples/sieve

@@ -38,3 +38,5 @@ header :matches "Subject" ["*money*","*Viagra*"]) {
 else {
      keep;
 }
+
+# Comment at EOL

spec/visual/samples/sqf

@@ -48,3 +48,5 @@ if (_target find "var:" == 0) then {
     _commandList = _commandList joinString _delimiter;
     copyToClipboard _commandList;
 };
+
+// comment at eof