Improve number literals for several languages

Using these examples:

~~~
0x21
1_000
1e3
~~~

Several languages failed that support these syntax. Here are ones I found:

Lexer      | 0x21 | 1_000 | 1e3
-----------|------|-------|-----
C#         | fail | fail  | fail
Go         |      | fail  |
JavaScript |      | fail  | fail
PHP        |      | fail  |
Python     |      | fail  |
Ruby       |      |       | fail

I fixed these issues, and added tests.

lexers/c/csharp.go

@@ -29,7 +29,7 @@ var CSharp = internal.Register(MustNewLexer(
 			{`\$@?"(""|[^"])*"`, LiteralString, nil},
 			{`"(\\\\|\\"|[^"\n])*["\n]`, LiteralString, nil},
 			{`'\\.'|'[^\\]'`, LiteralStringChar, nil},
-			{`[0-9](\.[0-9]*)?([eE][+-][0-9]+)?[flFLdD]?|0[xX][0-9a-fA-F]+[Ll]?`, LiteralNumber, nil},
+			{`0[xX][0-9a-fA-F]+[Ll]?|[0-9_](\.[0-9]*)?([eE][+-]?[0-9]+)?[flFLdD]?`, LiteralNumber, nil},
 			{`#[ \t]*(if|endif|else|elif|define|undef|line|error|warning|region|endregion|pragma)\b.*?\n`, CommentPreproc, nil},
 			{`\b(extern)(\s+)(alias)\b`, ByGroups(Keyword, Text, Keyword), nil},
 			{`(abstract|as|async|await|base|break|by|case|catch|checked|const|continue|default|delegate|do|else|enum|event|explicit|extern|false|finally|fixed|for|foreach|goto|if|implicit|in|interface|internal|is|let|lock|new|null|on|operator|out|override|params|private|protected|public|readonly|ref|return|sealed|sizeof|stackalloc|static|switch|this|throw|true|try|typeof|unchecked|unsafe|virtual|void|while|get|set|new|partial|yield|add|remove|value|alias|ascending|descending|from|group|into|orderby|select|thenby|where|join|equals)\b`, Keyword, nil},

lexers/circular/php.go

@@ -47,7 +47,7 @@ var phpCommonRules = Rules{
 		{`\d+e[+-]?[0-9]+`, LiteralNumberFloat, nil},
 		{`0[0-7]+`, LiteralNumberOct, nil},
 		{`0x[a-f0-9]+`, LiteralNumberHex, nil},
-		{`\d+`, LiteralNumberInteger, nil},
+		{`[\d_]+`, LiteralNumberInteger, nil},
 		{`0b[01]+`, LiteralNumberBin, nil},
 		{`'([^'\\]*(?:\\.[^'\\]*)*)'`, LiteralStringSingle, nil},
 		{"`([^`\\\\]*(?:\\\\.[^`\\\\]*)*)`", LiteralStringBacktick, nil},

lexers/g/go.go

@@ -38,7 +38,7 @@ var Go = internal.Register(MustNewLexer(
 			{`\.\d+([eE][+\-]?\d+)?`, LiteralNumberFloat, nil},
 			{`0[0-7]+`, LiteralNumberOct, nil},
 			{`0[xX][0-9a-fA-F]+`, LiteralNumberHex, nil},
-			{`(0|[1-9][0-9]*)`, LiteralNumberInteger, nil},
+			{`(0|[1-9][0-9_]*)`, LiteralNumberInteger, nil},
 			{`'(\\['"\\abfnrtv]|\\x[0-9a-fA-F]{2}|\\[0-7]{1,3}|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}|[^\\])'`, LiteralStringChar, nil},
 			{"(`)([^`]*)(`)", ByGroups(LiteralString, Using(TypeRemappingLexer(GoTextTemplate, TypeMapping{{Other, LiteralString, nil}})), LiteralString), nil},
 			{`"(\\\\|\\"|[^"])*"`, LiteralString, nil},

lexers/j/javascript.go

@@ -26,11 +26,11 @@ var JavascriptRules = Rules{
 		{`\A#! ?/.*?\n`, CommentHashbang, nil},
 		{`^(?=\s|/|<!--)`, Text, Push("slashstartsregex")},
 		Include("commentsandwhitespace"),
-		{`(\.\d+|[0-9]+\.[0-9]*)([eE][-+]?[0-9]+)?`, LiteralNumberFloat, nil},
+		{`\d+(\.\d*|[eE][+\-]?\d+)`, LiteralNumberFloat, nil},
 		{`0[bB][01]+`, LiteralNumberBin, nil},
 		{`0[oO][0-7]+`, LiteralNumberOct, nil},
 		{`0[xX][0-9a-fA-F]+`, LiteralNumberHex, nil},
-		{`[0-9]+`, LiteralNumberInteger, nil},
+		{`[0-9_]+`, LiteralNumberInteger, nil},
 		{`\.\.\.|=>`, Punctuation, nil},
 		{`\+\+|--|~|&&|\?|:|\|\||\\(?=\n)|(<<|>>>?|==?|!=?|[-<>+*%&|^/])=?`, Operator, Push("slashstartsregex")},
 		{`[{(\[;,]`, Punctuation, Push("slashstartsregex")},

lexers/p/python.go

@@ -67,7 +67,7 @@ var Python = internal.Register(MustNewLexer(
 			{`0[bB][01]+`, LiteralNumberBin, nil},
 			{`0[xX][a-fA-F0-9]+`, LiteralNumberHex, nil},
 			{`\d+L`, LiteralNumberIntegerLong, nil},
-			{`\d+j?`, LiteralNumberInteger, nil},
+			{`[\d_]+j?`, LiteralNumberInteger, nil},
 		},
 		"backtick": {
 			{"`.*?`", LiteralStringBacktick, nil},

lexers/r/ruby.go

@@ -39,7 +39,7 @@ var Ruby = internal.Register(MustNewLexer(
 			{`(0_?[0-7]+(?:_[0-7]+)*)(\s*)([/?])?`, ByGroups(LiteralNumberOct, Text, Operator), nil},
 			{`(0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*)(\s*)([/?])?`, ByGroups(LiteralNumberHex, Text, Operator), nil},
 			{`(0b[01]+(?:_[01]+)*)(\s*)([/?])?`, ByGroups(LiteralNumberBin, Text, Operator), nil},
-			{`([\d]+(?:_\d+)*)(\s*)([/?])?`, ByGroups(LiteralNumberInteger, Text, Operator), nil},
+			{`([\d]+(?:[_e]\d+)*)(\s*)([/?])?`, ByGroups(LiteralNumberInteger, Text, Operator), nil},
 			{`@@[a-zA-Z_]\w*`, NameVariableClass, nil},
 			{`@[a-zA-Z_]\w*`, NameVariableInstance, nil},
 			{`\$\w+`, NameVariableGlobal, nil},

lexers/testdata/csharp.actual

@@ -10,4 +10,7 @@ foreach (DriveInfo drive in drives)
   }
 }
 
+int n = 0x21 + 1_000;
+double n2 = 1e3;
+
 // Comment as last line should be recognised properly

lexers/testdata/csharp.expected

@@ -70,5 +70,27 @@
   {"type":"Text","value":"\n"},
   {"type":"Punctuation","value":"}"},
   {"type":"Text","value":"\n\n"},
+  {"type":"KeywordType","value":"int"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"n"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumber","value":"0x21"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"+"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumber","value":"1_000"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n"},
+  {"type":"KeywordType","value":"double"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"n2"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumber","value":"1e3"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n\n"},
   {"type":"CommentSingle","value":"// Comment as last line should be recognised properly\n"}
 ]

lexers/testdata/go.actual

@@ -6,6 +6,9 @@ func main() {
     fmt.Println("Hello World!")
 }
 
+var n int = 0x21 + 1_000
+var n2 float64 = 1e3
+
 func hello(a int) {
 	fmt.Println("Hello World!").Hello()
 

lexers/testdata/go.expected

@@ -23,6 +23,30 @@
   {"type":"Text","value":"\n"},
   {"type":"Punctuation","value":"}"},
   {"type":"Text","value":"\n\n"},
+  {"type":"KeywordDeclaration","value":"var"},
+  {"type":"Text","value":" "},
+  {"type":"NameOther","value":"n"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"int"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberHex","value":"0x21"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"+"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"1_000"},
+  {"type":"Text","value":"\n"},
+  {"type":"KeywordDeclaration","value":"var"},
+  {"type":"Text","value":" "},
+  {"type":"NameOther","value":"n2"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"float64"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberFloat","value":"1e3"},
+  {"type":"Text","value":"\n\n"},
   {"type":"KeywordDeclaration","value":"func"},
   {"type":"Text","value":" "},
   {"type":"NameFunction","value":"hello"},

lexers/testdata/javascript.actual

@@ -1,3 +1,6 @@
 sum = parseInt(num1) + parseInt(num2) // "+" means "add"
 alert("Sum = " + sum)  // "+" means combine into a string
 const filePath = String.raw`C:\Development\profile\aboutme.html`;
+
+let n = 0x21 + 1_000;
+let n2 = 1e3;

lexers/testdata/javascript.expected

@@ -37,5 +37,27 @@
   {"type":"NameOther","value":"raw"},
   {"type":"LiteralStringBacktick","value":"`C:\\Development\\profile\\aboutme.html`"},
   {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"KeywordDeclaration","value":"let"},
+  {"type":"Text","value":" "},
+  {"type":"NameOther","value":"n"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberHex","value":"0x21"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"+"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"1_000"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n"},
+  {"type":"KeywordDeclaration","value":"let"},
+  {"type":"Text","value":" "},
+  {"type":"NameOther","value":"n2"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberFloat","value":"1e3"},
+  {"type":"Punctuation","value":";"},
   {"type":"Text","value":"\n"}
 ]

lexers/testdata/php.actual

@@ -1,5 +1,8 @@
 $docs = $modx->getIterator('modResource', ["parent" => 84]);
 
+$n = 0x21 + 1_000;
+$n2 = 1e3;
+
 foreach($docs as $doc){
     $q=$doc->content;
     $doc->set("content", preg_replace("/Some value/i", "Replacement", $q));

lexers/testdata/php.expected

@@ -18,6 +18,24 @@
   {"type":"LiteralNumberInteger","value":"84"},
   {"type":"Punctuation","value":"]);"},
   {"type":"Text","value":"\n\n"},
+  {"type":"NameVariable","value":"$n"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberHex","value":"0x21"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"+"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"1_000"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n"},
+  {"type":"NameVariable","value":"$n2"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberFloat","value":"1e3"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n\n"},
   {"type":"Keyword","value":"foreach"},
   {"type":"Punctuation","value":"("},
   {"type":"NameVariable","value":"$docs"},

lexers/testdata/python.actual

@@ -0,0 +1,2 @@
+n = 0x21 + 1_000
+n2 = 1e3

lexers/testdata/python.expected

@@ -0,0 +1,18 @@
+[
+  {"type":"Name","value":"n"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberHex","value":"0x21"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"+"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"1_000"},
+  {"type":"Text","value":"\n"},
+  {"type":"Name","value":"n2"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberFloat","value":"1e3"},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/ruby.actual

@@ -0,0 +1,2 @@
+n = 0x21 + 1_000
+n2 = 1e3

lexers/testdata/ruby.expected

@@ -0,0 +1,18 @@
+[
+  {"type":"Name","value":"n"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberHex","value":"0x21"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"+"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"1_000"},
+  {"type":"Text","value":"\n"},
+  {"type":"Name","value":"n2"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"1e3"},
+  {"type":"Text","value":"\n"}
+]