Recognize Backslash as a token

Author: Thirumalai-Shaktivel

src/lpython/parser/parser.tab.hh

@@ -248,7 +248,7 @@
     UMINUS = 494,
     TK_DOCSTRING = 495,
     TK_LBRACE = 496,
-    TK_RBRACE = 497, 
+    TK_RBRACE = 497,
     TK_AMPERSAND = 498,
     TK_DOT = 499,
     TK_BACKQUOTE = 500,
@@ -269,8 +269,9 @@
     TK_RIGHTSHIFT_EQUAL = 515,
     TK_POW_EQUAL = 516,
     TK_DOUBLESLASH_EQUAL = 517,
-    KW_FROM = 518,
-    KW_DEF = 519
+    TK_BACKSLASH = 518,
+    KW_FROM = 519,
+    KW_DEF = 520
   };
 
 #endif // LPYTHON_PARSER_PARSER_TAB_H

src/lpython/parser/tokenizer.re

@@ -419,7 +419,7 @@ int Tokenizer::lex(Allocator &al, YYSTYPE &yylval, Location &loc, diag::Diagnost
             'in' { KW(IN) }
             'include' { KW(INCLUDE) }
             'inout' { KW(INOUT) }
-            'in' whitespace 'out' { KW(IN_OUT) }
+            // 'in' whitespace 'out' { KW(IN_OUT) }
             'inquire' { KW(INQUIRE) }
             'integer' { KW(INTEGER) }
             'intent' { KW(INTENT) }
@@ -537,6 +537,7 @@ int Tokenizer::lex(Allocator &al, YYSTYPE &yylval, Location &loc, diag::Diagnost
             ":" { RET(TK_COLON) }
             ";" { RET(TK_SEMICOLON) }
             "/" { RET(TK_SLASH) }
+            "\\" { RET(TK_BACKSLASH) }
             "%" { RET(TK_PERCENT) }
             "," { RET(TK_COMMA) }
             "*" { RET(TK_STAR) }
@@ -600,7 +601,7 @@ int Tokenizer::lex(Allocator &al, YYSTYPE &yylval, Location &loc, diag::Diagnost
 
             'True' { RET(TK_TRUE) }
             'False' { RET(TK_FALSE) }
-            
+
             // This is needed to ensure that 2.op.3 gets tokenized as
             // TK_INTEGER(2), TK_DEFOP(.op.), TK_INTEGER(3), and not
             // TK_REAL(2.), TK_NAME(op), TK_REAL(.3). The `.op.` can be a
@@ -817,6 +818,7 @@ std::string token2text(const int token)
         T(TK_MINUS, "-")
         T(TK_STAR, "*")
         T(TK_SLASH, "/")
+        T(TK_BACKSLASH, "\\")
         T(TK_COLON, ":")
         T(TK_SEMICOLON, ";")
         T(TK_COMMA, ",")
@@ -1110,8 +1112,8 @@ std::string pickle_token(int token, const LFortran::YYSTYPE &yystype)
 {
     std::string t;
     t += "(";
-    if (token >= yytokentype::TK_NAME && token <= TK_FALSE ||
-        token >= yytokentype::TK_DOCSTRING && token <= TK_DOUBLESLASH_EQUAL) {
+    if ((token >= yytokentype::TK_NAME && token <= TK_FALSE) ||
+            (token >= yytokentype::TK_DOCSTRING && token <= TK_BACKSLASH)) {
         t += "TOKEN";
     } else if (token == yytokentype::TK_NEWLINE) {
         t += "NEWLINE";

tests/reference/cpp-symbols1-658c990.json

@@ -2,11 +2,11 @@
     "basename": "cpp-symbols1-658c990",
     "cmd": "lpython --no-color --show-tokens {infile} -o {outfile}",
     "infile": "tests/tokens/symbols1.py",
-    "infile_hash": "c1a4fc8a1c2188105740fc26dd5e7b8e0aeab378fec8685ec6465f06",
+    "infile_hash": "4f8b111c9caf59245efae13d9d9efbf2ce5f2c42d0850079e6313903",
     "outfile": null,
     "outfile_hash": null,
     "stdout": "cpp-symbols1-658c990.stdout",
-    "stdout_hash": "84f6a8d52d509259941bca1eee0b4adb5a2749af89483e64a27228f2",
+    "stdout_hash": "0c55157185187ddf4454e6129e742698606b70075a4e689d340f770f",
     "stderr": null,
     "stderr_hash": null,
     "returncode": 0

tests/reference/cpp-symbols1-658c990.stdout

@@ -244,4 +244,23 @@
 (TOKEN "identifier" x) 518:518
 (TOKEN "=") 520:520
 (TOKEN "identifier" x) 522:522
-(EOF) 523:523
+(NEWLINE) 523:523
+(NEWLINE) 524:524
+(KEYWORD "if") 525:526
+(TOKEN "identifier" i) 528:528
+(TOKEN ">=") 530:531
+(TOKEN "integer" 1) 533:533
+(TOKEN "or") 535:536
+(TOKEN "\") 538:538
+(NEWLINE) 539:539
+(TOKEN "identifier" j) 548:548
+(TOKEN "<=") 550:551
+(TOKEN "integer" 10) 553:554
+(TOKEN ":") 555:555
+(NEWLINE) 556:556
+(KEYWORD "print") 561:565
+(TOKEN "(") 566:566
+(TOKEN "string" "Backlash") 567:576
+(TOKEN ")") 577:577
+(NEWLINE) 578:578
+(EOF) 579:579

tests/tokens/symbols1.py

@@ -45,4 +45,8 @@ def test(x):
 
 class t:
     def __init__(self, x):
-        self.x = x
+        self.x = x
+
+if i >= 1 or \
+        j <= 10:
+    print("Backlash")