From 404538bc370bb65ac2255b128d67c73f73c14e56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= <ondrej@certik.us>
Date: Sat, 2 Apr 2022 14:54:51 -0600
Subject: [PATCH] Do not recognize docstrings in the tokenizer

In Python, docstrings seem to be the first item in the body of a
function. They are not specifically recognized in the parser, and not in
the tokenizer either. Thus we should follow the same approach.

Tests were updated.

This also fixed the warning:

    + re2c -W -b tokenizer.re -o tokenizer.cpp
    tokenizer.re:684:20: warning: unreachable rule (shadowed by rule at line 677) [-Wunreachable-rules]
---
 src/lpython/parser/tokenizer.re               |  2 +-
 .../reference/tokens-docstring1-1355fbb.json  |  2 +-
 .../tokens-docstring1-1355fbb.stdout          | 20 +++++++++++++++----
 3 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/src/lpython/parser/tokenizer.re b/src/lpython/parser/tokenizer.re
index 615bec5894..2a8a8e9c4f 100644
--- a/src/lpython/parser/tokenizer.re
+++ b/src/lpython/parser/tokenizer.re
@@ -674,7 +674,7 @@ int Tokenizer::lex(Allocator &al, YYSTYPE &yylval, Location &loc, diag::Diagnost
                     return yytokentype::TK_EOLCOMMENT;
                 }
             }
-            docstring { RET(TK_DOCSTRING) }
+            //docstring { RET(TK_DOCSTRING) }
 
             // Include statements are ignored for now
             'include' whitespace string1 { continue; }
diff --git a/tests/reference/tokens-docstring1-1355fbb.json b/tests/reference/tokens-docstring1-1355fbb.json
index 25329371d7..0ca0c64c16 100644
--- a/tests/reference/tokens-docstring1-1355fbb.json
+++ b/tests/reference/tokens-docstring1-1355fbb.json
@@ -6,7 +6,7 @@
     "outfile": null,
     "outfile_hash": null,
     "stdout": "tokens-docstring1-1355fbb.stdout",
-    "stdout_hash": "8002935592daf4c952a5edb182c847a27e183ec70c3239fd060545f8",
+    "stdout_hash": "4b40183da86ff53034cdbf2b7a70333443f5a233a9086caec0b81d69",
     "stderr": null,
     "stderr_hash": null,
     "returncode": 0
diff --git a/tests/reference/tokens-docstring1-1355fbb.stdout b/tests/reference/tokens-docstring1-1355fbb.stdout
index 2a994af995..c6b8ce37ec 100644
--- a/tests/reference/tokens-docstring1-1355fbb.stdout
+++ b/tests/reference/tokens-docstring1-1355fbb.stdout
@@ -3,7 +3,10 @@
 (TOKEN "(") 9:9
 (TOKEN ")") 10:10
 (TOKEN ":") 11:11
-(TOKEN "docstring") 12:54
+(NEWLINE) 12:12
+(TOKEN "string" """A multi-line
+    docstring.
+    """) 17:54
 (NEWLINE) 55:55
 (NEWLINE) 56:56
 (KEYWORD "def") 57:59
@@ -11,7 +14,11 @@
 (TOKEN "(") 66:66
 (TOKEN ")") 67:67
 (TOKEN ":") 68:68
-(TOKEN "docstring") 69:116
+(NEWLINE) 69:69
+(TOKEN "string" """
+    A multi-line
+    docstring.
+    """) 74:116
 (NEWLINE) 117:117
 (NEWLINE) 118:118
 (KEYWORD "def") 119:121
@@ -19,8 +26,13 @@
 (TOKEN "(") 128:128
 (TOKEN ")") 129:129
 (TOKEN ":") 130:130
-(TOKEN "docstring") 131:166
+(NEWLINE) 131:131
+(TOKEN "string" """ A single-line docstring.""") 136:166
 (NEWLINE) 167:167
-(TOKEN "docstring") 168:199
+(NEWLINE) 168:168
+(TOKEN "string" """
+A multi-line
+docstring.
+""") 169:199
 (NEWLINE) 200:200
 (EOF) 201:201