Skip to content

Commit 4ad3d86

Browse files
timfeltimfel
committed
[GR-38986] Make graalpython unittests pass with bytecode interpreter.
PullRequest: graalpython/2292
2 parents d5d9a05 + fcb3de7 commit 4ad3d86

File tree

78 files changed

+1248
-677
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

78 files changed

+1248
-677
lines changed

‎graalpython/com.oracle.graal.python.pegparser.generator/pegjava/java_generator.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -179,8 +179,8 @@ def __str__(self) -> str:
179179
'CHECK_VERSION ( expr_ty , 5 , "The \'@\' operator is" , _PyAST_BinOp ( a , MatMult , b , EXTRA ) )': (1, 'checkVersion(5, "The \'@\' operator is", factory.createBinaryOp(ExprTy.BinOp.Operator.MATMULT, a, b, $RANGE))'),
180180
'CHECK_VERSION ( stmt_ty , 5 , "Async functions are" , _PyAST_AsyncFunctionDef ( n -> v . Name . id , ( params ) ? params : CHECK ( arguments_ty , _PyPegen_empty_arguments ( p ) ) , b , NULL , a , NEW_TYPE_COMMENT ( p , tc ) , EXTRA ) )': (1, 'checkVersion(5, "Async functions are", factory.createAsyncFunctionDef(((ExprTy.Name) n).id, params, b, a, newTypeComment((Token) tc), $RANGE))'),
181181
'CHECK_VERSION ( stmt_ty , 6 , "Variable annotation syntax is" , _PyAST_AnnAssign ( CHECK ( expr_ty , _PyPegen_set_expr_context ( p , a , Store ) ) , b , c , 1 , EXTRA ) )': (1, 'factory.createAnnAssignment(setExprContext(a, ExprContext.Store), b, (ExprTy) c, true, $RANGE);'),
182-
'RAISE_ERROR_KNOWN_LOCATION ( p , PyExc_SyntaxError , a -> lineno , a -> end_col_offset - 1 , "\':\' expected after dictionary key" )': (1, 'this.raiseErrorKnownLocation(ParserErrorCallback.ErrorType.Syntax,a,"\':\' expected after dictionary key")'),
183-
'RAISE_ERROR_KNOWN_LOCATION ( p , PyExc_SyntaxError , a -> lineno , a -> end_col_offset - 1 , "invalid syntax. Perhaps you forgot a comma?" )': (1, 'this.raiseErrorKnownLocation(ParserErrorCallback.ErrorType.Syntax,a,"invalid syntax.Perhaps you forgot a comma?")'),
182+
'RAISE_ERROR_KNOWN_LOCATION ( p , PyExc_SyntaxError , a -> lineno , a -> end_col_offset - 1 , "\':\' expected after dictionary key" )': (1, 'this.raiseErrorKnownLocation(ErrorCallback.ErrorType.Syntax,a,"\':\' expected after dictionary key")'),
183+
'RAISE_ERROR_KNOWN_LOCATION ( p , PyExc_SyntaxError , a -> lineno , a -> end_col_offset - 1 , "invalid syntax. Perhaps you forgot a comma?" )': (1, 'this.raiseErrorKnownLocation(ErrorCallback.ErrorType.Syntax,a,"invalid syntax.Perhaps you forgot a comma?")'),
184184
'RAISE_SYNTAX_ERROR ( "Cannot have two type comments on def" )': (1, 'this.raiseSyntaxError("Cannot have two type comments on def")'),
185185
'RAISE_SYNTAX_ERROR ( "bare * has associated type comment" )': (1, 'this.raiseSyntaxError("bare * has associated type comment")'),
186186
'RAISE_SYNTAX_ERROR ( "expected \':\'" )': (4, 'this.raiseSyntaxError("expected \':\'")'),
@@ -949,7 +949,7 @@ def generate(self, filename: str) -> None:
949949
with self.indent():
950950
self.print("super(tokenizer, factory, fexprParser);")
951951
self.print("}" )
952-
self.print("public %s(ParserTokenizer tokenizer, NodeFactory factory, FExprParser fexprParser, ParserErrorCallback errorCb) {" % className)
952+
self.print("public %s(ParserTokenizer tokenizer, NodeFactory factory, FExprParser fexprParser, ErrorCallback errorCb) {" % className)
953953
with self.indent():
954954
self.print("super(tokenizer, factory, fexprParser, errorCb);")
955955
self.print("}" )

‎graalpython/com.oracle.graal.python.pegparser.test/src/com/oracle/graal/python/pegparser/ParserTestBase.java

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ public class ParserTestBase {
7373

7474
private static final boolean REGENERATE_TREE = false;
7575

76-
protected ParserErrorCallback lastParserErrorCallback;
76+
protected ErrorCallback lastParserErrorCallback;
7777

7878
@Rule public TestName name = new TestName();
7979

@@ -125,15 +125,15 @@ public void checkSyntaxError(String source) {
125125
parse(source, getFileName(), 1);
126126
DefaultParserErrorCallback ec = (DefaultParserErrorCallback) lastParserErrorCallback;
127127
assertTrue("Expected Error.", ec.hasErrors());
128-
assertSame("Expected SyntaxError", ec.getErrors().get(0).getType(), ParserErrorCallback.ErrorType.Syntax);
128+
assertSame("Expected SyntaxError", ec.getErrors().get(0).getType(), ErrorCallback.ErrorType.Syntax);
129129
}
130130

131131
public void checkSyntaxErrorMessageContains(String source, String expectedMessage) {
132132
parse(source, getFileName(), 1);
133133
DefaultParserErrorCallback ec = (DefaultParserErrorCallback) lastParserErrorCallback;
134134
assertTrue("Expected Error.", ec.hasErrors());
135135
DefaultParserErrorCallback.Error error = ec.getErrors().get(0);
136-
assertSame("Expected SyntaxError not " + error.getType(), error.getType(), ParserErrorCallback.ErrorType.Syntax);
136+
assertSame("Expected SyntaxError not " + error.getType(), error.getType(), ErrorCallback.ErrorType.Syntax);
137137
assertTrue("The expected message:\n\"" + expectedMessage + "\"\nwas not found in\n\"" + error.getMessage() + "\"", error.getMessage().contains(expectedMessage));
138138
}
139139

@@ -142,7 +142,7 @@ public void checkSyntaxErrorMessage(String source, String expectedMessage) {
142142
DefaultParserErrorCallback ec = (DefaultParserErrorCallback) lastParserErrorCallback;
143143
assertTrue("Expected Error.", ec.hasErrors());
144144
DefaultParserErrorCallback.Error error = ec.getErrors().get(0);
145-
assertSame("Expected SyntaxError not " + error.getType(), error.getType(), ParserErrorCallback.ErrorType.Syntax);
145+
assertSame("Expected SyntaxError not " + error.getType(), error.getType(), ErrorCallback.ErrorType.Syntax);
146146
assertEquals("The expected message:\n\"" + expectedMessage + "\"\n was not found. The message is: \n\"" + error.getMessage() + "\"", error.getMessage(), expectedMessage);
147147
}
148148

@@ -202,9 +202,9 @@ public void checkError(String source, String... expectedErrors) {
202202
ParserTokenizer tokenizer = new ParserTokenizer(source);
203203
NodeFactory factory = new NodeFactoryImp();
204204
ArrayList<String> errors = new ArrayList<>();
205-
ParserErrorCallback errorCb = new ParserErrorCallback() {
205+
ErrorCallback errorCb = new ErrorCallback() {
206206
@Override
207-
public void onError(ParserErrorCallback.ErrorType type, SourceRange sourceRange, String message) {
207+
public void onError(ErrorCallback.ErrorType type, SourceRange sourceRange, String message) {
208208
errors.add(String.format("%s[%d:%d]:%s", type.name(), sourceRange.startOffset, sourceRange.endOffset, message));
209209
}
210210
};

‎graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/AbstractParser.java

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ public enum TargetsType {
8282
private static final String BARRY_AS_BDFL = "with Barry as BDFL, use '<>' instead of '!='";
8383

8484
private final ParserTokenizer tokenizer;
85-
private final ParserErrorCallback errorCb;
85+
private final ErrorCallback errorCb;
8686
private final FExprParser fexprParser;
8787
protected final NodeFactory factory;
8888

@@ -91,6 +91,9 @@ public enum TargetsType {
9191
protected int level = 0;
9292
protected boolean callInvalidRules = false;
9393

94+
private boolean isInteractive;
95+
private boolean parsingStarted;
96+
9497
/**
9598
* Indicates, whether there was found an error
9699
*/
@@ -120,11 +123,11 @@ public AbstractParser(ParserTokenizer tokenizer, NodeFactory factory, FExprParse
120123
this(tokenizer, factory, fexprParser, new DefaultParserErrorCallback(), flags);
121124
}
122125

123-
public AbstractParser(ParserTokenizer tokenizer, NodeFactory factory, FExprParser fexprParser, ParserErrorCallback errorCb) {
126+
public AbstractParser(ParserTokenizer tokenizer, NodeFactory factory, FExprParser fexprParser, ErrorCallback errorCb) {
124127
this(tokenizer, factory, fexprParser, errorCb, 0);
125128
}
126129

127-
public AbstractParser(ParserTokenizer tokenizer, NodeFactory factory, FExprParser fexprParser, ParserErrorCallback errorCb, int flags) {
130+
public AbstractParser(ParserTokenizer tokenizer, NodeFactory factory, FExprParser fexprParser, ErrorCallback errorCb, int flags) {
128131
this.tokenizer = tokenizer;
129132
this.factory = factory;
130133
this.fexprParser = fexprParser;
@@ -135,11 +138,12 @@ public AbstractParser(ParserTokenizer tokenizer, NodeFactory factory, FExprParse
135138
this.fill = 0;
136139
}
137140

138-
public ParserErrorCallback getErrorCallback() {
141+
public ErrorCallback getErrorCallback() {
139142
return errorCb;
140143
}
141144

142145
public SSTNode parse(InputType inputType) {
146+
isInteractive = inputType == InputType.SINGLE;
143147
SSTNode res = runParser(inputType);
144148
if (res == null) {
145149
resetParserState();
@@ -265,7 +269,13 @@ public Token getAndInitializeToken() {
265269
}
266270
reset(pos);
267271

268-
// TODO: handle reaching end in single_input mode
272+
if (isInteractive && token.type == Token.Kind.ENDMARKER && parsingStarted) {
273+
token.type = Token.Kind.NEWLINE;
274+
parsingStarted = false;
275+
// TODO: handle implicit DEDENT (PyPARSE_DONT_IMPLY_DEDENT)
276+
} else {
277+
parsingStarted = true;
278+
}
269279
return initializeToken(token);
270280
}
271281

@@ -843,7 +853,7 @@ SSTNode raiseSyntaxErrorInvalidTarget(TargetsType type, ExprTy expr) {
843853
SSTNode raiseSyntaxError(String msg, Object... arguments) {
844854
errorIndicator = true;
845855
Token errorToken = tokenizer.peekToken();
846-
errorCb.onError(ParserErrorCallback.ErrorType.Syntax, errorToken.sourceRange, msg, arguments);
856+
errorCb.onError(ErrorCallback.ErrorType.Syntax, errorToken.sourceRange, msg, arguments);
847857
return null;
848858
}
849859

@@ -852,7 +862,7 @@ SSTNode raiseSyntaxError(String msg, Object... arguments) {
852862
*/
853863
SSTNode raiseSyntaxErrorKnownLocation(Token errorToken, String msg, Object... argument) {
854864
errorIndicator = true;
855-
errorCb.onError(ParserErrorCallback.ErrorType.Syntax, errorToken.sourceRange, msg, argument);
865+
errorCb.onError(ErrorCallback.ErrorType.Syntax, errorToken.sourceRange, msg, argument);
856866
return null;
857867
}
858868

@@ -861,14 +871,14 @@ SSTNode raiseSyntaxErrorKnownLocation(Token errorToken, String msg, Object... ar
861871
*/
862872
SSTNode raiseSyntaxErrorKnownLocation(SSTNode where, String msg, Object... argument) {
863873
errorIndicator = true;
864-
errorCb.onError(ParserErrorCallback.ErrorType.Syntax, where.getSourceRange(), msg, argument);
874+
errorCb.onError(ErrorCallback.ErrorType.Syntax, where.getSourceRange(), msg, argument);
865875
return null;
866876
}
867877

868878
/**
869879
* RAISE_ERROR_KNOWN_LOCATION
870880
*/
871-
SSTNode raiseErrorKnownLocation(ParserErrorCallback.ErrorType typeError, SSTNode where, String msg, Object... argument) {
881+
SSTNode raiseErrorKnownLocation(ErrorCallback.ErrorType typeError, SSTNode where, String msg, Object... argument) {
872882
errorIndicator = true;
873883
errorCb.onError(typeError, where.getSourceRange(), msg, argument);
874884
return null;

‎graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/DefaultParserErrorCallback.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,10 +46,10 @@
4646
import java.util.ArrayList;
4747
import java.util.List;
4848

49-
public class DefaultParserErrorCallback implements ParserErrorCallback {
49+
public class DefaultParserErrorCallback implements ErrorCallback {
5050

5151
public static class Error {
52-
private final ParserErrorCallback.ErrorType type;
52+
private final ErrorCallback.ErrorType type;
5353
private SourceRange sourceRange;
5454
private final String message;
5555

‎graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/ParserErrorCallback.java renamed to ‎graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/ErrorCallback.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@
4343
import com.oracle.graal.python.pegparser.tokenizer.SourceRange;
4444

4545
@FunctionalInterface
46-
public interface ParserErrorCallback {
46+
public interface ErrorCallback {
4747
public enum ErrorType {
4848
Generic,
4949
Indentation,

‎graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/NodeFactory.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ default ExprTy createGetAttribute(ExprTy receiver, String name, SourceRange sour
104104

105105
public StmtTy createFor(ExprTy target, ExprTy iter, StmtTy[] block, StmtTy[] elseBlock, String typeComment, SourceRange sourceRange);
106106

107-
public ExprTy createString(String[] values, SourceRange[] sourceRanges, FExprParser exprParser, ParserErrorCallback errorCb);
107+
public ExprTy createString(String[] values, SourceRange[] sourceRanges, FExprParser exprParser, ErrorCallback errorCb);
108108

109109
public ExprTy createSubscript(ExprTy receiver, ExprTy slice, ExprContext context, SourceRange sourceRange);
110110

‎graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/NodeFactoryImp.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -251,7 +251,7 @@ public ExprTy createNumber(String numberWithUnderscores, SourceRange sourceRange
251251
}
252252

253253
@Override
254-
public ExprTy createString(String[] values, SourceRange[] sourceRanges, FExprParser exprParser, ParserErrorCallback errorCb) {
254+
public ExprTy createString(String[] values, SourceRange[] sourceRanges, FExprParser exprParser, ErrorCallback errorCb) {
255255
return StringLiteralUtils.createStringLiteral(values, sourceRanges, exprParser, errorCb);
256256
}
257257

‎graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/Parser.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -583,7 +583,7 @@ public final class Parser extends AbstractParser {
583583
public Parser(ParserTokenizer tokenizer, NodeFactory factory, FExprParser fexprParser) {
584584
super(tokenizer, factory, fexprParser);
585585
}
586-
public Parser(ParserTokenizer tokenizer, NodeFactory factory, FExprParser fexprParser, ParserErrorCallback errorCb) {
586+
public Parser(ParserTokenizer tokenizer, NodeFactory factory, FExprParser fexprParser, ErrorCallback errorCb) {
587587
super(tokenizer, factory, fexprParser, errorCb);
588588
}
589589

@@ -12328,7 +12328,7 @@ public ExprTy invalid_expression_rule()
1232812328
(expression_var = (ExprTy)expression_rule()) != null // expression
1232912329
)
1233012330
{
12331-
_res = this.raiseErrorKnownLocation(ParserErrorCallback.ErrorType.Syntax,a,"invalid syntax.Perhaps you forgot a comma?");
12331+
_res = this.raiseErrorKnownLocation(ErrorCallback.ErrorType.Syntax,a,"invalid syntax.Perhaps you forgot a comma?");
1233212332
cache.putResult(_mark, INVALID_EXPRESSION_ID, _res);
1233312333
return (ExprTy)_res;
1233412334
}
@@ -13815,7 +13815,7 @@ public ExprTy invalid_kvpair_rule()
1381513815
genLookahead__tmp_230_rule(false)
1381613816
)
1381713817
{
13818-
_res = this.raiseErrorKnownLocation(ParserErrorCallback.ErrorType.Syntax,a,"':' expected after dictionary key");
13818+
_res = this.raiseErrorKnownLocation(ErrorCallback.ErrorType.Syntax,a,"':' expected after dictionary key");
1381913819
cache.putResult(_mark, INVALID_KVPAIR_ID, _res);
1382013820
return (ExprTy)_res;
1382113821
}

‎graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/python.gram

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -908,4 +908,4 @@ invalid_kvpair:
908908
| a=expression !(':') {
909909
RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, a->lineno, a->end_col_offset - 1, "':' expected after dictionary key") }
910910
| expression ':' a='*' bitwise_or { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use a starred expression in a dictionary value") }
911-
| expression a=':' {RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "expression expected after dictionary key and ':'") }
911+
| expression a=':' {RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "expression expected after dictionary key and ':'") }

‎graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/scope/ScopeEnvironment.java

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
import java.util.Stack;
4949

5050
import com.oracle.graal.python.pegparser.ExprContext;
51+
import com.oracle.graal.python.pegparser.ErrorCallback;
5152
import com.oracle.graal.python.pegparser.scope.Scope.DefUse;
5253
import com.oracle.graal.python.pegparser.scope.Scope.ScopeFlags;
5354
import com.oracle.graal.python.pegparser.scope.Scope.ScopeType;
@@ -72,9 +73,15 @@
7273
public class ScopeEnvironment {
7374
final Scope topScope;
7475
final HashMap<SSTNode, Scope> blocks = new HashMap<>();
76+
private final ErrorCallback errorCallback;
7577

7678
public ScopeEnvironment(ModTy moduleNode) {
79+
this(moduleNode, null);
80+
}
81+
82+
public ScopeEnvironment(ModTy moduleNode, ErrorCallback errorCallback) {
7783
// First pass, similar to the entry point `symtable_enter_block' on CPython
84+
this.errorCallback = errorCallback;
7885
FirstPassVisitor visitor = new FirstPassVisitor(moduleNode, this);
7986
topScope = visitor.currentScope;
8087
moduleNode.accept(visitor);
@@ -432,7 +439,7 @@ public Void visit(AliasTy node) {
432439
}
433440
if ("*".equals(importedName)) {
434441
if (!currentScope.isModule()) {
435-
// TODO: syntax error: IMPORT_STAR not in module scope
442+
env.errorCallback.onError(ErrorCallback.ErrorType.Syntax, node.getSourceRange(), "import * only allowed at module level");
436443
}
437444
} else {
438445
addDef(importedName, DefUse.DefImport);

0 commit comments

Comments
 (0)