Interactive shell implementation (#2617)

* Initial interactive shell implementation

* remember global functions and variables declared

* avoid printing AST, ASR and LLVM IR

* fixed bug where error from previous cell is printed

* detect decorators as incomplete input

* clean up

* update according to code review suggestion

* removed Interactive_t related stuff by merging required changes into SymbolTable.visit_Module

* update according to code review suggestion

Author: Vipul-Cariappa

.gitignore

@@ -234,3 +234,6 @@ integration_tests/array_02_decl
 integration_tests/array_02_decl.c
 integration_tests/expr_12
 integration_tests/expr_12.c
+
+# Interactive Shell
+/input

src/bin/lpython.cpp

@@ -33,6 +33,7 @@
 #include <lpython/python_serialization.h>
 #include <lpython/parser/tokenizer.h>
 #include <lpython/parser/parser.h>
+#include <libasr/exception.h>
 
 #include <cpp-terminal/terminal.h>
 #include <cpp-terminal/prompt0.h>
@@ -743,6 +744,11 @@ void print_time_report(std::vector<std::pair<std::string, double>> &times, bool
 
 #ifdef HAVE_LFORTRAN_LLVM
 
+void section(const std::string &s)
+{
+    std::cout << color(LCompilers::style::bold) << color(LCompilers::fg::blue) << s << color(LCompilers::style::reset) << color(LCompilers::fg::reset) << std::endl;
+}
+
 int emit_llvm(const std::string &infile,
     const std::string &runtime_library_dir,
     LCompilers::PassManager& pass_manager,
@@ -792,6 +798,157 @@ int emit_llvm(const std::string &infile,
     return 0;
 }
 
+int interactive_python_repl(
+        LCompilers::PassManager& pass_manager,
+        CompilerOptions &compiler_options,
+        bool verbose)
+{
+    Allocator al(4*1024);
+    compiler_options.interactive = true;
+    LCompilers::PythonCompiler fe(compiler_options);
+    LCompilers::diag::Diagnostics diagnostics;
+    LCompilers::LocationManager lm;
+    std::vector<std::pair<std::string, double>> times;
+    LCompilers::PythonCompiler::EvalResult r;
+
+    std::string code_string;
+    std::cout << ">>> ";
+    size_t cell_count = 0;
+    for (std::string input; std::getline(std::cin, input);) {
+        if (input == "exit" || input == "quit") {
+            return 0;
+        }
+
+        if ((input.rfind("def", 0) == 0) ||
+            (input.rfind("for", 0) == 0) ||
+            (input.rfind("if", 0) == 0) ||
+            (input.rfind("else", 0) == 0) ||
+            (input.rfind("elif", 0) == 0) ||
+            (input.rfind("class", 0) == 0) ||
+            (input.rfind('@', 0) == 0) ||
+            (input.rfind(' ', 0) == 0) ||
+            (input.rfind('\t', 0) == 0)) {
+            // start of a block
+            code_string += input + "\n";
+            std::cout << "... ";
+            continue;
+        }
+        code_string += input + "\n";
+
+        {
+            cell_count++;
+            LCompilers::LocationManager::FileLocations fl;
+            fl.in_filename = "input";
+            std::ofstream out("input");
+            out << code_string;
+            lm.files.push_back(fl);
+            lm.init_simple(code_string);
+            lm.file_ends.push_back(code_string.size());
+        }
+
+        try {
+            auto evaluation_start_time = std::chrono::high_resolution_clock::now();
+            LCompilers::Result<LCompilers::PythonCompiler::EvalResult>
+            res = fe.evaluate(code_string, verbose, lm, pass_manager, diagnostics);
+            if (res.ok) {
+                r = res.result;
+            } else {
+                LCOMPILERS_ASSERT(diagnostics.has_error())
+                std::cerr << diagnostics.render(lm, compiler_options);
+                diagnostics.clear();
+                code_string = "";
+                std::cout << ">>> ";
+                continue;
+            }
+
+            auto evaluation_end_time = std::chrono::high_resolution_clock::now();
+            times.push_back(std::make_pair("evalution " + std::to_string(cell_count), std::chrono::duration
+                <double, std::milli>(evaluation_start_time - evaluation_end_time).count()));
+
+        } catch (const LCompilers::LCompilersException &e) {
+            std::cerr << "Internal Compiler Error: Unhandled exception" << std::endl;
+            std::vector<LCompilers::StacktraceItem> d = e.stacktrace_addresses();
+            get_local_addresses(d);
+            get_local_info(d);
+            std::cerr << stacktrace2str(d, LCompilers::stacktrace_depth);
+            std::cerr << e.name() + ": " << e.msg() << std::endl;
+
+            code_string = "";
+            std::cout << ">>> ";
+            continue;
+        }
+
+        if (verbose) {
+            section("AST:");
+            std::cout << r.ast << std::endl;
+            section("ASR:");
+            std::cout << r.asr << std::endl;
+            section("LLVM IR:");
+            std::cout << r.llvm_ir << std::endl;
+        }
+
+        switch (r.type) {
+            case (LCompilers::PythonCompiler::EvalResult::integer4) : {
+                if (verbose) std::cout << "Return type: integer" << std::endl;
+                if (verbose) section("Result:");
+                std::cout << r.i32 << std::endl;
+                break;
+            }
+            case (LCompilers::PythonCompiler::EvalResult::integer8) : {
+                if (verbose) std::cout << "Return type: integer(8)" << std::endl;
+                if (verbose) section("Result:");
+                std::cout << r.i64 << std::endl;
+                break;
+            }
+            case (LCompilers::PythonCompiler::EvalResult::real4) : {
+                if (verbose) std::cout << "Return type: real" << std::endl;
+                if (verbose) section("Result:");
+                std::cout << std::setprecision(8) << r.f32 << std::endl;
+                break;
+            }
+            case (LCompilers::PythonCompiler::EvalResult::real8) : {
+                if (verbose) std::cout << "Return type: real(8)" << std::endl;
+                if (verbose) section("Result:");
+                std::cout << std::setprecision(17) << r.f64 << std::endl;
+                break;
+            }
+            case (LCompilers::PythonCompiler::EvalResult::complex4) : {
+                if (verbose) std::cout << "Return type: complex" << std::endl;
+                if (verbose) section("Result:");
+                std::cout << std::setprecision(8) << "(" << r.c32.re << ", " << r.c32.im << ")" << std::endl;
+                break;
+            }
+            case (LCompilers::PythonCompiler::EvalResult::complex8) : {
+                if (verbose) std::cout << "Return type: complex(8)" << std::endl;
+                if (verbose) section("Result:");
+                std::cout << std::setprecision(17) << "(" << r.c64.re << ", " << r.c64.im << ")" << std::endl;
+                break;
+            }
+            case (LCompilers::PythonCompiler::EvalResult::statement) : {
+                if (verbose) {
+                    std::cout << "Return type: none" << std::endl;
+                    section("Result:");
+                    std::cout << "(statement)" << std::endl;
+                }
+                break;
+            }
+            case (LCompilers::PythonCompiler::EvalResult::none) : {
+                if (verbose) {
+                    std::cout << "Return type: none" << std::endl;
+                    section("Result:");
+                    std::cout << "(nothing to execute)" << std::endl;
+                }
+                break;
+            }
+            default : throw LCompilers::LCompilersException("Return type not supported");
+        }
+
+        code_string = "";
+        std::cout << ">>> ";
+    }
+    return 0;
+}
+
 /*
     Compiles python to object file, if `to_jit` is false
     otherwise execute python code using llvm JIT
@@ -1824,8 +1981,17 @@ int main(int argc, char *argv[])
         }
 
         if (arg_files.size() == 0) {
-            std::cerr << "Interactive prompt is not implemented yet in LPython" << std::endl;
+#ifdef HAVE_LFORTRAN_LLVM
+            lpython_pass_manager.parse_pass_arg(arg_pass, skip_pass);
+            lpython_pass_manager.use_default_passes();
+            compiler_options.po.disable_main = true;
+            compiler_options.emit_debug_line_column = false;
+            compiler_options.generate_object_code = false;
+            return interactive_python_repl(lpython_pass_manager, compiler_options, arg_v);
+#else
+            std::cerr << "Interactive prompt requires the LLVM backend to be enabled. Recompile with `WITH_LLVM=yes`." << std::endl;
             return 1;
+#endif
         }
 
         // TODO: for now we ignore the other filenames, only handle

src/libasr/diagnostics.h

@@ -127,6 +127,10 @@ struct Diagnostics {
         diagnostics.push_back(d);
     }
 
+    void clear() {
+        diagnostics.clear();
+    }
+
     void message_label(const std::string &message,
             const std::vector<Location> &locations,
             const std::string &error_label,

src/lpython/parser/parser.h

@@ -1,6 +1,7 @@
 #ifndef LPYTHON_PARSER_PARSER_H
 #define LPYTHON_PARSER_PARSER_H
 
+#include "lpython/python_ast.h"
 #include <libasr/containers.h>
 #include <libasr/diagnostics.h>
 #include <lpython/parser/tokenizer.h>

src/lpython/python_evaluator.cpp

@@ -1,10 +1,16 @@
 #include <iostream>
 #include <fstream>
+#include <string>
 
 #include <lpython/python_evaluator.h>
+#include <lpython/semantics/python_ast_to_asr.h>
+#include <lpython/python_ast.h>
+#include <lpython/pickle.h>
+#include <lpython/parser/parser.h>
 #include <libasr/codegen/asr_to_cpp.h>
 #include <libasr/exception.h>
 #include <libasr/asr.h>
+#include <libasr/asr_scopes.h>
 
 #ifdef HAVE_LFORTRAN_LLVM
 #include <libasr/codegen/evaluator.h>
@@ -26,16 +32,138 @@ PythonCompiler::PythonCompiler(CompilerOptions compiler_options)
     al{1024*1024},
 #ifdef HAVE_LFORTRAN_LLVM
     e{std::make_unique<LLVMEvaluator>()},
-    eval_count{0},
 #endif
-    compiler_options{compiler_options}
-//    symbol_table{nullptr}
+    eval_count{1},
+    compiler_options{compiler_options},
+    symbol_table{nullptr}
 {
 }
 
 PythonCompiler::~PythonCompiler() = default;
 
 
+Result<PythonCompiler::EvalResult> PythonCompiler::evaluate(
+#ifdef HAVE_LFORTRAN_LLVM
+            const std::string &code_orig, bool verbose, LocationManager &lm,
+            LCompilers::PassManager& pass_manager, diag::Diagnostics &diagnostics
+#else
+            const std::string &/*code_orig*/, bool /*verbose*/,
+                LocationManager &/*lm*/, LCompilers::PassManager& /*pass_manager*/,
+                diag::Diagnostics &/*diagnostics*/
+#endif
+            )
+{
+#ifdef HAVE_LFORTRAN_LLVM
+    EvalResult result;
+    result.type = EvalResult::none;
+
+    // Src -> AST
+    Result<LCompilers::LPython::AST::ast_t*> res = get_ast2(code_orig, diagnostics);
+    LCompilers::LPython::AST::ast_t* ast;
+    if (res.ok) {
+        ast = res.result;
+    } else {
+        return res.error;
+    }
+
+    if (verbose) {
+        result.ast = LCompilers::LPython::pickle_python(*ast, true, true);
+    }
+
+    // AST -> ASR
+    Result<ASR::TranslationUnit_t*> res2 = get_asr3(*ast, diagnostics, lm, true);
+    ASR::TranslationUnit_t* asr;
+    if (res2.ok) {
+        asr = res2.result;
+    } else {
+        LCOMPILERS_ASSERT(diagnostics.has_error())
+        return res2.error;
+    }
+
+    if (verbose) {
+        result.asr = pickle(*asr, true, true, true);
+    }
+
+    // ASR -> LLVM
+    std::string module_prefix = "__module___main___";
+    std::string module_name = "__main__";
+    std::string sym_name = module_name + "global_stmts_" + std::to_string(eval_count) + "__";
+    run_fn = module_prefix + sym_name;
+
+    Result<std::unique_ptr<LLVMModule>> res3 = get_llvm3(*asr,
+        pass_manager, diagnostics, lm.files.back().in_filename);
+    std::unique_ptr<LCompilers::LLVMModule> m;
+    if (res3.ok) {
+        m = std::move(res3.result);
+    } else {
+        LCOMPILERS_ASSERT(diagnostics.has_error())
+        return res3.error;
+    }
+
+    if (verbose) {
+        result.llvm_ir = m->str();
+    }
+
+    bool call_run_fn = false;
+    if (m->get_return_type(run_fn) != "none") {
+        call_run_fn = true;
+    }
+
+    e->add_module(std::move(m));
+    if (call_run_fn) {
+        e->voidfn(run_fn);
+    }
+
+    if (call_run_fn) {
+        ASR::down_cast<ASR::Module_t>(symbol_table->resolve_symbol(module_name))->m_symtab
+            ->erase_symbol(sym_name);
+    }
+
+    eval_count++;
+    return result;
+#else
+    throw LCompilersException("LLVM is not enabled");
+#endif
+}
+
+Result<LCompilers::LPython::AST::ast_t*> PythonCompiler::get_ast2(
+            const std::string &code_orig, diag::Diagnostics &diagnostics)
+{
+    // Src -> AST
+    const std::string *code=&code_orig;
+    std::string tmp;
+    Result<LCompilers::LPython::AST::Module_t*>
+        res = LCompilers::LPython::parse(al, *code, 0, diagnostics);
+    if (res.ok) {
+        return (LCompilers::LPython::AST::ast_t*)res.result;
+    } else {
+        LCOMPILERS_ASSERT(diagnostics.has_error())
+        return res.error;
+    }
+}
+
+Result<ASR::TranslationUnit_t*> PythonCompiler::get_asr3(
+    LCompilers::LPython::AST::ast_t &ast, diag::Diagnostics &diagnostics,
+    LocationManager &lm, bool is_interactive)
+{
+    ASR::TranslationUnit_t* asr;
+    // AST -> ASR
+    if (symbol_table) {
+        symbol_table->mark_all_variables_external(al);
+    }
+    auto res = LCompilers::LPython::python_ast_to_asr(al, lm, symbol_table, ast, diagnostics,
+        compiler_options, true, "__main__", "", false, is_interactive ? eval_count : 0);
+    if (res.ok) {
+        asr = res.result;
+    } else {
+        LCOMPILERS_ASSERT(diagnostics.has_error())
+        return res.error;
+    }
+    if (!symbol_table) symbol_table = asr->m_symtab;
+
+    return asr;
+}
+
 Result<std::unique_ptr<LLVMModule>> PythonCompiler::get_llvm3(
 #ifdef HAVE_LFORTRAN_LLVM
     ASR::TranslationUnit_t &asr, LCompilers::PassManager& lpm,
@@ -47,9 +175,6 @@ Result<std::unique_ptr<LLVMModule>> PythonCompiler::get_llvm3(
     )
 {
 #ifdef HAVE_LFORTRAN_LLVM
-    eval_count++;
-    run_fn = "__lfortran_evaluate_" + std::to_string(eval_count);
-
     if (compiler_options.emit_debug_info) {
         if (!compiler_options.emit_debug_line_column) {
             diagnostics.add(LCompilers::diag::Diagnostic(