Cursor based lexer

Author: MichaReiser

ast/Cargo.toml

@@ -7,14 +7,10 @@ edition = "2021"
 repository = "https://github.com/RustPython/Parser/"
 license = "MIT"
 
-[features]
-default = ["malachite-bigint"]
-
 [dependencies]
 rustpython-parser-core = { workspace = true }
 rustpython-literal = { workspace = true, optional = true }
 
 is-macro = { workspace = true }
-num-bigint = { workspace = true, optional = true }
-malachite-bigint = { workspace = true, optional = true }
+num-bigint = { workspace = true }
 static_assertions = "1.1.0"

ast/src/builtin.rs

@@ -2,8 +2,8 @@
 
 use rustpython_parser_core::text_size::TextRange;
 
-use crate::bigint::BigInt;
 use crate::Ranged;
+use num_bigint::BigInt;
 
 pub type String = std::string::String;
 

ast/src/lib.rs

@@ -20,11 +20,6 @@ mod generic;
 mod impls;
 mod ranged;
 
-#[cfg(feature = "malachite-bigint")]
-pub use malachite_bigint as bigint;
-#[cfg(all(feature = "num-bigint", not(feature = "malachite-bigint")))]
-pub use num_bigint as bigint;
-
 pub use builtin::*;
 pub use generic::*;
 pub use ranged::Ranged;

format/Cargo.toml

@@ -13,8 +13,6 @@ rustpython-literal = { workspace = true }
 bitflags = "2.3.1"
 itertools = "0.10.5"
 num-traits = { workspace = true }
-num-bigint = { workspace = true, optional = true }
-malachite-bigint = { workspace = true, optional = true }
+num-bigint = { workspace = true }
 
 [features]
-default = ["malachite-bigint"]

format/src/cformat.rs

@@ -9,7 +9,7 @@ use std::{
     str::FromStr,
 };
 
-use crate::bigint::{BigInt, Sign};
+use num_bigint::{BigInt, Sign};
 
 #[derive(Debug, PartialEq)]
 pub enum CFormatErrorType {

format/src/format.rs

@@ -6,7 +6,7 @@ use rustpython_literal::format::Case;
 use std::ops::Deref;
 use std::{cmp, str::FromStr};
 
-use crate::bigint::{BigInt, Sign};
+use num_bigint::{BigInt, Sign};
 
 trait FormatParse {
     fn parse(text: &str) -> (Option<Self>, &str)

format/src/lib.rs

@@ -1,8 +1,3 @@
-#[cfg(feature = "malachite-bigint")]
-pub use malachite_bigint as bigint;
-#[cfg(all(feature = "num-bigint", not(feature = "malachite-bigint")))]
-pub use num_bigint as bigint;
-
 pub use crate::format::*;
 
 pub mod cformat;

literal/src/escape.rs

@@ -385,7 +385,7 @@ impl<'a> Escape for AsciiEscape<'a> {
     fn layout(&self) -> &EscapeLayout {
         &self.layout
     }
-
+    #[allow(unsafe_code)]
     fn write_source(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result {
         formatter.write_str(unsafe {
             // SAFETY: this function must be called only when source is printable ascii characters

parser/Cargo.toml

@@ -9,11 +9,7 @@ license = "MIT"
 edition = "2021"
 
 [features]
-default = ["malachite-bigint"]
 serde = ["dep:serde", "rustpython-parser-core/serde"]
-full-lexer = []
-malachite-bigint = ["dep:malachite-bigint", "rustpython-ast/malachite-bigint"]
-num-bigint = ["dep:num-bigint", "rustpython-ast/num-bigint"]
 
 [build-dependencies]
 anyhow = { workspace = true }
@@ -28,17 +24,16 @@ rustpython-parser-core = { workspace = true }
 itertools = { workspace = true }
 is-macro = { workspace = true }
 log = { workspace = true }
-malachite-bigint = { workspace = true, optional = true }
-num-bigint = { workspace = true, optional = true }
+num-bigint = { workspace = true }
 num-traits = { workspace = true }
 unicode_names2 = { workspace = true }
 
 unic-emoji-char = "0.9.0"
 unic-ucd-ident  = "0.9.0"
 lalrpop-util = { version = "0.20.0", default-features = false }
-phf = "0.11.1"
 rustc-hash = "1.1.0"
 serde = { version = "1.0.133", optional = true, default-features = false, features = ["derive"] }
+static_assertions = "1.1.0"
 
 [dev-dependencies]
 insta = { workspace = true }

parser/build.rs

@@ -1,13 +1,10 @@
 use std::fmt::Write as _;
 use std::fs::File;
-use std::io::{BufRead, BufReader, BufWriter, Write};
+use std::io::{BufRead, BufReader};
 use std::path::{Path, PathBuf};
 use tiny_keccak::{Hasher, Sha3};
 
 fn main() -> anyhow::Result<()> {
-    let out_dir = PathBuf::from(std::env::var_os("OUT_DIR").unwrap());
-    gen_phf(&out_dir);
-
     const SOURCE: &str = "src/python.lalrpop";
     println!("cargo:rerun-if-changed={SOURCE}");
 
@@ -16,6 +13,7 @@ fn main() -> anyhow::Result<()> {
 
     #[cfg(feature = "lalrpop")]
     {
+        let out_dir = PathBuf::from(std::env::var_os("OUT_DIR").unwrap());
         target = out_dir.join("src/python.rs");
     }
     #[cfg(not(feature = "lalrpop"))]
@@ -113,55 +111,3 @@ fn sha_equal(expected_sha3_str: &str, actual_sha3: &[u8; 32]) -> bool {
     }
     *actual_sha3 == expected_sha3
 }
-
-fn gen_phf(out_dir: &Path) {
-    let mut kwds = phf_codegen::Map::new();
-    let kwds = kwds
-        // Alphabetical keywords:
-        .entry("...", "Tok::Ellipsis")
-        .entry("False", "Tok::False")
-        .entry("None", "Tok::None")
-        .entry("True", "Tok::True")
-        // more so "standard" keywords
-        .entry("and", "Tok::And")
-        .entry("as", "Tok::As")
-        .entry("assert", "Tok::Assert")
-        .entry("async", "Tok::Async")
-        .entry("await", "Tok::Await")
-        .entry("break", "Tok::Break")
-        .entry("case", "Tok::Case")
-        .entry("class", "Tok::Class")
-        .entry("continue", "Tok::Continue")
-        .entry("def", "Tok::Def")
-        .entry("del", "Tok::Del")
-        .entry("elif", "Tok::Elif")
-        .entry("else", "Tok::Else")
-        .entry("except", "Tok::Except")
-        .entry("finally", "Tok::Finally")
-        .entry("for", "Tok::For")
-        .entry("from", "Tok::From")
-        .entry("global", "Tok::Global")
-        .entry("if", "Tok::If")
-        .entry("import", "Tok::Import")
-        .entry("in", "Tok::In")
-        .entry("is", "Tok::Is")
-        .entry("lambda", "Tok::Lambda")
-        .entry("match", "Tok::Match")
-        .entry("nonlocal", "Tok::Nonlocal")
-        .entry("not", "Tok::Not")
-        .entry("or", "Tok::Or")
-        .entry("pass", "Tok::Pass")
-        .entry("raise", "Tok::Raise")
-        .entry("return", "Tok::Return")
-        .entry("try", "Tok::Try")
-        .entry("type", "Tok::Type")
-        .entry("while", "Tok::While")
-        .entry("with", "Tok::With")
-        .entry("yield", "Tok::Yield")
-        .build();
-    writeln!(
-        BufWriter::new(File::create(out_dir.join("keywords.rs")).unwrap()),
-        "{kwds}",
-    )
-    .unwrap();
-}