Merge with magic parsing

Author: MichaReiser

ast/src/generic.rs

@@ -1,5 +1,5 @@
 #![allow(clippy::derive_partial_eq_without_eq)]
-use crate::text_size::TextRange;
+use crate::text_size::{TextRange, TextSize};
 pub(crate) use crate::{builtin::*, ConversionFlag, Node};
 use std::fmt::{self, Debug};
 

parser/src/lexer.rs

@@ -573,8 +573,12 @@ impl<'source> Lexer<'source> {
                 self.consume_ascii_character(c)
             } else if is_unicode_identifier_start(c) {
                 let identifier = self.lex_identifier(c)?;
+                self.state = State::Other;
+
                 Ok((identifier, self.token_range()))
             } else if is_emoji_presentation(c) {
+                self.state = State::Other;
+
                 Ok((
                     Tok::Name {
                         name: c.to_string(),
@@ -1426,9 +1430,8 @@ baz = %matplotlib \
 
     fn assert_no_jupyter_magic(tokens: &[Tok]) {
         for tok in tokens {
-            match tok {
-                Tok::MagicCommand { .. } => panic!("Unexpected magic command token: {:?}", tok),
-                _ => {}
+            if let Tok::MagicCommand { .. } = tok {
+                panic!("Unexpected magic command token: {:?}", tok)
             }
         }
     }

parser/src/parser.rs

@@ -396,41 +396,11 @@ pub fn parse_tokens(
 ) -> Result<ast::Mod, ParseError> {
     let lxr = lxr.into_iter();
 
-<<<<<<< HEAD
-    let lxr =
-        lxr.filter_ok(|(tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline));
-    parse_filtered_tokens(lxr, mode, source_path)
-=======
-    match mode {
-        Mode::Module | Mode::Interactive | Mode::Expression => parse_filtered_tokens(
-            lxr.filter_ok(|(tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline)),
-            mode,
-            source_path,
-        ),
-        Mode::Jupyter => {
-            let mut after_magic = false;
-            parse_filtered_tokens(
-                lxr.filter_ok(|(tok, _)| match tok {
-                    Tok::Comment(..) | Tok::NonLogicalNewline => {
-                        after_magic = false;
-                        false
-                    }
-                    Tok::Newline => !after_magic,
-                    Tok::MagicCommand { .. } => {
-                        after_magic = true;
-                        false
-                    }
-                    _ => {
-                        after_magic = false;
-                        true
-                    }
-                }),
-                mode,
-                source_path,
-            )
-        }
-    }
->>>>>>> 58ac178 (Use single filter call)
+    parse_filtered_tokens(
+        lxr.filter_ok(|(tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline)),
+        mode,
+        source_path,
+    )
 }
 
 fn parse_filtered_tokens(