Merge pull request #95 from benesch/hex-literals

Support hexadecimal string literals

Author: benesch

src/sqlast/value.rs

@@ -11,6 +11,8 @@ pub enum Value {
     SingleQuotedString(String),
     /// N'string value'
     NationalStringLiteral(String),
+    /// X'hex value'
+    HexStringLiteral(String),
     /// Boolean value true or false
     Boolean(bool),
     /// NULL value in insert statements,
@@ -24,6 +26,7 @@ impl ToString for Value {
             Value::Double(v) => v.to_string(),
             Value::SingleQuotedString(v) => format!("'{}'", escape_single_quote_string(v)),
             Value::NationalStringLiteral(v) => format!("N'{}'", v),
+            Value::HexStringLiteral(v) => format!("X'{}'", v),
             Value::Boolean(v) => v.to_string(),
             Value::Null => "NULL".to_string(),
         }

src/sqlparser.rs

@@ -239,7 +239,10 @@ impl Parser {
                     expr: Box::new(self.parse_subexpr(Self::PLUS_MINUS_PREC)?),
                 })
             }
-            Token::Number(_) | Token::SingleQuotedString(_) | Token::NationalStringLiteral(_) => {
+            Token::Number(_)
+            | Token::SingleQuotedString(_)
+            | Token::NationalStringLiteral(_)
+            | Token::HexStringLiteral(_) => {
                 self.prev_token();
                 self.parse_sql_value()
             }
@@ -1042,6 +1045,7 @@ impl Parser {
                 Token::NationalStringLiteral(ref s) => {
                     Ok(Value::NationalStringLiteral(s.to_string()))
                 }
+                Token::HexStringLiteral(ref s) => Ok(Value::HexStringLiteral(s.to_string())),
                 _ => parser_err!(format!("Unsupported value: {:?}", t)),
             },
             None => parser_err!("Expecting a value, but found EOF"),

src/sqltokenizer.rs

@@ -37,6 +37,8 @@ pub enum Token {
     SingleQuotedString(String),
     /// "National" string literal: i.e: N'string'
     NationalStringLiteral(String),
+    /// Hexadecimal string literal: i.e.: X'deadbeef'
+    HexStringLiteral(String),
     /// Comma
     Comma,
     /// Whitespace (space, tab, etc)
@@ -97,6 +99,7 @@ impl ToString for Token {
             Token::Char(ref c) => c.to_string(),
             Token::SingleQuotedString(ref s) => format!("'{}'", s),
             Token::NationalStringLiteral(ref s) => format!("N'{}'", s),
+            Token::HexStringLiteral(ref s) => format!("X'{}'", s),
             Token::Comma => ",".to_string(),
             Token::Whitespace(ws) => ws.to_string(),
             Token::Eq => "=".to_string(),
@@ -286,6 +289,23 @@ impl<'a> Tokenizer<'a> {
                         }
                     }
                 }
+                // The spec only allows an uppercase 'X' to introduce a hex
+                // string, but PostgreSQL, at least, allows a lowercase 'x' too.
+                x @ 'x' | x @ 'X' => {
+                    chars.next(); // consume, to check the next char
+                    match chars.peek() {
+                        Some('\'') => {
+                            // X'...' - a <binary string literal>
+                            let s = self.tokenize_single_quoted_string(chars);
+                            Ok(Some(Token::HexStringLiteral(s)))
+                        }
+                        _ => {
+                            // regular identifier starting with an "X"
+                            let s = self.tokenize_word(x, chars);
+                            Ok(Some(Token::make_word(&s, None)))
+                        }
+                    }
+                }
                 // identifier or keyword
                 ch if self.dialect.is_identifier_start(ch) => {
                     chars.next(); // consume the first char

tests/sqlparser_common.rs

@@ -952,9 +952,9 @@ fn parse_aggregate_with_group_by() {
 
 #[test]
 fn parse_literal_string() {
-    let sql = "SELECT 'one', N'national string'";
+    let sql = "SELECT 'one', N'national string', X'deadBEEF'";
     let select = verified_only_select(sql);
-    assert_eq!(2, select.projection.len());
+    assert_eq!(3, select.projection.len());
     assert_eq!(
         &ASTNode::SQLValue(Value::SingleQuotedString("one".to_string())),
         expr_from_projection(&select.projection[0])
@@ -963,6 +963,12 @@ fn parse_literal_string() {
         &ASTNode::SQLValue(Value::NationalStringLiteral("national string".to_string())),
         expr_from_projection(&select.projection[1])
     );
+    assert_eq!(
+        &ASTNode::SQLValue(Value::HexStringLiteral("deadBEEF".to_string())),
+        expr_from_projection(&select.projection[2])
+    );
+
+    one_statement_parses_to("SELECT x'deadBEEF'", "SELECT X'deadBEEF'");
 }
 
 #[test]