Implement Fluent 0.5 Private Messages

fluent/migrate/transforms.py

@@ -245,7 +245,7 @@ def createVariant(zipped_enum):
             # variant.  Then evaluate it to a migrated FTL node.
             value = evaluate(ctx, self.foreach(variant))
             return FTL.Variant(
-                key=FTL.Symbol(key),
+                key=FTL.VariantName(key),
                 value=value,
                 default=index == last_index
             )

fluent/syntax/ast.py

@@ -261,9 +261,9 @@ def __init__(self, name, **kwargs):
         super(Identifier, self).__init__(**kwargs)
         self.name = name
 
-class Symbol(Identifier):
+class VariantName(Identifier):
     def __init__(self, name, **kwargs):
-        super(Symbol, self).__init__(name, **kwargs)
+        super(VariantName, self).__init__(name, **kwargs)
 
 
 class BaseComment(Entry):

fluent/syntax/errors.py

@@ -25,9 +25,9 @@ def get_error_message(code, args):
     if code == 'E0007':
         return 'Keyword cannot end with a whitespace'
     if code == 'E0008':
-        return 'Callee has to be a simple identifier'
+        return 'The callee has to be a simple, upper-case identifier'
     if code == 'E0009':
-        return 'Key has to be a simple identifier'
+        return 'The key has to be a simple identifier'
     if code == 'E0010':
         return 'Expected one of the variants to be marked as default (*)'
     if code == 'E0011':
@@ -38,4 +38,12 @@ def get_error_message(code, args):
         return 'Expected literal'
     if code == 'E0015':
         return 'Only one variant can be marked as default (*)'
+    if code == 'E0016':
+        return 'Message references cannot be used as selectors'
+    if code == 'E0017':
+        return 'Variants cannot be used as selectors'
+    if code == 'E0018':
+        return 'Attributes of public messages cannot be used as selectors'
+    if code == 'E0019':
+        return 'Attributes of private messages cannot be used as placeables'
     return code

fluent/syntax/ftlstream.py

@@ -73,20 +73,31 @@ def take_char(self, f):
             return ch
         return None
 
-    def is_id_start(self):
-        if self.ch is None:
+    def is_char_id_start(self, ch=None):
+        if ch is None:
             return False
 
-        cc = ord(self.ch)
-
+        cc = ord(ch)
         return (cc >= 97 and cc <= 122) or \
-               (cc >= 65 and cc <= 90) or \
-            cc == 95
+               (cc >= 65 and cc <= 90)
+
+    def is_message_id_start(self):
+        if self.current_is('-'):
+            self.peek()
+
+        ch = self.current_peek()
+        is_id = self.is_char_id_start(ch)
+        self.reset_peek()
+        return is_id
 
     def is_number_start(self):
-        cc = ord(self.ch)
+        if self.current_is('-'):
+            self.peek()
 
-        return (cc >= 48 and cc <= 57) or cc == 45
+        cc = ord(self.current_peek())
+        is_digit = cc >= 48 and cc <= 57
+        self.reset_peek()
+        return is_digit
 
     def is_peek_next_line_zero_four_style_comment(self):
         if not self.current_peek_is('\n'):
@@ -211,19 +222,24 @@ def skip_to_next_entry_start(self):
             if self.current_is('\n') and not self.peek_char_is('\n'):
                 self.next()
 
-                if self.ch is None or self.is_id_start() or \
+                if self.ch is None or self.is_message_id_start() or \
                    (self.current_is('/') and self.peek_char_is('/')) or \
                    (self.current_is('[') and self.peek_char_is('[')):
                     break
             self.next()
 
-    def take_id_start(self):
-        if self.is_id_start():
+    def take_id_start(self, allow_private):
+        if allow_private and self.current_is('-'):
+            self.next()
+            return '-'
+
+        if self.is_char_id_start(self.ch):
             ret = self.ch
             self.next()
             return ret
 
-        raise ParseError('E0004', 'a-zA-Z_')
+        allowed_range = 'a-zA-Z-' if allow_private else 'a-zA-Z'
+        raise ParseError('E0004', allowed_range)
 
     def take_id_char(self):
         def closure(ch):
@@ -234,7 +250,7 @@ def closure(ch):
                     cc == 95 or cc == 45)
         return self.take_char(closure)
 
-    def take_symb_char(self):
+    def take_variant_name_char(self):
         def closure(ch):
             if ch is None:
                 return False

fluent/syntax/parser.py

@@ -105,7 +105,7 @@ def get_entry(self, ps):
                 return ast.GroupComment(comment.content)
             return None
 
-        if ps.is_id_start() \
+        if ps.is_message_id_start() \
            and (comment is None or isinstance(comment, ast.Comment)):
             return self.get_message(ps, comment)
 
@@ -187,7 +187,7 @@ def skip_section(self, ps):
 
         ps.skip_inline_ws()
 
-        self.get_symbol(ps)
+        self.get_variant_name(ps)
 
         ps.skip_inline_ws()
 
@@ -199,7 +199,7 @@ def skip_section(self, ps):
 
     @with_span
     def get_message(self, ps, comment):
-        id = self.get_identifier(ps)
+        id = self.get_private_identifier(ps)
 
         ps.skip_inline_ws()
 
@@ -225,7 +225,7 @@ def get_message(self, ps, comment):
     def get_attribute(self, ps):
         ps.expect_char('.')
 
-        key = self.get_identifier(ps)
+        key = self.get_public_identifier(ps)
 
         ps.skip_inline_ws()
         ps.expect_char('=')
@@ -252,10 +252,17 @@ def get_attributes(self, ps):
         return attrs
 
     @with_span
-    def get_identifier(self, ps):
+    def get_private_identifier(self, ps):
+        return self.get_identifier(ps, True)
+
+    @with_span
+    def get_public_identifier(self, ps):
+        return self.get_identifier(ps, False)
+
+    def get_identifier(self, ps, allow_private):
         name = ''
 
-        name += ps.take_id_start()
+        name += ps.take_id_start(allow_private)
 
         ch = ps.take_id_char()
         while ch:
@@ -270,10 +277,11 @@ def get_variant_key(self, ps):
         if ch is None:
             raise ParseError('E0013')
 
-        if ps.is_number_start():
+        cc = ord(ch)
+        if ((cc >= 48 and cc <= 57) or cc == 45):  # 0-9, -
             return self.get_number(ps)
 
-        return self.get_symbol(ps)
+        return self.get_variant_name(ps)
 
     @with_span
     def get_variant(self, ps, has_default):
@@ -323,19 +331,19 @@ def get_variants(self, ps):
         return variants
 
     @with_span
-    def get_symbol(self, ps):
+    def get_variant_name(self, ps):
         name = ''
 
-        name += ps.take_id_start()
+        name += ps.take_id_start(False)
 
         while True:
-            ch = ps.take_symb_char()
+            ch = ps.take_variant_name_char()
             if ch:
                 name += ch
             else:
                 break
 
-        return ast.Symbol(name.rstrip())
+        return ast.VariantName(name.rstrip())
 
     def get_digits(self, ps):
         num = ''
@@ -455,22 +463,37 @@ def get_expression(self, ps):
 
         if ps.current_is('-'):
             ps.peek()
+
             if not ps.current_peek_is('>'):
                 ps.reset_peek()
-            else:
-                ps.next()
-                ps.next()
+                return selector
 
-                ps.skip_inline_ws()
+            if isinstance(selector, ast.MessageReference):
+                raise ParseError('E0016')
 
-                variants = self.get_variants(ps)
+            if isinstance(selector, ast.AttributeExpression) and \
+               not selector.id.name.startswith('-'):
+                raise ParseError('E0018')
 
-                if len(variants) == 0:
-                    raise ParseError('E0011')
+            if isinstance(selector, ast.VariantExpression):
+                raise ParseError('E0017')
 
-                ps.expect_indent()
+            ps.next()
+            ps.next()
 
-                return ast.SelectExpression(selector, variants)
+            ps.skip_inline_ws()
+
+            variants = self.get_variants(ps)
+
+            if len(variants) == 0:
+                raise ParseError('E0011')
+
+            ps.expect_indent()
+
+            return ast.SelectExpression(selector, variants)
+        elif isinstance(selector, ast.AttributeExpression) and \
+                selector.id.name.startswith('-'):
+            raise ParseError('E0019')
 
         return selector
 
@@ -485,7 +508,7 @@ def get_selector_expression(self, ps):
 
         if (ch == '.'):
             ps.next()
-            attr = self.get_identifier(ps)
+            attr = self.get_public_identifier(ps)
             return ast.AttributeExpression(literal.id, attr)
 
         if (ch == '['):
@@ -501,7 +524,7 @@ def get_selector_expression(self, ps):
 
             ps.expect_char(')')
 
-            if not re.match('^[A-Z_-]+$', literal.id.name):
+            if not re.match('^[A-Z][A-Z_?-]*$', literal.id.name):
                 raise ParseError('E0008')
 
             return ast.CallExpression(
@@ -582,14 +605,16 @@ def get_literal(self, ps):
         if ch is None:
             raise ParseError('E0014')
 
-        if ps.is_number_start():
+        if ch == '$':
+            ps.next()
+            name = self.get_public_identifier(ps)
+            return ast.ExternalArgument(name)
+        elif ps.is_message_id_start():
+            name = self.get_private_identifier(ps)
+            return ast.MessageReference(name)
+        elif ps.is_number_start():
             return self.get_number(ps)
         elif ch == '"':
             return self.get_string(ps)
-        elif ch == '$':
-            ps.next()
-            name = self.get_identifier(ps)
-            return ast.ExternalArgument(name)
 
-        name = self.get_identifier(ps)
-        return ast.MessageReference(name)
+        raise ParseError('E0014')

fluent/syntax/serializer.py

@@ -78,11 +78,11 @@ def serialize_section(section):
     if section.comment:
         return "\n\n{}\n[[ {} ]]\n\n".format(
             serialize_comment(section.comment),
-            serialize_symbol(section.name)
+            serialize_variant_name(section.name)
         )
     else:
         return "\n\n[[ {} ]]\n\n".format(
-            serialize_symbol(section.name)
+            serialize_variant_name(section.name)
         )
 
 
@@ -269,13 +269,13 @@ def serialize_identifier(identifier):
     return identifier.name
 
 
-def serialize_symbol(symbol):
+def serialize_variant_name(symbol):
     return symbol.name
 
 
 def serialize_variant_key(key):
-    if isinstance(key, ast.Symbol):
-        return serialize_symbol(key)
+    if isinstance(key, ast.VariantName):
+        return serialize_variant_name(key)
     if isinstance(key, ast.NumberExpression):
         return serialize_number_expression(key)
     raise Exception('Unknown variant key type: {}'.format(key.type))

tests/syntax/fixtures_behavior/attribute_expression_with_wrong_attr.ftl

@@ -1,7 +1,7 @@
 key = { foo.23 }
 
-//~ ERROR E0004, pos 12, args "a-zA-Z_"
+//~ ERROR E0004, pos 12, args "a-zA-Z"
 
 key = { foo. }
 
-//~ ERROR E0004, pos 31, args "a-zA-Z_"
+//~ ERROR E0004, pos 31, args "a-zA-Z"

tests/syntax/fixtures_behavior/attribute_of_private_as_placeable.ftl

@@ -0,0 +1,2 @@
+err1 = { -brand.gender }
+//~ ERROR E0019, pos 23

tests/syntax/fixtures_behavior/attribute_of_public_as_selector.ftl

@@ -0,0 +1,6 @@
+err1 =
+    { foo.bar ->
+        [1] One
+       *[2] Two
+    }
+//~ ERROR E0018, pos 21

tests/syntax/fixtures_behavior/broken_number.ftl

@@ -5,7 +5,7 @@ key = { -2.4. }
 //~ ERROR E0003, pos 30, args "}"
 
 key = { -.4 }
-//~ ERROR E0004, pos 44, args "0-9"
+//~ ERROR E0014, pos 43
 
 key = { -2..4 }
 //~ ERROR E0004, pos 61, args "0-9"

tests/syntax/fixtures_behavior/indent.ftl

@@ -3,7 +3,7 @@
 
 key2 = {
 a }
-//~ ERROR E0004, pos 20, args "a-zA-Z_"
+//~ ERROR E0014, pos 20
 //~ ERROR E0005, pos 23, args "a"
 
 key3 = { a
@@ -12,4 +12,4 @@ key3 = { a
 
 key4 = {
 { a }}
-//~ ERROR E0004, pos 48, args "a-zA-Z_"
+//~ ERROR E0014, pos 48

tests/syntax/fixtures_behavior/message_reference_as_selector.ftl

@@ -0,0 +1,5 @@
+err1 =
+    { foo ->
+       *[1] One
+    }
+//~ ERROR E0016, pos 17

tests/syntax/fixtures_behavior/non_id_attribute_name.ftl

@@ -1,3 +1,3 @@
 key = Value
     .2 = Foo
-//~ ERROR E0004, pos 17, args "a-zA-Z_"
+//~ ERROR E0004, pos 17, args "a-zA-Z"