Add 0.5 comments support (#32)

* Add 0.5 comments support

* Support 0.5 comments in fluent.migrate

Author: zbraniecki

fluent/migrate/merge.py

@@ -26,11 +26,7 @@ def merge_body(body):
 
     def merge_entry(entry):
         # All standalone comments will be merged.
-        if isinstance(entry, FTL.Comment):
-            return entry
-
-        # All section headers will be merged.
-        if isinstance(entry, FTL.Section):
+        if isinstance(entry, FTL.BaseComment):
             return entry
 
         # Ignore Junk
@@ -56,4 +52,4 @@ def merge_entry(entry):
             return evaluate(ctx, transform)
 
     body = merge_body(reference.body)
-    return FTL.Resource(body, reference.comment)
+    return FTL.Resource(body)

fluent/syntax/ast.py

@@ -151,10 +151,9 @@ def add_span(self, start, end):
 
 
 class Resource(SyntaxNode):
-    def __init__(self, body=None, comment=None, **kwargs):
+    def __init__(self, body=None, **kwargs):
         super(Resource, self).__init__(**kwargs)
         self.body = body or []
-        self.comment = comment
 
 
 class Entry(SyntaxNode):
@@ -266,16 +265,30 @@ class Symbol(Identifier):
     def __init__(self, name, **kwargs):
         super(Symbol, self).__init__(name, **kwargs)
 
-class Comment(Entry):
+
+class BaseComment(Entry):
     def __init__(self, content=None, **kwargs):
-        super(Comment, self).__init__(**kwargs)
+        super(BaseComment, self).__init__(**kwargs)
         self.content = content
 
-class Section(Entry):
-    def __init__(self, name, comment=None, **kwargs):
-        super(Section, self).__init__(**kwargs)
-        self.name = name
-        self.comment = comment
+
+class Comment(BaseComment):
+    def __init__(self, content=None, **kwargs):
+        zero_four_style = kwargs.pop("zero_four_style", False)
+        super(Comment, self).__init__(content, **kwargs)
+        if zero_four_style:
+            self.zero_four_style = True
+
+
+class GroupComment(BaseComment):
+    def __init__(self, content=None, **kwargs):
+        super(GroupComment, self).__init__(content, **kwargs)
+
+
+class ResourceComment(BaseComment):
+    def __init__(self, content=None, **kwargs):
+        super(ResourceComment, self).__init__(content, **kwargs)
+
 
 class Function(Identifier):
     def __init__(self, name, **kwargs):

fluent/syntax/ftlstream.py

@@ -88,11 +88,12 @@ def is_number_start(self):
 
         return (cc >= 48 and cc <= 57) or cc == 45
 
-    def is_peek_next_line_comment(self):
+    def is_peek_next_line_zero_four_style_comment(self):
         if not self.current_peek_is('\n'):
             return False
 
         self.peek()
+
         if self.current_peek_is('/'):
             self.peek()
             if self.current_peek_is('/'):
@@ -102,6 +103,34 @@ def is_peek_next_line_comment(self):
         self.reset_peek()
         return False
 
+    # -1 - any
+    #  0 - comment
+    #  1 - group comment
+    #  2 - resource comment
+    def is_peek_next_line_comment(self, level=-1):
+        if not self.current_peek_is('\n'):
+            return False
+
+        i = 0
+
+        while (i <= level or (level == -1 and i < 3)):
+            self.peek()
+            if not self.current_peek_is('#'):
+                if i != level and level != -1:
+                    self.reset_peek()
+                    return False
+                break
+            i += 1
+
+        self.peek()
+
+        if self.current_peek() in [' ', '\n']:
+            self.reset_peek()
+            return True
+
+        self.reset_peek()
+        return False
+
     def is_peek_next_line_variant_start(self):
         if not self.current_peek_is('\n'):
             return False

fluent/syntax/parser.py

@@ -18,8 +18,8 @@ def decorated(self, ps, *args):
         if node.span is not None:
             return node
 
-        # Spans of Messages and Sections should include the attached Comment.
-        if isinstance(node, ast.Message) or isinstance(node, ast.Section):
+        # Spans of Messages should include the attached Comment.
+        if isinstance(node, ast.Message):
             if node.comment is not None:
                 start = node.comment.span.start
 
@@ -35,8 +35,6 @@ def __init__(self, with_spans=True):
         self.with_spans = with_spans
 
     def parse(self, source):
-        comment = None
-
         ps = FTLParserStream(source)
         ps.skip_blank_lines()
 
@@ -45,15 +43,21 @@ def parse(self, source):
         while ps.current():
             entry = self.get_entry_or_junk(ps)
 
-            if isinstance(entry, ast.Comment) and len(entries) == 0:
-                comment = entry
+            if entry is None:
+                continue
+
+            if isinstance(entry, ast.Comment) \
+               and hasattr(entry, "zero_four_style") and len(entries) == 0:
+                comment = ast.ResourceComment(entry.content)
+                comment.span = entry.span
+                entries.append(comment)
             else:
                 entries.append(entry)
 
             ps.skip_inline_ws()
             ps.skip_blank_lines()
 
-        res = ast.Resource(entries, comment)
+        res = ast.Resource(entries)
 
         if self.with_spans:
             res.add_span(0, ps.get_index())
@@ -88,17 +92,21 @@ def get_entry_or_junk(self, ps):
     def get_entry(self, ps):
         comment = None
 
-        if ps.current_is('/'):
+        if ps.current_is('/') or ps.current_is('#'):
             comment = self.get_comment(ps)
 
             # The Comment content doesn't include the trailing newline. Consume
             # this newline to be ready for the next entry. None stands for EOF.
             ps.expect_char('\n' if ps.current() else None)
 
         if ps.current_is('['):
-            return self.get_section(ps, comment)
+            self.skip_section(ps)
+            if comment:
+                return ast.GroupComment(comment.content)
+            return None
 
-        if ps.is_id_start():
+        if ps.is_id_start() \
+           and (comment is None or isinstance(comment, ast.Comment)):
             return self.get_message(ps, comment)
 
         if comment:
@@ -107,23 +115,20 @@ def get_entry(self, ps):
         raise ParseError('E0002')
 
     @with_span
-    def get_comment(self, ps):
+    def get_zero_four_style_comment(self, ps):
         ps.expect_char('/')
         ps.expect_char('/')
         ps.take_char_if(' ')
 
         content = ''
 
-        def until_eol(x):
-            return x != '\n'
-
         while True:
-            ch = ps.take_char(until_eol)
+            ch = ps.take_char(lambda x: x != '\n')
             while ch:
                 content += ch
-                ch = ps.take_char(until_eol)
+                ch = ps.take_char(lambda x: x != '\n')
 
-            if ps.is_peek_next_line_comment():
+            if ps.is_peek_next_line_zero_four_style_comment():
                 content += '\n'
                 ps.next()
                 ps.expect_char('/')
@@ -132,23 +137,65 @@ def until_eol(x):
             else:
                 break
 
-        return ast.Comment(content)
+        comment = ast.Comment(content)
+        comment.zero_four_style = True
+        return comment
 
     @with_span
-    def get_section(self, ps, comment):
+    def get_comment(self, ps):
+        if ps.current_is('/'):
+            return self.get_zero_four_style_comment(ps)
+
+        # 0 - comment
+        # 1 - group comment
+        # 2 - resource comment
+        level = -1
+        content = ''
+
+        while True:
+            i = -1
+            while ps.current_is('#') and (i < (2 if level == -1 else level)):
+                ps.next()
+                i += 1
+
+            if level == -1:
+                level = i
+
+            if not ps.current_is('\n'):
+                ps.expect_char(' ')
+                ch = ps.take_char(lambda x: x != '\n')
+                while ch:
+                    content += ch
+                    ch = ps.take_char(lambda x: x != '\n')
+
+            if ps.is_peek_next_line_comment(level):
+                content += '\n'
+                ps.next()
+            else:
+                break
+
+        if level == 0:
+            return ast.Comment(content)
+        elif level == 1:
+            return ast.GroupComment(content)
+        elif level == 2:
+            return ast.ResourceComment(content)
+
+    def skip_section(self, ps):
         ps.expect_char('[')
         ps.expect_char('[')
 
         ps.skip_inline_ws()
 
-        symb = self.get_symbol(ps)
+        self.get_symbol(ps)
 
         ps.skip_inline_ws()
 
         ps.expect_char(']')
         ps.expect_char(']')
 
-        return ast.Section(symb, comment)
+        ps.skip_inline_ws()
+        ps.next()
 
     @with_span
     def get_message(self, ps, comment):

fluent/syntax/serializer.py

@@ -16,39 +16,61 @@ def contain_new_line(elems):
 
 
 class FluentSerializer(object):
+    HAS_ENTRIES = 1
+
     def __init__(self, with_junk=False):
         self.with_junk = with_junk
 
     def serialize(self, resource):
+        state = 0
+
         parts = []
-        if resource.comment:
-            parts.append(
-                "{}\n\n".format(
-                    serialize_comment(resource.comment)
-                )
-            )
         for entry in resource.body:
             if not isinstance(entry, ast.Junk) or self.with_junk:
-                parts.append(self.serialize_entry(entry))
+                parts.append(self.serialize_entry(entry, state))
+                if not state & self.HAS_ENTRIES:
+                    state |= self.HAS_ENTRIES
 
         return "".join(parts)
 
-    def serialize_entry(self, entry):
+    def serialize_entry(self, entry, state=0):
         if isinstance(entry, ast.Message):
             return serialize_message(entry)
-        if isinstance(entry, ast.Section):
-            return serialize_section(entry)
         if isinstance(entry, ast.Comment):
-            return "\n{}\n\n".format(serialize_comment(entry))
+            if state & self.HAS_ENTRIES:
+                return "\n{}\n\n".format(serialize_comment(entry))
+            return "{}\n\n".format(serialize_comment(entry))
+        if isinstance(entry, ast.GroupComment):
+            if state & self.HAS_ENTRIES:
+                return "\n{}\n\n".format(serialize_group_comment(entry))
+            return "{}\n\n".format(serialize_group_comment(entry))
+        if isinstance(entry, ast.ResourceComment):
+            if state & self.HAS_ENTRIES:
+                return "\n{}\n\n".format(serialize_resource_comment(entry))
+            return "{}\n\n".format(serialize_resource_comment(entry))
         if isinstance(entry, ast.Junk):
             return serialize_junk(entry)
         raise Exception('Unknown entry type: {}'.format(entry.type))
 
 
 def serialize_comment(comment):
-    return "".join([
-        "{}{}".format("// ", line)
-        for line in comment.content.splitlines(True)
+    return "\n".join([
+        "#" if len(line) == 0 else "# {}".format(line)
+        for line in comment.content.splitlines(False)
+    ])
+
+
+def serialize_group_comment(comment):
+    return "\n".join([
+        "##" if len(line) == 0 else "## {}".format(line)
+        for line in comment.content.splitlines(False)
+    ])
+
+
+def serialize_resource_comment(comment):
+    return "\n".join([
+        "###" if len(line) == 0 else "### {}".format(line)
+        for line in comment.content.splitlines(False)
     ])
 
 

tests/migrate/fixtures/en-US/aboutDialog.ftl

@@ -1,6 +1,6 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 update-failed = Update failed. <a>Download manually</a>!
 channel-desc = You are on the { $channelname } channel.

tests/migrate/fixtures/en-US/aboutDownloads.ftl

@@ -1,6 +1,6 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 title = Downloads
 header = Your Downloads

tests/migrate/fixtures/en-US/privacy.ftl

@@ -1,6 +1,6 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 dnt-description =
     Send websites a “Do Not Track” signal