From f3f9053f0fd0650e7dfad01e61e5fcaf796fbc11 Mon Sep 17 00:00:00 2001 From: Axel Hecht Date: Thu, 7 Feb 2019 13:03:23 +0100 Subject: [PATCH] Add a read-only visitor pattern. This includes some performance tests to compare the visitor and a traversal. Locally, the visitor is twice as fast. This is the most basic Visitor pattern. I've had versions that were more generic, and performance gains vanished quickly. In particular optionally collecting results on exit halfed the performance benefits. If we want to factor that in, we should have an independent base class for that. --- fluent.syntax/fluent/syntax/ast.py | 30 ++ .../syntax/fixtures_perf/workload-low.ftl | 378 ++++++++++++++++++ fluent.syntax/tests/syntax/test_visitor.py | 116 ++++++ 3 files changed, 524 insertions(+) create mode 100644 fluent.syntax/tests/syntax/fixtures_perf/workload-low.ftl create mode 100644 fluent.syntax/tests/syntax/test_visitor.py diff --git a/fluent.syntax/fluent/syntax/ast.py b/fluent.syntax/fluent/syntax/ast.py index 7ecd1826..60670c79 100644 --- a/fluent.syntax/fluent/syntax/ast.py +++ b/fluent.syntax/fluent/syntax/ast.py @@ -3,6 +3,36 @@ import json +class Visitor(object): + '''Read-only visitor pattern. + + Subclass this to gather information from an AST. + To generally define which nodes not to descend in to, overload + `generic_visit`. + To handle specific node types, add methods like `visit_Pattern`. + The boolean value of the returned value determines if the visitor + descends into the children of the given AST node. + ''' + def visit(self, value): + if isinstance(value, BaseNode): + self.visit_node(value) + if isinstance(value, list): + for node in value: + self.visit(node) + + def visit_node(self, node): + nodename = type(node).__name__ + visit = getattr(self, 'visit_{}'.format(nodename), self.generic_visit) + should_descend = visit(node) + if not should_descend: + return + for propname, propvalue in vars(node).items(): + self.visit(propvalue) + + def generic_visit(self, node): + return True + + def to_json(value, fn=None): if isinstance(value, BaseNode): return value.to_json(fn) diff --git a/fluent.syntax/tests/syntax/fixtures_perf/workload-low.ftl b/fluent.syntax/tests/syntax/fixtures_perf/workload-low.ftl new file mode 100644 index 00000000..8ad46ee8 --- /dev/null +++ b/fluent.syntax/tests/syntax/fixtures_perf/workload-low.ftl @@ -0,0 +1,378 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +# browser/locales/en-US/browser/menubar.ftl + +## File menu + +file-menu = + .label = File + .accesskey = F +tab-menuitem = + .label = New Tab + .accesskey = T +tab-key = + .key = t +new-user-context-menu = + .label = New Container Tab + .accesskey = C +new-navigator-menuitem = + .label = New Window + .accesskey = N +new-navigator-key = + .key = N +new-private-window-menuitem = + .label = New Private Window + .accesskey = W +new-non-remote-window-menuitem = + .label = New Non-e10s Window + +# Only displayed on OS X, and only on windows that aren't main browser windows, +# or when there are no windows but Firefox is still running. +open-location-menuitem = + .label = Open Location… +open-file-menuitem = + .label = Open File… + .accesskey = O +open-file-key = + .key = o + +close-menuitem = + .label = Close + .accesskey = C +close-key = + .key = W +close-window-menuitem = + .label = Close Window + .accesskey = d + +# [accesskey2] is for content area context menu +save-page-menuitem = + .label = Save Page As… + .accesskey = A + .accesskey2 = P +save-page-key = + .key = s + +email-page-menuitem = + .label = Email Link… + .accesskey = E + +print-setup-menuitem = + .label = Page Setup… + .accesskey = u +print-preview-menuitem = + .label = Print Preview… + .accesskey = v +print-menuitem = + .label = Print… + .accesskey = P +print-key = + .key = p + +go-offline-menuitem = + .label = Work Offline + .accesskey = k + +quit-application-menuitem = + .label = Quit + .accesskey = Q +quit-application-menuitem-win = + .label = Exit + .accesskey = x +quit-application-menuitem-mac = + .label = Quit { -brand-shorter-name } +# Used by both Linux and OSX builds +quit-application-key-unix = + .key = Q + +## Edit menu + +edit-menu = + .label = Edit + .accesskey = E +undo-menuitem = + .label = Undo + .accesskey = U +undo-key = + .key = Z +redo-menuitem = + .label = Redo + .accesskey = R +redo-key = + .key = Y +cut-menuitem = + .label = Cut + .accesskey = t +cut-key = + .key = X +copy-menuitem = + .label = Copy + .accesskey = C +copy-key = + .key = C +paste-menuitem = + .label = Paste + .accesskey = P +paste-key = + .key = V +delete-menuitem = + .label = Delete + .accesskey = D +delete-key = + .key = D +select-all-menuitem = + .label = Select All + .accesskey = A +select-all-key = + .key = A + +find-on-menuitem = + .label = Find in This Page… + .accesskey = F +find-on-key = + .key = f +find-again-menuitem = + .label = Find Again + .accesskey = g +find-again-key1 = + .key = g +find-again-key2 = + .keycode = VK_F3 +find-selection-key = + .key = e + +bidi-switch-text-direction-menuitem = + .label = Switch Text Direction + .accesskey = w +bidi-switch-text-direction-key = + .key = X + +preferences-menuitem = + .label = Options + .accesskey = O +preferences-menuitem-unix = + .label = Preferences + .accesskey = n + + +## View menu + +view-menu = + .label = View + .accesskey = V +view-toolbars-menu = + .label = Toolbars + .accesskey = T +view-sidebar-menu = + .label = Sidebar + .accesskey = e +view-customize-toolbar-menuitem = + .label = Customize… + .accesskey = C + +full-zoom-menu = + .label = Zoom + .accesskey = Z +full-zoom-enlarge-menuitem = + .label = Zoom In + .accesskey = I +full-zoom-enlarge-key1 = + .key = + +full-zoom-enlarge-key2 = + .key = {""} +full-zoom-enlarge-key3 = + .key = {""} +full-zoom-reduce-menuitem = + .label = Zoom Out + .accesskey = O +full-zoom-reduce-key1 = + .key = - +full-zoom-reduce-key2 = + .key = {""} +full-zoom-reset-menuitem = + .label = Reset + .accesskey = R +full-zoom-reset-key1 = + .key = 0 +full-zoom-reset-key2 = + .key = {""} +full-zoom-toggle-menuitem = + .label = Zoom Text Only + .accesskey = T + +page-style-menu = + .label = Page Style + .accesskey = y +page-style-no-style-menuitem = + .label = No Style + .accesskey = n +page-style-persistent-only-menuitem = + .label = Basic Page Style + .accesskey = b + +show-all-tabs-menuitem = + .label = Show All Tabs + .accesskey = A +bidi-switch-page-direction-menuitem = + .label = Switch Page Direction + .accesskey = D + +## Full Screen controls +## Match what Safari and other Apple applications use on OS X Lion. + +enter-full-screen-menuitem = + .label = Enter Full Screen + .accesskey = F +exit-full-screen-menuitem = + .label = Exit Full Screen + .accesskey = F +full-screen-menuitem = + .label = Full Screen + .accesskey = F +full-screen-key = + .key = f + + +## History menu + +history-menu = + .label = History + .accesskey = s +show-all-history-menuitem = + .label = Show All History +show-all-history-key = + .key = H +clear-recent-history-menuitem = + .label = Clean Recent History… +history-synced-tabs-menuitem = + .label = Synced Tabs +history-restore-last-session-menuitem = + .label = Restore Previous Session +history-undo-menu = + .label = Recently Closed Tabs +history-undo-window-menu = + .label = Recently Closed Windows + + +## Bookmarks menu + +bookmarks-menu = + .label = Bookmarks + .accesskey = B +show-all-bookmarks-menuitem = + .label = Show All Bookmarks +show-all-bookmarks-key = + .key = b +# The "key" attribute should not contain the letters A-F since the are reserved +# shortcut keys on Linux. +show-all-bookmarks-key-gtk = + .key = o +bookmark-this-page-broadcaster = + .label = Bookmark This Page +edit-this-page-broadcaster = + .label = Edit This Page +bookmark-this-page-key = + .key = d +subscribe-to-page-menuitem = + .label = Subscribe to This Page… +subscribe-to-page-menupopup = + .label = Subscribe to This Page… +add-cur-pages-menuitem = + .label = Bookmark All Tabs… +recent-bookmarks-menuitem = + .label = Recently Bookmarked + +other-bookmarks-menu = + .label = Other Bookmarks +personalbar-menu = + .label = Bookmarks Toolbar + .accesskey = B + + +## Tools menu + +tools-menu = + .label = Tools + .accesskey = T +downloads-menuitem = + .label = Downloads + .accesskey = D +downloads-key = + .key = j +downloads-key-unix = + .key = y +addons-menuitem = + .label = Add-ons + .accesskey = A +addons-key = + .key = A + +sync-sign-in-menuitem = + .label = Sign In To { sync-brand-short-name }… + .accesskey = Y +sync-sync-now-menuitem = + .label = Sync Now + .accesskey = S +sync-re-auth-menuitem = + .label = Reconnect to { sync-brand-short-name }… + .accesskey = R +sync-toolbar-button = + .label = Sync + +web-developer-menu = + .label = Web Developer + .accesskey = W + +page-source-broadcaster = + .label = Page Source + .accesskey = o +page-source-key = + .key = u +page-info-menuitem = + .label = Page Info + .accesskey = I +page-info-key = + .key = i +mirror-tab-menu = + .label = Mirror Tab + .accesskey = m + + +# browser/locales/en-US/browser/toolbar.ftl + +urlbar-textbox = + .placeholder = Search or enter address + .accesskey = d + + +## Toolbar items + +view-bookmarks-broadcaster = + .label = Bookmarks +view-bookmarks-key = + .key = b +view-bookmarks-key-win = + .key = i + +view-history-broadcaster = + .label = History +view-history-key = + .key = h +view-tabs-broadcaster = + .label = Synced Tabs + + +# browser/branding/official/locales/en-US/brand.ftl + +-brand-shorter-name = Firefox +-brand-short-name = Firefox +-brand-full-name = Mozilla Firefox +vendor-short-name = Mozilla + +trademark-info = + Firefox and the Firefox logos are trademarks of the Mozilla Foundation. + +sync-brand-short-name = Sync diff --git a/fluent.syntax/tests/syntax/test_visitor.py b/fluent.syntax/tests/syntax/test_visitor.py new file mode 100644 index 00000000..c9c313dc --- /dev/null +++ b/fluent.syntax/tests/syntax/test_visitor.py @@ -0,0 +1,116 @@ +from __future__ import unicode_literals + +import codecs +from collections import defaultdict +import os +import unittest +import timeit + +from fluent.syntax.parser import FluentParser +from fluent.syntax import ast +from tests.syntax import dedent_ftl + + +class MockVisitor(ast.Visitor): + def __init__(self): + self.calls = defaultdict(int) + self.pattern_calls = 0 + + def generic_visit(self, node): + self.calls[type(node).__name__] += 1 + return super(MockVisitor, self).generic_visit(node) + + def visit_Pattern(self, node): + self.pattern_calls += 1 + return False + + +class TestVisitor(unittest.TestCase): + def test_resource(self): + resource = FluentParser().parse(dedent_ftl('''\ + one = Message + # Comment + two = Messages + three = Messages with + .an = Attribute + ''')) + mv = MockVisitor() + mv.visit(resource) + self.assertEqual(mv.pattern_calls, 4) + self.assertDictEqual( + mv.calls, + { + 'Resource': 1, + 'Comment': 1, + 'Message': 3, + 'Identifier': 4, + 'Attribute': 1, + 'Span': 10, + } + ) + + +class WordCounter(object): + def __init__(self): + self.word_count = 0 + def __call__(self, node): + if isinstance(node, ast.TextElement): + self.word_count += len(node.value.split()) + return node + +class VisitorCounter(ast.Visitor): + def __init__(self): + self.word_count = 0 + + def generic_visit(self, node): + return not isinstance(node, (ast.Span, ast.Annotation)) + + def visit_TextElement(self, node): + self.word_count += len(node.value.split()) + return False + + +class TestPerf(unittest.TestCase): + def setUp(self): + parser = FluentParser() + workload = os.path.join( + os.path.dirname(__file__), 'fixtures_perf', 'workload-low.ftl' + ) + with codecs.open(workload, encoding='utf-8') as f: + self.resource = parser.parse(f.read()) + + def test_traverse(self): + counter = WordCounter() + self.resource.traverse(counter) + self.assertEqual(counter.word_count, 277) + + def test_visitor(self): + counter = VisitorCounter() + counter.visit(self.resource) + self.assertEqual(counter.word_count, 277) + + +def gather_stats(method, repeat=10, number=50): + t = timeit.Timer( + setup=''' +from tests.syntax import test_visitor +test = test_visitor.TestPerf('test_{}') +test.setUp() +'''.format(method), + stmt='test.test_{}()'.format(method) + ) + return [ + result/number for result in + t.repeat(repeat=repeat, number=number) + ] + + + +if __name__=='__main__': + for m in ('traverse', 'visitor'): + results = gather_stats(m) + try: + import statistics + print("{}:\t{}".format(m, statistics.mean(results))) + except ImportError: + print("{}:\t{}".format(m, sum(results)/len(results)))