diff --git a/.github/workflows/fluent.runtime.yml b/.github/workflows/fluent.runtime.yml index ccae91b7..774eb7aa 100644 --- a/.github/workflows/fluent.runtime.yml +++ b/.github/workflows/fluent.runtime.yml @@ -39,14 +39,13 @@ jobs: run: | python -m pip install wheel python -m pip install --upgrade pip - python -m pip install fluent.syntax==${{ matrix.fluent-syntax }} + python -m pip install fluent.syntax==${{ matrix.fluent-syntax }} six python -m pip install . - name: Test working-directory: ./fluent.runtime run: | ./runtests.py lint: - name: flake8 runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -54,11 +53,21 @@ jobs: with: python-version: 3.9 - name: Install dependencies + working-directory: ./fluent.runtime run: | python -m pip install wheel python -m pip install --upgrade pip - python -m pip install flake8==6 - - name: lint + python -m pip install . + python -m pip install flake8==6 mypy==1 types-babel types-pytz + - name: Install latest fluent.syntax + working-directory: ./fluent.syntax + run: | + python -m pip install . + - name: flake8 working-directory: ./fluent.runtime run: | python -m flake8 + - name: mypy + working-directory: ./fluent.runtime + run: | + python -m mypy fluent/ diff --git a/.github/workflows/fluent.syntax.yml b/.github/workflows/fluent.syntax.yml index 00cc1b3c..6eedbf6e 100644 --- a/.github/workflows/fluent.syntax.yml +++ b/.github/workflows/fluent.syntax.yml @@ -35,12 +35,12 @@ jobs: run: | python -m pip install wheel python -m pip install --upgrade pip + python -m pip install . - name: Test working-directory: ./fluent.syntax run: | ./runtests.py - syntax: - name: flake8 + lint: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -48,10 +48,16 @@ jobs: with: python-version: 3.9 - name: Install dependencies + working-directory: ./fluent.syntax run: | python -m pip install --upgrade pip - python -m pip install flake8==6 - - name: lint + python -m pip install . + python -m pip install flake8==6 mypy==1 + - name: flake8 working-directory: ./fluent.syntax run: | python -m flake8 + - name: mypy + working-directory: ./fluent.syntax + run: | + python -m mypy fluent/ diff --git a/fluent.docs/setup.py b/fluent.docs/setup.py index 15436d93..138d54d9 100644 --- a/fluent.docs/setup.py +++ b/fluent.docs/setup.py @@ -1,6 +1,9 @@ -from setuptools import setup, find_namespace_packages +from setuptools import setup setup( name='fluent.docs', - packages=find_namespace_packages(include=['fluent.*']), + packages=['fluent.docs'], + install_requires=[ + 'typing-extensions>=3.7,<5' + ], ) diff --git a/fluent.runtime/fluent/runtime/__init__.py b/fluent.runtime/fluent/runtime/__init__.py index ba597112..74044378 100644 --- a/fluent.runtime/fluent/runtime/__init__.py +++ b/fluent.runtime/fluent/runtime/__init__.py @@ -1,14 +1,7 @@ -import babel -import babel.numbers -import babel.plural - from fluent.syntax import FluentParser -from fluent.syntax.ast import Message, Term +from fluent.syntax.ast import Resource -from .builtins import BUILTINS -from .prepare import Compiler -from .resolver import ResolverEnvironment, CurrentEnvironment -from .utils import native_to_fluent +from .bundle import FluentBundle from .fallback import FluentLocalization, AbstractResourceLoader, FluentResourceLoader @@ -21,94 +14,6 @@ ] -def FluentResource(source): +def FluentResource(source: str) -> Resource: parser = FluentParser() return parser.parse(source) - - -class FluentBundle: - """ - Bundles are single-language stores of translations. They are - aggregate parsed Fluent resources in the Fluent syntax and can - format translation units (entities) to strings. - - Always use `FluentBundle.get_message` to retrieve translation units from - a bundle. Generate the localized string by using `format_pattern` on - `message.value` or `message.attributes['attr']`. - Translations can contain references to other entities or - external arguments, conditional logic in form of select expressions, traits - which describe their grammatical features, and can use Fluent builtins. - See the documentation of the Fluent syntax for more information. - """ - - def __init__(self, locales, functions=None, use_isolating=True): - self.locales = locales - _functions = BUILTINS.copy() - if functions: - _functions.update(functions) - self._functions = _functions - self.use_isolating = use_isolating - self._messages = {} - self._terms = {} - self._compiled = {} - self._compiler = Compiler() - self._babel_locale = self._get_babel_locale() - self._plural_form = babel.plural.to_python(self._babel_locale.plural_form) - - def add_resource(self, resource, allow_overrides=False): - # TODO - warn/error about duplicates - for item in resource.body: - if not isinstance(item, (Message, Term)): - continue - map_ = self._messages if isinstance(item, Message) else self._terms - full_id = item.id.name - if full_id not in map_ or allow_overrides: - map_[full_id] = item - - def has_message(self, message_id): - return message_id in self._messages - - def get_message(self, message_id): - return self._lookup(message_id) - - def _lookup(self, entry_id, term=False): - if term: - compiled_id = '-' + entry_id - else: - compiled_id = entry_id - try: - return self._compiled[compiled_id] - except LookupError: - pass - entry = self._terms[entry_id] if term else self._messages[entry_id] - self._compiled[compiled_id] = self._compiler(entry) - return self._compiled[compiled_id] - - def format_pattern(self, pattern, args=None): - if args is not None: - fluent_args = { - argname: native_to_fluent(argvalue) - for argname, argvalue in args.items() - } - else: - fluent_args = {} - - errors = [] - env = ResolverEnvironment(context=self, - current=CurrentEnvironment(args=fluent_args), - errors=errors) - try: - result = pattern(env) - except ValueError as e: - errors.append(e) - result = '{???}' - return [result, errors] - - def _get_babel_locale(self): - for lc in self.locales: - try: - return babel.Locale.parse(lc.replace('-', '_')) - except babel.UnknownLocaleError: - continue - # TODO - log error - return babel.Locale.default() diff --git a/fluent.runtime/fluent/runtime/builtins.py b/fluent.runtime/fluent/runtime/builtins.py index 3b8bd4e9..4881c04f 100644 --- a/fluent.runtime/fluent/runtime/builtins.py +++ b/fluent.runtime/fluent/runtime/builtins.py @@ -1,10 +1,11 @@ -from .types import fluent_date, fluent_number +from typing import Any, Callable, Dict +from .types import FluentType, fluent_date, fluent_number NUMBER = fluent_number DATETIME = fluent_date -BUILTINS = { +BUILTINS: Dict[str, Callable[[Any], FluentType]] = { 'NUMBER': NUMBER, 'DATETIME': DATETIME, } diff --git a/fluent.runtime/fluent/runtime/bundle.py b/fluent.runtime/fluent/runtime/bundle.py new file mode 100644 index 00000000..45e32852 --- /dev/null +++ b/fluent.runtime/fluent/runtime/bundle.py @@ -0,0 +1,110 @@ +import babel +import babel.numbers +import babel.plural +from typing import Any, Callable, Dict, List, TYPE_CHECKING, Tuple, Union, cast +from typing_extensions import Literal + +from fluent.syntax import ast as FTL + +from .builtins import BUILTINS +from .prepare import Compiler +from .resolver import CurrentEnvironment, Message, Pattern, ResolverEnvironment +from .utils import native_to_fluent + +if TYPE_CHECKING: + from .types import FluentNone, FluentType + +PluralCategory = Literal['zero', 'one', 'two', 'few', 'many', 'other'] + + +class FluentBundle: + """ + Bundles are single-language stores of translations. They are + aggregate parsed Fluent resources in the Fluent syntax and can + format translation units (entities) to strings. + + Always use `FluentBundle.get_message` to retrieve translation units from + a bundle. Generate the localized string by using `format_pattern` on + `message.value` or `message.attributes['attr']`. + Translations can contain references to other entities or + external arguments, conditional logic in form of select expressions, traits + which describe their grammatical features, and can use Fluent builtins. + See the documentation of the Fluent syntax for more information. + """ + + def __init__(self, + locales: List[str], + functions: Union[Dict[str, Callable[[Any], 'FluentType']], None] = None, + use_isolating: bool = True): + self.locales = locales + self._functions = {**BUILTINS, **(functions or {})} + self.use_isolating = use_isolating + self._messages: Dict[str, Union[FTL.Message, FTL.Term]] = {} + self._terms: Dict[str, Union[FTL.Message, FTL.Term]] = {} + self._compiled: Dict[str, Message] = {} + # The compiler is not typed, and this cast is only valid for the public API + self._compiler = cast(Callable[[Union[FTL.Message, FTL.Term]], Message], Compiler()) + self._babel_locale = self._get_babel_locale() + self._plural_form = cast(Callable[[Any], Callable[[Union[int, float]], PluralCategory]], + babel.plural.to_python)(self._babel_locale.plural_form) + + def add_resource(self, resource: FTL.Resource, allow_overrides: bool = False) -> None: + # TODO - warn/error about duplicates + for item in resource.body: + if not isinstance(item, (FTL.Message, FTL.Term)): + continue + map_ = self._messages if isinstance(item, FTL.Message) else self._terms + full_id = item.id.name + if full_id not in map_ or allow_overrides: + map_[full_id] = item + + def has_message(self, message_id: str) -> bool: + return message_id in self._messages + + def get_message(self, message_id: str) -> Message: + return self._lookup(message_id) + + def _lookup(self, entry_id: str, term: bool = False) -> Message: + if term: + compiled_id = '-' + entry_id + else: + compiled_id = entry_id + try: + return self._compiled[compiled_id] + except LookupError: + pass + entry = self._terms[entry_id] if term else self._messages[entry_id] + self._compiled[compiled_id] = self._compiler(entry) + return self._compiled[compiled_id] + + def format_pattern(self, + pattern: Pattern, + args: Union[Dict[str, Any], None] = None + ) -> Tuple[Union[str, 'FluentNone'], List[Exception]]: + if args is not None: + fluent_args = { + argname: native_to_fluent(argvalue) + for argname, argvalue in args.items() + } + else: + fluent_args = {} + + errors: List[Exception] = [] + env = ResolverEnvironment(context=self, + current=CurrentEnvironment(args=fluent_args), + errors=errors) + try: + result = pattern(env) + except ValueError as e: + errors.append(e) + result = '{???}' + return (result, errors) + + def _get_babel_locale(self) -> babel.Locale: + for lc in self.locales: + try: + return babel.Locale.parse(lc.replace('-', '_')) + except babel.UnknownLocaleError: + continue + # TODO - log error + return babel.Locale.default() diff --git a/fluent.runtime/fluent/runtime/errors.py b/fluent.runtime/fluent/runtime/errors.py index 1c718285..bd1592cc 100644 --- a/fluent.runtime/fluent/runtime/errors.py +++ b/fluent.runtime/fluent/runtime/errors.py @@ -1,7 +1,9 @@ +from typing import cast + + class FluentFormatError(ValueError): - def __eq__(self, other): - return ((other.__class__ == self.__class__) and - other.args == self.args) + def __eq__(self, other: object) -> bool: + return ((other.__class__ == self.__class__) and cast(ValueError, other).args == self.args) class FluentReferenceError(FluentFormatError): diff --git a/fluent.runtime/fluent/runtime/fallback.py b/fluent.runtime/fluent/runtime/fallback.py index b25b4822..bd880306 100644 --- a/fluent.runtime/fluent/runtime/fallback.py +++ b/fluent.runtime/fluent/runtime/fallback.py @@ -1,5 +1,14 @@ import codecs import os +from typing import Any, Callable, Dict, Generator, List, TYPE_CHECKING, Type, Union, cast + +from fluent.syntax import FluentParser + +from .bundle import FluentBundle + +if TYPE_CHECKING: + from fluent.syntax.ast import Resource + from .types import FluentType class FluentLocalization: @@ -9,41 +18,42 @@ class FluentLocalization: This handles language fallback, bundle creation and string localization. It uses the given resource loader to load and parse Fluent data. """ + def __init__( - self, locales, resource_ids, resource_loader, - use_isolating=False, - bundle_class=None, functions=None, + self, + locales: List[str], + resource_ids: List[str], + resource_loader: 'AbstractResourceLoader', + use_isolating: bool = False, + bundle_class: Type[FluentBundle] = FluentBundle, + functions: Union[Dict[str, Callable[[Any], 'FluentType']], None] = None, ): self.locales = locales self.resource_ids = resource_ids self.resource_loader = resource_loader self.use_isolating = use_isolating - if bundle_class is None: - from fluent.runtime import FluentBundle - self.bundle_class = FluentBundle - else: - self.bundle_class = bundle_class + self.bundle_class = bundle_class self.functions = functions - self._bundle_cache = [] + self._bundle_cache: List[FluentBundle] = [] self._bundle_it = self._iterate_bundles() - def format_value(self, msg_id, args=None): + def format_value(self, msg_id: str, args: Union[Dict[str, Any], None] = None) -> str: for bundle in self._bundles(): if not bundle.has_message(msg_id): continue msg = bundle.get_message(msg_id) if not msg.value: continue - val, errors = bundle.format_pattern(msg.value, args) - return val + val, _errors = bundle.format_pattern(msg.value, args) + return cast(str, val) # Never FluentNone when format_pattern called externally return msg_id - def _create_bundle(self, locales): + def _create_bundle(self, locales: List[str]) -> FluentBundle: return self.bundle_class( locales, functions=self.functions, use_isolating=self.use_isolating ) - def _bundles(self): + def _bundles(self) -> Generator[FluentBundle, None, None]: bundle_pointer = 0 while True: if bundle_pointer == len(self._bundle_cache): @@ -54,7 +64,7 @@ def _bundles(self): yield self._bundle_cache[bundle_pointer] bundle_pointer += 1 - def _iterate_bundles(self): + def _iterate_bundles(self) -> Generator[FluentBundle, None, None]: for first_loc in range(0, len(self.locales)): locs = self.locales[first_loc:] for resources in self.resource_loader.resources(locs[0], self.resource_ids): @@ -68,7 +78,8 @@ class AbstractResourceLoader: """ Interface to implement for resource loaders. """ - def resources(self, locale, resource_ids): + + def resources(self, locale: str, resource_ids: List[str]) -> Generator[List['Resource'], None, None]: """ Yield lists of FluentResource objects, corresponding to each of the resource_ids. @@ -89,26 +100,25 @@ class FluentResourceLoader(AbstractResourceLoader): This loader does not support loading resources for one bundle from different roots. """ - def __init__(self, roots): + + def __init__(self, roots: Union[str, List[str]]): """ Create a resource loader. The roots may be a string for a single location on disk, or a list of strings. """ self.roots = [roots] if isinstance(roots, str) else roots - from fluent.runtime import FluentResource - self.Resource = FluentResource - def resources(self, locale, resource_ids): + def resources(self, locale: str, resource_ids: List[str]) -> Generator[List['Resource'], None, None]: for root in self.roots: - resources = [] + resources: List[Any] = [] for resource_id in resource_ids: path = self.localize_path(os.path.join(root, resource_id), locale) if not os.path.isfile(path): continue content = codecs.open(path, 'r', 'utf-8').read() - resources.append(self.Resource(content)) + resources.append(FluentParser().parse(content)) if resources: yield resources - def localize_path(self, path, locale): + def localize_path(self, path: str, locale: str) -> str: return path.format(locale=locale) diff --git a/fluent.runtime/fluent/runtime/prepare.py b/fluent.runtime/fluent/runtime/prepare.py index 450745b8..927105a3 100644 --- a/fluent.runtime/fluent/runtime/prepare.py +++ b/fluent.runtime/fluent/runtime/prepare.py @@ -1,38 +1,36 @@ +from typing import Any, Dict, List from fluent.syntax import ast as FTL from . import resolver class Compiler: - def __call__(self, item): + def __call__(self, item: Any) -> Any: if isinstance(item, FTL.BaseNode): return self.compile(item) if isinstance(item, (tuple, list)): return [self(elem) for elem in item] return item - def compile(self, node): - nodename = type(node).__name__ + def compile(self, node: Any) -> Any: + nodename: str = type(node).__name__ if not hasattr(resolver, nodename): return node - kwargs = vars(node).copy() + kwargs: Dict[str, Any] = vars(node).copy() for propname, propvalue in kwargs.items(): kwargs[propname] = self(propvalue) handler = getattr(self, 'compile_' + nodename, self.compile_generic) return handler(nodename, **kwargs) - def compile_generic(self, nodename, **kwargs): + def compile_generic(self, nodename: str, **kwargs: Any) -> Any: return getattr(resolver, nodename)(**kwargs) - def compile_Placeable(self, _, expression, **kwargs): + def compile_Placeable(self, _: Any, expression: Any, **kwargs: Any) -> Any: if isinstance(expression, resolver.Literal): return expression return resolver.Placeable(expression=expression, **kwargs) - def compile_Pattern(self, _, elements, **kwargs): - if ( - len(elements) == 1 and - isinstance(elements[0], resolver.Placeable) - ): + def compile_Pattern(self, _: Any, elements: List[Any], **kwargs: Any) -> Any: + if len(elements) == 1 and isinstance(elements[0], resolver.Placeable): # Don't isolate isolated placeables return resolver.NeverIsolatingPlaceable(elements[0].expression) if any( diff --git a/fluent.runtime/fluent/runtime/py.typed b/fluent.runtime/fluent/runtime/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/fluent.runtime/fluent/runtime/resolver.py b/fluent.runtime/fluent/runtime/resolver.py index 5394313d..e4d0caa5 100644 --- a/fluent.runtime/fluent/runtime/resolver.py +++ b/fluent.runtime/fluent/runtime/resolver.py @@ -1,12 +1,15 @@ -import contextlib - import attr +import contextlib +from typing import Any, Dict, Generator, List, Set, TYPE_CHECKING, Union, cast from fluent.syntax import ast as FTL from .errors import FluentCyclicReferenceError, FluentFormatError, FluentReferenceError from .types import FluentType, FluentNone, FluentInt, FluentFloat from .utils import reference_to_id, unknown_reference_error_obj +if TYPE_CHECKING: + from .bundle import FluentBundle + """ The classes in this module are used to transform the source @@ -38,22 +41,22 @@ class CurrentEnvironment: # For Messages, VariableReference nodes are interpreted as external args, # but for Terms they are the values explicitly passed using CallExpression # syntax. So we have to be able to change 'args' for this purpose. - args = attr.ib() + args: Dict[str, Any] = attr.ib(factory=dict) # This controls whether we need to report an error if a VariableReference # refers to an arg that is not present in the args dict. - error_for_missing_arg = attr.ib(default=True) + error_for_missing_arg: bool = attr.ib(default=True) @attr.s class ResolverEnvironment: - context = attr.ib() - errors = attr.ib() - part_count = attr.ib(default=0, init=False) - active_patterns = attr.ib(factory=set, init=False) - current = attr.ib(factory=CurrentEnvironment) + context: 'FluentBundle' = attr.ib() + errors: List[Exception] = attr.ib() + part_count: int = attr.ib(default=0, init=False) + active_patterns: Set[FTL.Pattern] = attr.ib(factory=set, init=False) + current: CurrentEnvironment = attr.ib(factory=CurrentEnvironment) @contextlib.contextmanager - def modified(self, **replacements): + def modified(self, **replacements: Any) -> Generator['ResolverEnvironment', None, None]: """ Context manager that modifies the 'current' attribute of the environment, restoring the old data at the end. @@ -65,7 +68,7 @@ def modified(self, **replacements): yield self self.current = old_current - def modified_for_term_reference(self, args=None): + def modified_for_term_reference(self, args: Union[Dict[str, Any], None] = None) -> Any: return self.modified(args=args if args is not None else {}, error_for_missing_arg=False) @@ -79,46 +82,59 @@ class BaseResolver: classes that don't show up in the evaluation, but need to be part of the compiled tree structure. """ - def __call__(self, env): + + def __call__(self, env: ResolverEnvironment) -> Any: raise NotImplementedError class Literal(BaseResolver): - pass - - -class EntryResolver(BaseResolver): - '''Entries (Messages and Terms) have attributes. - In the AST they're a list, the resolver wants a dict. The helper method - here should be called from the constructor. - ''' - def _fix_attributes(self): - self.attributes = { - attr.id.name: attr.value - for attr in self.attributes - } - - -class Message(FTL.Message, EntryResolver): - def __init__(self, id, **kwargs): - super().__init__(id, **kwargs) - self._fix_attributes() - - -class Term(FTL.Term, EntryResolver): - def __init__(self, id, value, **kwargs): - super().__init__(id, value, **kwargs) - self._fix_attributes() + value: str + + +class Message(FTL.Entry, BaseResolver): + id: 'Identifier' + value: Union['Pattern', None] + attributes: Dict[str, 'Pattern'] + + def __init__(self, + id: 'Identifier', + value: Union['Pattern', None] = None, + attributes: Union[List['Attribute'], None] = None, + comment: Any = None, + **kwargs: Any): + super().__init__(**kwargs) + self.id = id + self.value = value + self.attributes = {attr.id.name: attr.value for attr in attributes} if attributes else {} + + +class Term(FTL.Entry, BaseResolver): + id: 'Identifier' + value: 'Pattern' + attributes: Dict[str, 'Pattern'] + + def __init__(self, + id: 'Identifier', + value: 'Pattern', + attributes: Union[List['Attribute'], None] = None, + comment: Any = None, + **kwargs: Any): + super().__init__(**kwargs) + self.id = id + self.value = value + self.attributes = {attr.id.name: attr.value for attr in attributes} if attributes else {} class Pattern(FTL.Pattern, BaseResolver): # Prevent messages with too many sub parts, for CPI DOS protection MAX_PARTS = 1000 - def __init__(self, *args, **kwargs): + elements: List[Union['TextElement', 'Placeable']] # type: ignore + + def __init__(self, *args: Any, **kwargs: Any): super().__init__(*args, **kwargs) - def __call__(self, env): + def __call__(self, env: ResolverEnvironment) -> Union[str, FluentNone]: if self in env.active_patterns: env.errors.append(FluentCyclicReferenceError("Cyclic reference")) return FluentNone() @@ -137,7 +153,7 @@ def __call__(self, env): return retval -def resolve(fluentish, env): +def resolve(fluentish: Any, env: ResolverEnvironment) -> Any: if isinstance(fluentish, FluentType): return fluentish.format(env.context._babel_locale) if isinstance(fluentish, str): @@ -150,12 +166,16 @@ def resolve(fluentish, env): class TextElement(FTL.TextElement, Literal): - def __call__(self, env): + value: str + + def __call__(self, env: ResolverEnvironment) -> str: return self.value class Placeable(FTL.Placeable, BaseResolver): - def __call__(self, env): + expression: Union['InlineExpression', 'Placeable', 'SelectExpression'] + + def __call__(self, env: ResolverEnvironment) -> Any: inner = resolve(self.expression(env), env) if not env.context.use_isolating: return inner @@ -163,49 +183,70 @@ def __call__(self, env): class NeverIsolatingPlaceable(FTL.Placeable, BaseResolver): - def __call__(self, env): + expression: Union['InlineExpression', Placeable, 'SelectExpression'] + + def __call__(self, env: ResolverEnvironment) -> Any: inner = resolve(self.expression(env), env) return inner class StringLiteral(FTL.StringLiteral, Literal): - def __call__(self, env): + value: str + + def __call__(self, env: ResolverEnvironment) -> str: return self.parse()['value'] class NumberLiteral(FTL.NumberLiteral, BaseResolver): - def __init__(self, value, **kwargs): + value: Union[FluentFloat, FluentInt] # type: ignore + + def __init__(self, value: str, **kwargs: Any): super().__init__(value, **kwargs) - if '.' in self.value: + if '.' in cast(str, self.value): self.value = FluentFloat(self.value) else: self.value = FluentInt(self.value) - def __call__(self, env): + def __call__(self, env: ResolverEnvironment) -> Union[FluentFloat, FluentInt]: return self.value -class EntryReference(BaseResolver): - def __call__(self, env): - try: - entry = env.context._lookup(self.id.name, term=isinstance(self, FTL.TermReference)) - if self.attribute: - pattern = entry.attributes[self.attribute.name] - else: - pattern = entry.value - return pattern(env) - except LookupError: - ref_id = reference_to_id(self) - env.errors.append(unknown_reference_error_obj(ref_id)) - return FluentNone(f'{{{ref_id}}}') +def resolveEntryReference( + ref: Union['MessageReference', 'TermReference'], + env: ResolverEnvironment +) -> Union[str, FluentNone]: + try: + entry = env.context._lookup(ref.id.name, term=isinstance(ref, FTL.TermReference)) + pattern: Pattern + if ref.attribute: + pattern = entry.attributes[ref.attribute.name] + else: + pattern = entry.value # type: ignore + return pattern(env) + except LookupError: + ref_id = reference_to_id(ref) + env.errors.append(unknown_reference_error_obj(ref_id)) + return FluentNone(f'{{{ref_id}}}') + except TypeError: + ref_id = reference_to_id(ref) + env.errors.append(FluentReferenceError(f"No pattern: {ref_id}")) + return FluentNone(ref_id) + + +class MessageReference(FTL.MessageReference, BaseResolver): + id: 'Identifier' + attribute: Union['Identifier', None] + def __call__(self, env: ResolverEnvironment) -> Union[str, FluentNone]: + return resolveEntryReference(self, env) -class MessageReference(FTL.MessageReference, EntryReference): - pass +class TermReference(FTL.TermReference, BaseResolver): + id: 'Identifier' + attribute: Union['Identifier', None] + arguments: Union['CallArguments', None] -class TermReference(FTL.TermReference, EntryReference): - def __call__(self, env): + def __call__(self, env: ResolverEnvironment) -> Union[str, FluentNone]: if self.arguments: if self.arguments.positional: env.errors.append(FluentFormatError("Ignored positional arguments passed to term '{}'" @@ -214,11 +255,13 @@ def __call__(self, env): else: kwargs = None with env.modified_for_term_reference(args=kwargs): - return super().__call__(env) + return resolveEntryReference(self, env) class VariableReference(FTL.VariableReference, BaseResolver): - def __call__(self, env): + id: 'Identifier' + + def __call__(self, env: ResolverEnvironment) -> Any: name = self.id.name try: arg_val = env.current.args[name] @@ -236,17 +279,18 @@ def __call__(self, env): class Attribute(FTL.Attribute, BaseResolver): - pass + id: 'Identifier' + value: Pattern class SelectExpression(FTL.SelectExpression, BaseResolver): - def __call__(self, env): - key = self.selector(env) - return self.select_from_select_expression(env, key=key) + selector: 'InlineExpression' + variants: List['Variant'] # type: ignore - def select_from_select_expression(self, env, key): - default = None - found = None + def __call__(self, env: ResolverEnvironment) -> Union[str, FluentNone]: + key = self.selector(env) + default: Union['Variant', None] = None + found: Union['Variant', None] = None for variant in self.variants: if variant.default: default = variant @@ -256,15 +300,18 @@ def select_from_select_expression(self, env, key): break if found is None: + if default is None: + env.errors.append(FluentFormatError("No default")) + return FluentNone() found = default return found.value(env) -def is_number(val): +def is_number(val: Any) -> bool: return isinstance(val, (int, float)) -def match(val1, val2, env): +def match(val1: Any, val2: Any, env: ResolverEnvironment) -> bool: if val1 is None or isinstance(val1, FluentNone): return False if val2 is None or isinstance(val2, FluentNone): @@ -272,28 +319,36 @@ def match(val1, val2, env): if is_number(val1): if not is_number(val2): # Could be plural rule match - return env.context._plural_form(val1) == val2 + return cast(bool, env.context._plural_form(val1) == val2) elif is_number(val2): return match(val2, val1, env) - return val1 == val2 + return cast(bool, val1 == val2) class Variant(FTL.Variant, BaseResolver): - pass + key: Union['Identifier', NumberLiteral] + value: Pattern + default: bool class Identifier(FTL.Identifier, BaseResolver): - def __call__(self, env): + name: str + + def __call__(self, env: ResolverEnvironment) -> str: return self.name class CallArguments(FTL.CallArguments, BaseResolver): - pass + positional: List[Union['InlineExpression', Placeable]] # type: ignore + named: List['NamedArgument'] # type: ignore class FunctionReference(FTL.FunctionReference, BaseResolver): - def __call__(self, env): + id: Identifier + arguments: CallArguments + + def __call__(self, env: ResolverEnvironment) -> Any: args = [arg(env) for arg in self.arguments.positional] kwargs = {kwarg.name.name: kwarg.value(env) for kwarg in self.arguments.named} function_name = self.id.name @@ -312,4 +367,9 @@ def __call__(self, env): class NamedArgument(FTL.NamedArgument, BaseResolver): - pass + name: Identifier + value: Union[NumberLiteral, StringLiteral] + + +InlineExpression = Union[NumberLiteral, StringLiteral, MessageReference, + TermReference, VariableReference, FunctionReference] diff --git a/fluent.runtime/fluent/runtime/types.py b/fluent.runtime/fluent/runtime/types.py index c55df7bb..e57e653c 100644 --- a/fluent.runtime/fluent/runtime/types.py +++ b/fluent.runtime/fluent/runtime/types.py @@ -4,8 +4,11 @@ import attr import pytz +from babel import Locale from babel.dates import format_date, format_time, get_datetime_format, get_timezone from babel.numbers import NumberPattern, parse_pattern +from typing import Any, Dict, Type, TypeVar, Union, cast +from typing_extensions import Literal FORMAT_STYLE_DECIMAL = "decimal" FORMAT_STYLE_CURRENCY = "currency" @@ -43,18 +46,18 @@ class FluentType: - def format(self, locale): + def format(self, locale: Locale) -> str: raise NotImplementedError() class FluentNone(FluentType): - def __init__(self, name=None): + def __init__(self, name: Union[str, None] = None): self.name = name - def __eq__(self, other): + def __eq__(self, other: Any) -> bool: return isinstance(other, FluentNone) and self.name == other.name - def format(self, locale): + def format(self, locale: Locale) -> str: return self.name or "???" @@ -65,17 +68,19 @@ class NumberFormatOptions: # we can stick to Fluent spec more easily. # See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/NumberFormat - style = attr.ib(default=FORMAT_STYLE_DECIMAL, - validator=attr.validators.in_(FORMAT_STYLE_OPTIONS)) - currency = attr.ib(default=None) - currencyDisplay = attr.ib(default=CURRENCY_DISPLAY_SYMBOL, - validator=attr.validators.in_(CURRENCY_DISPLAY_OPTIONS)) - useGrouping = attr.ib(default=True) - minimumIntegerDigits = attr.ib(default=None) - minimumFractionDigits = attr.ib(default=None) - maximumFractionDigits = attr.ib(default=None) - minimumSignificantDigits = attr.ib(default=None) - maximumSignificantDigits = attr.ib(default=None) + style: Literal['decimal', 'currency', 'percent'] = attr.ib( + default=FORMAT_STYLE_DECIMAL, + validator=attr.validators.in_(FORMAT_STYLE_OPTIONS)) + currency: Union[str, None] = attr.ib(default=None) + currencyDisplay: Literal['symbol', 'code', 'name'] = attr.ib( + default=CURRENCY_DISPLAY_SYMBOL, + validator=attr.validators.in_(CURRENCY_DISPLAY_OPTIONS)) + useGrouping: bool = attr.ib(default=True) + minimumIntegerDigits: Union[int, None] = attr.ib(default=None) + minimumFractionDigits: Union[int, None] = attr.ib(default=None) + maximumFractionDigits: Union[int, None] = attr.ib(default=None) + minimumSignificantDigits: Union[int, None] = attr.ib(default=None) + maximumSignificantDigits: Union[int, None] = attr.ib(default=None) class FluentNumber(FluentType): @@ -83,12 +88,14 @@ class FluentNumber(FluentType): default_number_format_options = NumberFormatOptions() def __new__(cls, - value, - **kwargs): - self = super().__new__(cls, value) + value: Union[int, float, Decimal, 'FluentNumber'], + **kwargs: Any) -> 'FluentNumber': + self = super().__new__(cls, value) # type: ignore return self._init(value, kwargs) - def _init(self, value, kwargs): + def _init(self, + value: Union[int, float, Decimal, 'FluentNumber'], + kwargs: Dict[str, Any]) -> 'FluentNumber': self.options = merge_options(NumberFormatOptions, getattr(value, 'options', self.default_number_format_options), kwargs) @@ -98,21 +105,24 @@ def _init(self, value, kwargs): return self - def format(self, locale): + def format(self, locale: Locale) -> str: + selfnum = cast(float, self) if self.options.style == FORMAT_STYLE_DECIMAL: - base_pattern = locale.decimal_formats.get(None) + base_pattern = cast(NumberPattern, locale.decimal_formats.get(None)) pattern = self._apply_options(base_pattern) - return pattern.apply(self, locale) + return pattern.apply(selfnum, locale) elif self.options.style == FORMAT_STYLE_PERCENT: - base_pattern = locale.percent_formats.get(None) + base_pattern = cast(NumberPattern, locale.percent_formats.get(None)) pattern = self._apply_options(base_pattern) - return pattern.apply(self, locale) + return pattern.apply(selfnum, locale) elif self.options.style == FORMAT_STYLE_CURRENCY: base_pattern = locale.currency_formats['standard'] pattern = self._apply_options(base_pattern) - return pattern.apply(self, locale, currency=self.options.currency) + return pattern.apply(selfnum, locale, currency=self.options.currency) + # never happens + return '???' - def _apply_options(self, pattern): + def _apply_options(self, pattern: NumberPattern) -> NumberPattern: # We are essentially trying to copy the # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/NumberFormat # API using Babel number formatting routines, which is slightly awkward @@ -130,7 +140,7 @@ def _apply_options(self, pattern): if self.options.currencyDisplay == CURRENCY_DISPLAY_CODE: # Not sure of the correct algorithm here, but this seems to # work: - def replacer(s): + def replacer(s: str) -> str: return s.replace("¤", "¤¤") pattern.suffix = (replacer(pattern.suffix[0]), replacer(pattern.suffix[1])) @@ -144,15 +154,14 @@ def replacer(s): warnings.warn("Unsupported currencyDisplayValue {}, falling back to {}" .format(CURRENCY_DISPLAY_NAME, CURRENCY_DISPLAY_SYMBOL)) - if (self.options.minimumSignificantDigits is not None - or self.options.maximumSignificantDigits is not None): + minSD = self.options.minimumSignificantDigits + maxSD = self.options.maximumSignificantDigits + if (minSD is not None or maxSD is not None): # This triggers babel routines into 'significant digits' mode: pattern.pattern = '@' # We then manually set int_prec, and leave the rest as they are. - min_digits = (1 if self.options.minimumSignificantDigits is None - else self.options.minimumSignificantDigits) - max_digits = (min_digits if self.options.maximumSignificantDigits is None - else self.options.maximumSignificantDigits) + min_digits = minSD if minSD is not None else 1 + max_digits = maxSD if maxSD is not None else min_digits pattern.int_prec = (min_digits, max_digits) else: if self.options.minimumIntegerDigits is not None: @@ -165,7 +174,10 @@ def replacer(s): return pattern -def merge_options(options_class, base, kwargs): +Options = TypeVar('Options', bound=Union[NumberFormatOptions, 'DateFormatOptions']) + + +def merge_options(options_class: Type[Options], base: Union[Options, None], kwargs: Dict[str, Any]) -> Options: """ Given an 'options_class', an optional 'base' object to copy from, and some keyword arguments, create a new options instance @@ -188,7 +200,7 @@ def merge_options(options_class, base, kwargs): for k in kwargs.keys(): setattr(retval, k, getattr(kwarg_options, k)) - return retval + return retval # type: ignore # We want types that inherit from both FluentNumber and a native type, @@ -213,7 +225,10 @@ class FluentDecimal(FluentNumber, Decimal): pass -def fluent_number(number, **kwargs): +def fluent_number( + number: Union[int, float, Decimal, FluentNumber, FluentNone], + **kwargs: Any +) -> Union[FluentNumber, FluentNone]: if isinstance(number, FluentNumber) and not kwargs: return number if isinstance(number, int): @@ -232,7 +247,7 @@ def fluent_number(number, **kwargs): _UNGROUPED_PATTERN = parse_pattern("#0") -def clone_pattern(pattern): +def clone_pattern(pattern: NumberPattern) -> NumberPattern: return NumberPattern(pattern.pattern, pattern.prefix, pattern.suffix, @@ -249,25 +264,28 @@ class DateFormatOptions: # See https://projectfluent.org/fluent/guide/functions.html#datetime # Developer only - timeZone = attr.ib(default=None) + timeZone: Union[str, None] = attr.ib(default=None) # Other - hour12 = attr.ib(default=None) - weekday = attr.ib(default=None) - era = attr.ib(default=None) - year = attr.ib(default=None) - month = attr.ib(default=None) - day = attr.ib(default=None) - hour = attr.ib(default=None) - minute = attr.ib(default=None) - second = attr.ib(default=None) - timeZoneName = attr.ib(default=None) + hour12: Union[bool, None] = attr.ib(default=None) + weekday: Literal["long", "short", "narrow", None] = attr.ib(default=None) + era: Literal["long", "short", "narrow", None] = attr.ib(default=None) + year: Literal["numeric", "2-digit", None] = attr.ib(default=None) + month: Literal["numeric", "2-digit", "long", "short", "narrow", None] = attr.ib(default=None) + day: Literal["numeric", "2-digit", None] = attr.ib(default=None) + hour: Literal["numeric", "2-digit", None] = attr.ib(default=None) + minute: Literal["numeric", "2-digit", None] = attr.ib(default=None) + second: Literal["numeric", "2-digit", None] = attr.ib(default=None) + timeZoneName: Literal["long", "short", "longOffset", "shortOffset", + "longGeneric", "shortGeneric", None] = attr.ib(default=None) # See https://github.com/tc39/proposal-ecma402-datetime-style - dateStyle = attr.ib(default=None, - validator=attr.validators.in_(DATE_STYLE_OPTIONS)) - timeStyle = attr.ib(default=None, - validator=attr.validators.in_(TIME_STYLE_OPTIONS)) + dateStyle: Literal["full", "long", "medium", "short", None] = attr.ib( + default=None, + validator=attr.validators.in_(DATE_STYLE_OPTIONS)) + timeStyle: Literal["full", "long", "medium", "short", None] = attr.ib( + default=None, + validator=attr.validators.in_(TIME_STYLE_OPTIONS)) _SUPPORTED_DATETIME_OPTIONS = ['dateStyle', 'timeStyle', 'timeZone'] @@ -278,7 +296,7 @@ class FluentDateType(FluentType): # some Python implementation (e.g. PyPy) implement some methods. # So we leave those alone, and implement another `_init_options` # which is called from other constructors. - def _init_options(self, dt_obj, kwargs): + def _init_options(self, dt_obj: Union[date, datetime], kwargs: Dict[str, Any]) -> None: if 'timeStyle' in kwargs and not isinstance(self, datetime): raise TypeError("timeStyle option can only be specified for datetime instances, not date instance") @@ -289,32 +307,33 @@ def _init_options(self, dt_obj, kwargs): if k not in _SUPPORTED_DATETIME_OPTIONS: warnings.warn(f"FluentDateType option {k} is not yet supported") - def format(self, locale): + def format(self, locale: Locale) -> str: if isinstance(self, datetime): selftz = _ensure_datetime_tzinfo(self, tzinfo=self.options.timeZone) else: - selftz = self - - if self.options.dateStyle is None and self.options.timeStyle is None: - return format_date(selftz, format='medium', locale=locale) - elif self.options.dateStyle is None and self.options.timeStyle is not None: - return format_time(selftz, format=self.options.timeStyle, locale=locale) - elif self.options.dateStyle is not None and self.options.timeStyle is None: - return format_date(selftz, format=self.options.dateStyle, locale=locale) - else: - # Both date and time. Logic copied from babel.dates.format_datetime, - # with modifications. - # Which datetime format do we pick? We arbitrarily pick dateStyle. + selftz = cast(datetime, self) + + ds = self.options.dateStyle + ts = self.options.timeStyle + if ds is None: + if ts is None: + return format_date(selftz, format='medium', locale=locale) + else: + return format_time(selftz, format=ts, locale=locale) + elif ts is None: + return format_date(selftz, format=ds, locale=locale) + + # Both date and time. Logic copied from babel.dates.format_datetime, + # with modifications. + # Which datetime format do we pick? We arbitrarily pick dateStyle. - return (get_datetime_format(self.options.dateStyle, locale=locale) - .replace("'", "") - .replace('{0}', format_time(selftz, self.options.timeStyle, tzinfo=None, - locale=locale)) - .replace('{1}', format_date(selftz, self.options.dateStyle, locale=locale)) - ) + return (cast(str, get_datetime_format(ds, locale=locale)) + .replace("'", "") + .replace('{0}', format_time(selftz, ts, tzinfo=None, locale=locale)) + .replace('{1}', format_date(selftz, ds, locale=locale))) -def _ensure_datetime_tzinfo(dt, tzinfo=None): +def _ensure_datetime_tzinfo(dt: datetime, tzinfo: Union[str, None] = None) -> datetime: """ Ensure the datetime passed has an attached tzinfo. """ @@ -323,14 +342,12 @@ def _ensure_datetime_tzinfo(dt, tzinfo=None): dt = dt.replace(tzinfo=pytz.UTC) if tzinfo is not None: dt = dt.astimezone(get_timezone(tzinfo)) - if hasattr(tzinfo, 'normalize'): # pytz - dt = tzinfo.normalize(datetime) return dt class FluentDate(FluentDateType, date): @classmethod - def from_date(cls, dt_obj, **kwargs): + def from_date(cls, dt_obj: date, **kwargs: Any) -> 'FluentDate': obj = cls(dt_obj.year, dt_obj.month, dt_obj.day) obj._init_options(dt_obj, kwargs) return obj @@ -338,7 +355,7 @@ def from_date(cls, dt_obj, **kwargs): class FluentDateTime(FluentDateType, datetime): @classmethod - def from_date_time(cls, dt_obj, **kwargs): + def from_date_time(cls, dt_obj: datetime, **kwargs: Any) -> 'FluentDateTime': obj = cls(dt_obj.year, dt_obj.month, dt_obj.day, dt_obj.hour, dt_obj.minute, dt_obj.second, dt_obj.microsecond, tzinfo=dt_obj.tzinfo) @@ -346,7 +363,10 @@ def from_date_time(cls, dt_obj, **kwargs): return obj -def fluent_date(dt, **kwargs): +def fluent_date( + dt: Union[date, datetime, FluentDateType, FluentNone], + **kwargs: Any +) -> Union[FluentDateType, FluentNone]: if isinstance(dt, FluentDateType) and not kwargs: return dt if isinstance(dt, datetime): diff --git a/fluent.runtime/fluent/runtime/utils.py b/fluent.runtime/fluent/runtime/utils.py index 56a61a35..86d44cec 100644 --- a/fluent.runtime/fluent/runtime/utils.py +++ b/fluent.runtime/fluent/runtime/utils.py @@ -1,7 +1,8 @@ from datetime import date, datetime from decimal import Decimal +from typing import Any, Union -from fluent.syntax.ast import TermReference +from fluent.syntax.ast import MessageReference, TermReference from .types import FluentInt, FluentFloat, FluentDecimal, FluentDate, FluentDateTime from .errors import FluentReferenceError @@ -10,7 +11,7 @@ ATTRIBUTE_SEPARATOR = '.' -def native_to_fluent(val): +def native_to_fluent(val: Any) -> Any: """ Convert a python type to a Fluent Type. """ @@ -28,7 +29,7 @@ def native_to_fluent(val): return val -def reference_to_id(ref): +def reference_to_id(ref: Union[MessageReference, TermReference]) -> str: """ Returns a string reference for a MessageReference or TermReference AST node. @@ -39,6 +40,7 @@ def reference_to_id(ref): -term -term.attr """ + start: str if isinstance(ref, TermReference): start = TERM_SIGIL + ref.id.name else: @@ -49,7 +51,7 @@ def reference_to_id(ref): return start -def unknown_reference_error_obj(ref_id): +def unknown_reference_error_obj(ref_id: str) -> FluentReferenceError: if ATTRIBUTE_SEPARATOR in ref_id: return FluentReferenceError(f"Unknown attribute: {ref_id}") if ref_id.startswith(TERM_SIGIL): diff --git a/fluent.runtime/setup.cfg b/fluent.runtime/setup.cfg index d7fea75c..8186efa9 100644 --- a/fluent.runtime/setup.cfg +++ b/fluent.runtime/setup.cfg @@ -12,3 +12,6 @@ max-line-length=120 line_length=120 skip_glob=.tox not_skip=__init__.py + +[mypy] +strict = True diff --git a/fluent.runtime/setup.py b/fluent.runtime/setup.py index a528f199..41a5e11e 100755 --- a/fluent.runtime/setup.py +++ b/fluent.runtime/setup.py @@ -1,8 +1,8 @@ -from setuptools import setup, find_namespace_packages -import os +from os import path +from setuptools import setup -this_directory = os.path.abspath(os.path.dirname(__file__)) -with open(os.path.join(this_directory, 'README.rst'), 'rb') as f: +this_directory = path.abspath(path.dirname(__file__)) +with open(path.join(this_directory, 'README.rst'), 'rb') as f: long_description = f.read().decode('utf-8') @@ -25,14 +25,15 @@ 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3 :: Only', ], - packages=find_namespace_packages(include=['fluent.*']), + packages=['fluent.runtime'], + package_data={'fluent.runtime': ['py.typed']}, # These should also be duplicated in tox.ini and /.github/workflows/fluent.runtime.yml install_requires=[ 'fluent.syntax>=0.17,<0.20', 'attrs', 'babel', 'pytz', - 'six', + 'typing-extensions>=3.7,<5' ], test_suite='tests', ) diff --git a/fluent.runtime/tests/test_types.py b/fluent.runtime/tests/test_types.py index b1272bfe..4ab356f9 100644 --- a/fluent.runtime/tests/test_types.py +++ b/fluent.runtime/tests/test_types.py @@ -45,7 +45,7 @@ def test_disallow_nonexistant_options(self): fluent_number, 1, not_a_real_option=True, - ) + ) def test_style_validation(self): self.assertRaises(ValueError, @@ -213,14 +213,14 @@ def test_timeStyle_datetime(self): fd = fluent_date(self.a_datetime, timeStyle='short') en_US = Locale.parse('en_US') en_GB = Locale.parse('en_GB') - self.assertEqual(fd.format(en_US), '2:15 PM') + self.assertRegex(fd.format(en_US), '^2:15\\sPM$') self.assertEqual(fd.format(en_GB), '14:15') def test_dateStyle_and_timeStyle_datetime(self): fd = fluent_date(self.a_datetime, timeStyle='short', dateStyle='short') en_US = Locale.parse('en_US') en_GB = Locale.parse('en_GB') - self.assertEqual(fd.format(en_US), '2/1/18, 2:15 PM') + self.assertRegex(fd.format(en_US), '^2/1/18, 2:15\\sPM$') self.assertEqual(fd.format(en_GB), '01/02/2018, 14:15') def test_validate_dateStyle(self): @@ -246,7 +246,7 @@ def test_timeZone(self): fd1 = fluent_date(dt1, dateStyle='short', timeStyle='short') self.assertEqual(fd1.format(en_GB), '02/07/2018, 00:30') fd1b = fluent_date(dt1, dateStyle='full', timeStyle='full') - self.assertEqual(fd1b.format(en_GB), 'Monday, 2 July 2018 at 00:30:00 British Summer Time') + self.assertRegex(fd1b.format(en_GB), '^Monday, 2 July 2018(,| at) 00:30:00 British Summer Time$') fd1c = fluent_date(dt1, dateStyle='short') self.assertEqual(fd1c.format(en_GB), '02/07/2018') fd1d = fluent_date(dt1, timeStyle='short') @@ -259,7 +259,7 @@ def test_timeZone(self): self.assertEqual(fd2.format(en_GB), '02/07/2018, 00:30') fd2b = fluent_date(dt2, dateStyle='full', timeStyle='full', timeZone='Europe/London') - self.assertEqual(fd2b.format(en_GB), 'Monday, 2 July 2018 at 00:30:00 British Summer Time') + self.assertRegex(fd2b.format(en_GB), '^Monday, 2 July 2018(,| at) 00:30:00 British Summer Time$') fd2c = fluent_date(dt2, dateStyle='short', timeZone='Europe/London') self.assertEqual(fd2c.format(en_GB), '02/07/2018') @@ -290,7 +290,7 @@ def test_disallow_nonexistant_options(self): fluent_date, self.a_date, not_a_real_option=True, - ) + ) def test_dont_wrap_unnecessarily(self): f1 = fluent_date(self.a_date) diff --git a/fluent.runtime/tox.ini b/fluent.runtime/tox.ini index 26901620..511cb53c 100644 --- a/fluent.runtime/tox.ini +++ b/fluent.runtime/tox.ini @@ -12,7 +12,7 @@ deps = attrs==19.1.0 babel==2.7.0 pytz==2019.2 - six==1.12.0 + typing-extensions==3.7 syntax: . commands = ./runtests.py @@ -20,4 +20,3 @@ commands = ./runtests.py basepython = python3 deps = . - six diff --git a/fluent.syntax/fluent/syntax/__init__.py b/fluent.syntax/fluent/syntax/__init__.py index 0975b110..1ff31745 100644 --- a/fluent.syntax/fluent/syntax/__init__.py +++ b/fluent.syntax/fluent/syntax/__init__.py @@ -1,15 +1,33 @@ +from typing import Any + +from . import ast +from .errors import ParseError from .parser import FluentParser from .serializer import FluentSerializer +from .stream import FluentParserStream +from .visitor import Transformer, Visitor + +__all__ = [ + 'FluentParser', + 'FluentParserStream', + 'FluentSerializer', + 'ParseError', + 'Transformer', + 'Visitor', + 'ast', + 'parse', + 'serialize' +] -def parse(source, **kwargs): +def parse(source: str, **kwargs: Any) -> ast.Resource: """Create an ast.Resource from a Fluent Syntax source. """ parser = FluentParser(**kwargs) return parser.parse(source) -def serialize(resource, **kwargs): +def serialize(resource: ast.Resource, **kwargs: Any) -> str: """Serialize an ast.Resource to a unicode string. """ serializer = FluentSerializer(**kwargs) diff --git a/fluent.syntax/fluent/syntax/ast.py b/fluent.syntax/fluent/syntax/ast.py index acd5a1f8..d2e48490 100644 --- a/fluent.syntax/fluent/syntax/ast.py +++ b/fluent.syntax/fluent/syntax/ast.py @@ -1,9 +1,13 @@ import re import sys import json +from typing import Any, Callable, Dict, List, TypeVar, Union, cast +Node = TypeVar('Node', bound='BaseNode') +ToJsonFn = Callable[[Dict[str, Any]], Any] -def to_json(value, fn=None): + +def to_json(value: Any, fn: Union[ToJsonFn, None] = None) -> Any: if isinstance(value, BaseNode): return value.to_json(fn) if isinstance(value, list): @@ -14,7 +18,7 @@ def to_json(value, fn=None): return value -def from_json(value): +def from_json(value: Any) -> Any: if isinstance(value, dict): cls = getattr(sys.modules[__name__], value['type']) args = { @@ -29,7 +33,7 @@ def from_json(value): return value -def scalars_equal(node1, node2, ignored_fields): +def scalars_equal(node1: Any, node2: Any, ignored_fields: List[str]) -> bool: """Compare two nodes which are not lists.""" if type(node1) != type(node2): @@ -38,7 +42,7 @@ def scalars_equal(node1, node2, ignored_fields): if isinstance(node1, BaseNode): return node1.equals(node2, ignored_fields) - return node1 == node2 + return cast(bool, node1 == node2) class BaseNode: @@ -48,9 +52,9 @@ class BaseNode: Annotation. Implements __str__, to_json and traverse. """ - def clone(self): + def clone(self: Node) -> Node: """Create a deep clone of the current node.""" - def visit(value): + def visit(value: Any) -> Any: """Clone node and its descendants.""" if isinstance(value, BaseNode): return value.clone() @@ -65,7 +69,7 @@ def visit(value): **{name: visit(value) for name, value in vars(self).items()} ) - def equals(self, other, ignored_fields=['span']): + def equals(self, other: 'BaseNode', ignored_fields: List[str] = ['span']) -> bool: """Compare two nodes. Nodes are deeply compared on a field by field basis. If possible, False @@ -104,7 +108,7 @@ def equals(self, other, ignored_fields=['span']): return True - def to_json(self, fn=None): + def to_json(self, fn: Union[ToJsonFn, None] = None) -> Any: obj = { name: to_json(value, fn) for name, value in vars(self).items() @@ -114,23 +118,23 @@ def to_json(self, fn=None): ) return fn(obj) if fn else obj - def __str__(self): + def __str__(self) -> str: return json.dumps(self.to_json()) class SyntaxNode(BaseNode): """Base class for AST nodes which can have Spans.""" - def __init__(self, span=None, **kwargs): + def __init__(self, span: Union['Span', None] = None, **kwargs: Any): super().__init__(**kwargs) self.span = span - def add_span(self, start, end): + def add_span(self, start: int, end: int) -> None: self.span = Span(start, end) class Resource(SyntaxNode): - def __init__(self, body=None, **kwargs): + def __init__(self, body: Union[List['EntryType'], None] = None, **kwargs: Any): super().__init__(**kwargs) self.body = body or [] @@ -140,8 +144,12 @@ class Entry(SyntaxNode): class Message(Entry): - def __init__(self, id, value=None, attributes=None, - comment=None, **kwargs): + def __init__(self, + id: 'Identifier', + value: Union['Pattern', None] = None, + attributes: Union[List['Attribute'], None] = None, + comment: Union['Comment', None] = None, + **kwargs: Any): super().__init__(**kwargs) self.id = id self.value = value @@ -150,8 +158,8 @@ def __init__(self, id, value=None, attributes=None, class Term(Entry): - def __init__(self, id, value, attributes=None, - comment=None, **kwargs): + def __init__(self, id: 'Identifier', value: 'Pattern', attributes: Union[List['Attribute'], None] = None, + comment: Union['Comment', None] = None, **kwargs: Any): super().__init__(**kwargs) self.id = id self.value = value @@ -160,7 +168,7 @@ def __init__(self, id, value, attributes=None, class Pattern(SyntaxNode): - def __init__(self, elements, **kwargs): + def __init__(self, elements: List[Union['TextElement', 'Placeable']], **kwargs: Any): super().__init__(**kwargs) self.elements = elements @@ -170,13 +178,15 @@ class PatternElement(SyntaxNode): class TextElement(PatternElement): - def __init__(self, value, **kwargs): + def __init__(self, value: str, **kwargs: Any): super().__init__(**kwargs) self.value = value class Placeable(PatternElement): - def __init__(self, expression, **kwargs): + def __init__(self, + expression: Union['InlineExpression', 'Placeable', 'SelectExpression'], + **kwargs: Any): super().__init__(**kwargs) self.expression = expression @@ -187,20 +197,21 @@ class Expression(SyntaxNode): class Literal(Expression): """An abstract base class for literals.""" - def __init__(self, value, **kwargs): + + def __init__(self, value: str, **kwargs: Any): super().__init__(**kwargs) self.value = value - def parse(self): + def parse(self) -> Dict[str, Any]: return {'value': self.value} class StringLiteral(Literal): - def parse(self): - def from_escape_sequence(matchobj): + def parse(self) -> Dict[str, str]: + def from_escape_sequence(matchobj: Any) -> str: c, codepoint4, codepoint6 = matchobj.groups() if c: - return c + return cast(str, c) codepoint = int(codepoint4 or codepoint6, 16) if codepoint <= 0xD7FF or 0xE000 <= codepoint: return chr(codepoint) @@ -218,7 +229,7 @@ def from_escape_sequence(matchobj): class NumberLiteral(Literal): - def parse(self): + def parse(self) -> Dict[str, Union[float, int]]: value = float(self.value) decimal_position = self.value.find('.') precision = 0 @@ -231,14 +242,18 @@ def parse(self): class MessageReference(Expression): - def __init__(self, id, attribute=None, **kwargs): + def __init__(self, id: 'Identifier', attribute: Union['Identifier', None] = None, **kwargs: Any): super().__init__(**kwargs) self.id = id self.attribute = attribute class TermReference(Expression): - def __init__(self, id, attribute=None, arguments=None, **kwargs): + def __init__(self, + id: 'Identifier', + attribute: Union['Identifier', None] = None, + arguments: Union['CallArguments', None] = None, + **kwargs: Any): super().__init__(**kwargs) self.id = id self.attribute = attribute @@ -246,41 +261,44 @@ def __init__(self, id, attribute=None, arguments=None, **kwargs): class VariableReference(Expression): - def __init__(self, id, **kwargs): + def __init__(self, id: 'Identifier', **kwargs: Any): super().__init__(**kwargs) self.id = id class FunctionReference(Expression): - def __init__(self, id, arguments, **kwargs): + def __init__(self, id: 'Identifier', arguments: 'CallArguments', **kwargs: Any): super().__init__(**kwargs) self.id = id self.arguments = arguments class SelectExpression(Expression): - def __init__(self, selector, variants, **kwargs): + def __init__(self, selector: 'InlineExpression', variants: List['Variant'], **kwargs: Any): super().__init__(**kwargs) self.selector = selector self.variants = variants class CallArguments(SyntaxNode): - def __init__(self, positional=None, named=None, **kwargs): + def __init__(self, + positional: Union[List[Union['InlineExpression', Placeable]], None] = None, + named: Union[List['NamedArgument'], None] = None, + **kwargs: Any): super().__init__(**kwargs) self.positional = [] if positional is None else positional self.named = [] if named is None else named class Attribute(SyntaxNode): - def __init__(self, id, value, **kwargs): + def __init__(self, id: 'Identifier', value: Pattern, **kwargs: Any): super().__init__(**kwargs) self.id = id self.value = value class Variant(SyntaxNode): - def __init__(self, key, value, default=False, **kwargs): + def __init__(self, key: Union['Identifier', NumberLiteral], value: Pattern, default: bool = False, **kwargs: Any): super().__init__(**kwargs) self.key = key self.value = value @@ -288,59 +306,71 @@ def __init__(self, key, value, default=False, **kwargs): class NamedArgument(SyntaxNode): - def __init__(self, name, value, **kwargs): + def __init__(self, name: 'Identifier', value: Union[NumberLiteral, StringLiteral], **kwargs: Any): super().__init__(**kwargs) self.name = name self.value = value class Identifier(SyntaxNode): - def __init__(self, name, **kwargs): + def __init__(self, name: str, **kwargs: Any): super().__init__(**kwargs) self.name = name class BaseComment(Entry): - def __init__(self, content=None, **kwargs): + def __init__(self, content: Union[str, None] = None, **kwargs: Any): super().__init__(**kwargs) self.content = content class Comment(BaseComment): - def __init__(self, content=None, **kwargs): + def __init__(self, content: Union[str, None] = None, **kwargs: Any): super().__init__(content, **kwargs) class GroupComment(BaseComment): - def __init__(self, content=None, **kwargs): + def __init__(self, content: Union[str, None] = None, **kwargs: Any): super().__init__(content, **kwargs) class ResourceComment(BaseComment): - def __init__(self, content=None, **kwargs): + def __init__(self, content: Union[str, None] = None, **kwargs: Any): super().__init__(content, **kwargs) class Junk(SyntaxNode): - def __init__(self, content=None, annotations=None, **kwargs): + def __init__(self, + content: Union[str, None] = None, + annotations: Union[List['Annotation'], None] = None, + **kwargs: Any): super().__init__(**kwargs) self.content = content self.annotations = annotations or [] - def add_annotation(self, annot): + def add_annotation(self, annot: 'Annotation') -> None: self.annotations.append(annot) class Span(BaseNode): - def __init__(self, start, end, **kwargs): + def __init__(self, start: int, end: int, **kwargs: Any): super().__init__(**kwargs) self.start = start self.end = end class Annotation(SyntaxNode): - def __init__(self, code, arguments=None, message=None, **kwargs): + def __init__(self, + code: str, + arguments: Union[List[Any], None] = None, + message: Union[str, None] = None, + **kwargs: Any): super().__init__(**kwargs) self.code = code self.arguments = arguments or [] self.message = message + + +EntryType = Union[Message, Term, Comment, GroupComment, ResourceComment, Junk] +InlineExpression = Union[NumberLiteral, StringLiteral, MessageReference, + TermReference, VariableReference, FunctionReference] diff --git a/fluent.syntax/fluent/syntax/errors.py b/fluent.syntax/fluent/syntax/errors.py index 36859d20..01037482 100644 --- a/fluent.syntax/fluent/syntax/errors.py +++ b/fluent.syntax/fluent/syntax/errors.py @@ -1,11 +1,14 @@ +from typing import Tuple, Union + + class ParseError(Exception): - def __init__(self, code, *args): + def __init__(self, code: str, *args: Union[str, None]): self.code = code self.args = args self.message = get_error_message(code, args) -def get_error_message(code, args): +def get_error_message(code: str, args: Tuple[Union[str, None], ...]) -> str: if code == 'E00001': return 'Generic error' if code == 'E0002': diff --git a/fluent.syntax/fluent/syntax/parser.py b/fluent.syntax/fluent/syntax/parser.py index fb296368..b9771b43 100644 --- a/fluent.syntax/fluent/syntax/parser.py +++ b/fluent.syntax/fluent/syntax/parser.py @@ -1,11 +1,14 @@ import re +from typing import Any, Callable, List, Set, TypeVar, Union, cast from . import ast -from .stream import EOF, EOL, FluentParserStream +from .stream import EOL, FluentParserStream from .errors import ParseError +R = TypeVar("R", bound=ast.SyntaxNode) -def with_span(fn): - def decorated(self, ps, *args, **kwargs): + +def with_span(fn: Callable[..., R]) -> Callable[..., R]: + def decorated(self: 'FluentParser', ps: FluentParserStream, *args: Any, **kwargs: Any) -> Any: if not self.with_spans: return fn(self, ps, *args, **kwargs) @@ -30,16 +33,17 @@ class FluentParser: ``with_spans`` enables source information in the form of :class:`.ast.Span` objects for each :class:`.ast.SyntaxNode`. """ - def __init__(self, with_spans=True): + + def __init__(self, with_spans: bool = True): self.with_spans = with_spans - def parse(self, source): + def parse(self, source: str) -> ast.Resource: """Create a :class:`.ast.Resource` from a Fluent source. """ ps = FluentParserStream(source) ps.skip_blank_block() - entries = [] + entries: List[ast.EntryType] = [] last_comment = None while ps.current_char: @@ -62,7 +66,7 @@ def parse(self, source): if isinstance(entry, (ast.Message, ast.Term)): entry.comment = last_comment if self.with_spans: - entry.span.start = entry.comment.span.start + cast(ast.Span, entry.span).start = cast(ast.Span, entry.comment.span).start else: entries.append(last_comment) # In either case, the stashed comment has been dealt with; @@ -78,7 +82,7 @@ def parse(self, source): return res - def parse_entry(self, source): + def parse_entry(self, source: str) -> ast.EntryType: """Parse the first :class:`.ast.Entry` in source. Skip all encountered comments and start parsing at the first :class:`.ast.Message` @@ -99,7 +103,7 @@ def parse_entry(self, source): return self.get_entry_or_junk(ps) - def get_entry_or_junk(self, ps): + def get_entry_or_junk(self, ps: FluentParserStream) -> ast.EntryType: entry_start_pos = ps.index try: @@ -119,12 +123,12 @@ def get_entry_or_junk(self, ps): junk = ast.Junk(slice) if self.with_spans: junk.add_span(entry_start_pos, next_entry_start) - annot = ast.Annotation(err.code, err.args, err.message) + annot = ast.Annotation(err.code, list(err.args) if err.args else None, err.message) annot.add_span(error_index, error_index) junk.add_annotation(annot) return junk - def get_entry(self, ps): + def get_entry(self, ps: FluentParserStream) -> ast.EntryType: if ps.current_char == '#': return self.get_comment(ps) @@ -137,7 +141,7 @@ def get_entry(self, ps): raise ParseError('E0002') @with_span - def get_comment(self, ps): + def get_comment(self, ps: FluentParserStream) -> Union[ast.Comment, ast.GroupComment, ast.ResourceComment]: # 0 - comment # 1 - group comment # 2 - resource comment @@ -162,7 +166,7 @@ def get_comment(self, ps): ch = ps.take_char(lambda x: x != EOL) if ps.is_next_line_comment(level=level): - content += ps.current_char + content += cast(str, ps.current_char) ps.next() else: break @@ -174,8 +178,11 @@ def get_comment(self, ps): elif level == 2: return ast.ResourceComment(content) + # never happens if ps.current_char == '#' when called + return cast(ast.Comment, None) + @with_span - def get_message(self, ps): + def get_message(self, ps: FluentParserStream) -> ast.Message: id = self.get_identifier(ps) ps.skip_blank_inline() ps.expect_char('=') @@ -189,7 +196,7 @@ def get_message(self, ps): return ast.Message(id, value, attrs) @with_span - def get_term(self, ps): + def get_term(self, ps: FluentParserStream) -> ast.Term: ps.expect_char('-') id = self.get_identifier(ps) @@ -204,7 +211,7 @@ def get_term(self, ps): return ast.Term(id, value, attrs) @with_span - def get_attribute(self, ps): + def get_attribute(self, ps: FluentParserStream) -> ast.Attribute: ps.expect_char('.') key = self.get_identifier(ps) @@ -218,8 +225,8 @@ def get_attribute(self, ps): return ast.Attribute(key, value) - def get_attributes(self, ps): - attrs = [] + def get_attributes(self, ps: FluentParserStream) -> List[ast.Attribute]: + attrs: List[ast.Attribute] = [] ps.peek_blank() while ps.is_attribute_start(): @@ -231,8 +238,11 @@ def get_attributes(self, ps): return attrs @with_span - def get_identifier(self, ps): + def get_identifier(self, ps: FluentParserStream) -> ast.Identifier: name = ps.take_id_start() + if name is None: + raise ParseError('E0004', 'a-zA-Z') + ch = ps.take_id_char() while ch: name += ch @@ -240,10 +250,10 @@ def get_identifier(self, ps): return ast.Identifier(name) - def get_variant_key(self, ps): + def get_variant_key(self, ps: FluentParserStream) -> Union[ast.Identifier, ast.NumberLiteral]: ch = ps.current_char - if ch is EOF: + if ch is None: raise ParseError('E0013') cc = ord(ch) @@ -253,7 +263,7 @@ def get_variant_key(self, ps): return self.get_identifier(ps) @with_span - def get_variant(self, ps, has_default): + def get_variant(self, ps: FluentParserStream, has_default: bool) -> ast.Variant: default_index = False if ps.current_char == '*': @@ -276,8 +286,8 @@ def get_variant(self, ps, has_default): return ast.Variant(key, value, default_index) - def get_variants(self, ps): - variants = [] + def get_variants(self, ps: FluentParserStream) -> List[ast.Variant]: + variants: List[ast.Variant] = [] has_default = False ps.skip_blank() @@ -299,7 +309,7 @@ def get_variants(self, ps): return variants - def get_digits(self, ps): + def get_digits(self, ps: FluentParserStream) -> str: num = '' ch = ps.take_digit() @@ -313,7 +323,7 @@ def get_digits(self, ps): return num @with_span - def get_number(self, ps): + def get_number(self, ps: FluentParserStream) -> ast.NumberLiteral: num = '' if ps.current_char == '-': @@ -329,7 +339,7 @@ def get_number(self, ps): return ast.NumberLiteral(num) - def maybe_get_pattern(self, ps): + def maybe_get_pattern(self, ps: FluentParserStream) -> Union[ast.Pattern, None]: '''Parse an inline or a block Pattern, or None maybe_get_pattern distinguishes between patterns which start on the @@ -352,8 +362,8 @@ def maybe_get_pattern(self, ps): return None @with_span - def get_pattern(self, ps, is_block): - elements = [] + def get_pattern(self, ps: FluentParserStream, is_block: bool) -> ast.Pattern: + elements: List[Any] = [] if is_block: # A block pattern is a pattern which starts on a new line. Measure # the indent of this first line for the dedentation logic. @@ -362,7 +372,8 @@ def get_pattern(self, ps, is_block): elements.append(self.Indent(first_indent, blank_start, ps.index)) common_indent_length = len(first_indent) else: - common_indent_length = float('infinity') + # Should get fixed by the subsequent min() operation + common_indent_length = cast(int, float('infinity')) while ps.current_char: if ps.current_char == EOL: @@ -383,6 +394,7 @@ def get_pattern(self, ps, is_block): if ps.current_char == '}': raise ParseError('E0027') + element: Union[ast.TextElement, ast.Placeable] if ps.current_char == '{': element = self.get_placeable(ps) else: @@ -394,17 +406,20 @@ def get_pattern(self, ps, is_block): return ast.Pattern(dedented) class Indent(ast.SyntaxNode): - def __init__(self, value, start, end): + def __init__(self, value: str, start: int, end: int): super(FluentParser.Indent, self).__init__() self.value = value self.add_span(start, end) - def dedent(self, elements, common_indent): + def dedent(self, + elements: List[Union[ast.TextElement, ast.Placeable, Indent]], + common_indent: int + ) -> List[Union[ast.TextElement, ast.Placeable]]: '''Dedent a list of elements by removing the maximum common indent from the beginning of text lines. The common indent is calculated in get_pattern. ''' - trimmed = [] + trimmed: List[Union[ast.TextElement, ast.Placeable]] = [] for element in elements: if isinstance(element, ast.Placeable): @@ -422,7 +437,7 @@ def dedent(self, elements, common_indent): # Join adjacent TextElements by replacing them with their sum. sum = ast.TextElement(prev.value + element.value) if self.with_spans: - sum.add_span(prev.span.start, element.span.end) + sum.add_span(cast(ast.Span, prev.span).start, cast(ast.Span, element.span).end) trimmed[-1] = sum continue @@ -431,7 +446,7 @@ def dedent(self, elements, common_indent): # TextElements, convert it into a new TextElement. text_element = ast.TextElement(element.value) if self.with_spans: - text_element.add_span(element.span.start, element.span.end) + text_element.add_span(cast(ast.Span, element.span).start, cast(ast.Span, element.span).end) element = text_element trimmed.append(element) @@ -446,7 +461,7 @@ def dedent(self, elements, common_indent): return trimmed @with_span - def get_text_element(self, ps): + def get_text_element(self, ps: FluentParserStream) -> ast.TextElement: buf = '' while ps.current_char: @@ -463,7 +478,7 @@ def get_text_element(self, ps): return ast.TextElement(buf) - def get_escape_sequence(self, ps): + def get_escape_sequence(self, ps: FluentParserStream) -> str: next = ps.current_char if next == '\\' or next == '"': @@ -478,7 +493,7 @@ def get_escape_sequence(self, ps): raise ParseError('E0025', next) - def get_unicode_escape_sequence(self, ps, u, digits): + def get_unicode_escape_sequence(self, ps: FluentParserStream, u: str, digits: int) -> str: ps.expect_char(u) sequence = '' for _ in range(digits): @@ -490,7 +505,7 @@ def get_unicode_escape_sequence(self, ps, u, digits): return f'\\{u}{sequence}' @with_span - def get_placeable(self, ps): + def get_placeable(self, ps: FluentParserStream) -> ast.Placeable: ps.expect_char('{') ps.skip_blank() expression = self.get_expression(ps) @@ -498,7 +513,9 @@ def get_placeable(self, ps): return ast.Placeable(expression) @with_span - def get_expression(self, ps): + def get_expression(self, ps: FluentParserStream) -> Union[ast.InlineExpression, + ast.Placeable, + ast.SelectExpression]: selector = self.get_inline_expression(ps) ps.skip_blank() @@ -547,7 +564,7 @@ def get_expression(self, ps): return selector @with_span - def get_inline_expression(self, ps): + def get_inline_expression(self, ps: FluentParserStream) -> Union[ast.InlineExpression, ast.Placeable]: if ps.current_char == '{': return self.get_placeable(ps) @@ -598,7 +615,9 @@ def get_inline_expression(self, ps): raise ParseError('E0028') @with_span - def get_call_argument(self, ps): + def get_call_argument(self, + ps: FluentParserStream + ) -> Union[ast.InlineExpression, ast.NamedArgument, ast.Placeable]: exp = self.get_inline_expression(ps) ps.skip_blank() @@ -616,10 +635,10 @@ def get_call_argument(self, ps): raise ParseError('E0009') @with_span - def get_call_arguments(self, ps): - positional = [] - named = [] - argument_names = set() + def get_call_arguments(self, ps: FluentParserStream) -> ast.CallArguments: + positional: List[Union[ast.InlineExpression, ast.Placeable]] = [] + named: List[ast.NamedArgument] = [] + argument_names: Set[str] = set() ps.expect_char('(') ps.skip_blank() @@ -652,7 +671,7 @@ def get_call_arguments(self, ps): return ast.CallArguments(positional, named) @with_span - def get_string(self, ps): + def get_string(self, ps: FluentParserStream) -> ast.StringLiteral: value = '' ps.expect_char('"') @@ -674,7 +693,7 @@ def get_string(self, ps): return ast.StringLiteral(value) @with_span - def get_literal(self, ps): + def get_literal(self, ps: FluentParserStream) -> Union[ast.NumberLiteral, ast.StringLiteral]: if ps.is_number_start(): return self.get_number(ps) if ps.current_char == '"': diff --git a/fluent.syntax/fluent/syntax/py.typed b/fluent.syntax/fluent/syntax/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/fluent.syntax/fluent/syntax/serializer.py b/fluent.syntax/fluent/syntax/serializer.py index b6e16b1c..68ea89b3 100644 --- a/fluent.syntax/fluent/syntax/serializer.py +++ b/fluent.syntax/fluent/syntax/serializer.py @@ -1,23 +1,24 @@ +from typing import List, Union from . import ast -def indent_except_first_line(content): +def indent_except_first_line(content: str) -> str: return " ".join( content.splitlines(True) ) -def includes_new_line(elem): +def includes_new_line(elem: Union[ast.TextElement, ast.Placeable]) -> bool: return isinstance(elem, ast.TextElement) and "\n" in elem.value -def is_select_expr(elem): +def is_select_expr(elem: Union[ast.TextElement, ast.Placeable]) -> bool: return ( isinstance(elem, ast.Placeable) and isinstance(elem.expression, ast.SelectExpression)) -def should_start_on_new_line(pattern): +def should_start_on_new_line(pattern: ast.Pattern) -> bool: is_multiline = any(is_select_expr(elem) for elem in pattern.elements) \ or any(includes_new_line(elem) for elem in pattern.elements) @@ -38,17 +39,17 @@ class FluentSerializer: """ HAS_ENTRIES = 1 - def __init__(self, with_junk=False): + def __init__(self, with_junk: bool = False): self.with_junk = with_junk - def serialize(self, resource): + def serialize(self, resource: ast.Resource) -> str: "Serialize a :class:`.ast.Resource` to a string." if not isinstance(resource, ast.Resource): raise Exception('Unknown resource type: {}'.format(type(resource))) state = 0 - parts = [] + parts: List[str] = [] for entry in resource.body: if not isinstance(entry, ast.Junk) or self.with_junk: parts.append(self.serialize_entry(entry, state)) @@ -57,7 +58,7 @@ def serialize(self, resource): return "".join(parts) - def serialize_entry(self, entry, state=0): + def serialize_entry(self, entry: ast.EntryType, state: int = 0) -> str: "Serialize an :class:`.ast.Entry` to a string." if isinstance(entry, ast.Message): return serialize_message(entry) @@ -80,7 +81,10 @@ def serialize_entry(self, entry, state=0): raise Exception('Unknown entry type: {}'.format(type(entry))) -def serialize_comment(comment, prefix="#"): +def serialize_comment(comment: Union[ast.Comment, ast.GroupComment, ast.ResourceComment], prefix: str = "#") -> str: + if not comment.content: + return f'{prefix}\n' + prefixed = "\n".join([ prefix if len(line) == 0 else f"{prefix} {line}" for line in comment.content.split("\n") @@ -89,12 +93,12 @@ def serialize_comment(comment, prefix="#"): return f'{prefixed}\n' -def serialize_junk(junk): - return junk.content +def serialize_junk(junk: ast.Junk) -> str: + return junk.content or '' -def serialize_message(message): - parts = [] +def serialize_message(message: ast.Message) -> str: + parts: List[str] = [] if message.comment: parts.append(serialize_comment(message.comment)) @@ -112,8 +116,8 @@ def serialize_message(message): return ''.join(parts) -def serialize_term(term): - parts = [] +def serialize_term(term: ast.Term) -> str: + parts: List[str] = [] if term.comment: parts.append(serialize_comment(term.comment)) @@ -129,14 +133,14 @@ def serialize_term(term): return ''.join(parts) -def serialize_attribute(attribute): +def serialize_attribute(attribute: ast.Attribute) -> str: return "\n .{} ={}".format( attribute.id.name, indent_except_first_line(serialize_pattern(attribute.value)) ) -def serialize_pattern(pattern): +def serialize_pattern(pattern: ast.Pattern) -> str: content = "".join(serialize_element(elem) for elem in pattern.elements) content = indent_except_first_line(content) @@ -146,7 +150,7 @@ def serialize_pattern(pattern): return f' {content}' -def serialize_element(element): +def serialize_element(element: ast.PatternElement) -> str: if isinstance(element, ast.TextElement): return element.value if isinstance(element, ast.Placeable): @@ -154,7 +158,7 @@ def serialize_element(element): raise Exception('Unknown element type: {}'.format(type(element))) -def serialize_placeable(placeable): +def serialize_placeable(placeable: ast.Placeable) -> str: expr = placeable.expression if isinstance(expr, ast.Placeable): return "{{{}}}".format(serialize_placeable(expr)) @@ -164,9 +168,10 @@ def serialize_placeable(placeable): return "{{ {}}}".format(serialize_expression(expr)) if isinstance(expr, ast.Expression): return "{{ {} }}".format(serialize_expression(expr)) + raise Exception('Unknown expression type: {}'.format(type(expr))) -def serialize_expression(expression): +def serialize_expression(expression: Union[ast.Expression, ast.Placeable]) -> str: if isinstance(expression, ast.StringLiteral): return f'"{expression.value}"' if isinstance(expression, ast.NumberLiteral): @@ -199,7 +204,7 @@ def serialize_expression(expression): raise Exception('Unknown expression type: {}'.format(type(expression))) -def serialize_variant(variant): +def serialize_variant(variant: ast.Variant) -> str: return "\n{}[{}]{}".format( " *" if variant.default else " ", serialize_variant_key(variant.key), @@ -207,7 +212,7 @@ def serialize_variant(variant): ) -def serialize_call_arguments(expr): +def serialize_call_arguments(expr: ast.CallArguments) -> str: positional = ", ".join( serialize_expression(arg) for arg in expr.positional) named = ", ".join( @@ -217,14 +222,14 @@ def serialize_call_arguments(expr): return '({})'.format(positional or named) -def serialize_named_argument(arg): +def serialize_named_argument(arg: ast.NamedArgument) -> str: return "{}: {}".format( arg.name.name, serialize_expression(arg.value) ) -def serialize_variant_key(key): +def serialize_variant_key(key: Union[ast.Identifier, ast.NumberLiteral]) -> str: if isinstance(key, ast.Identifier): return key.name if isinstance(key, ast.NumberLiteral): diff --git a/fluent.syntax/fluent/syntax/stream.py b/fluent.syntax/fluent/syntax/stream.py index 6b38179e..150ac933 100644 --- a/fluent.syntax/fluent/syntax/stream.py +++ b/fluent.syntax/fluent/syntax/stream.py @@ -1,19 +1,21 @@ +from typing import Callable, Union +from typing_extensions import Literal from .errors import ParseError class ParserStream: - def __init__(self, string): + def __init__(self, string: str): self.string = string self.index = 0 self.peek_offset = 0 - def get(self, offset): + def get(self, offset: int) -> Union[str, None]: try: return self.string[offset] except IndexError: return None - def char_at(self, offset): + def char_at(self, offset: int) -> Union[str, None]: # When the cursor is at CRLF, return LF but don't move the cursor. The # cursor still points to the EOL position, which in this case is the # beginning of the compound CRLF sequence. This ensures slices of @@ -25,14 +27,14 @@ def char_at(self, offset): return self.get(offset) @property - def current_char(self): + def current_char(self) -> Union[str, None]: return self.char_at(self.index) @property - def current_peek(self): + def current_peek(self) -> Union[str, None]: return self.char_at(self.index + self.peek_offset) - def next(self): + def next(self) -> Union[str, None]: self.peek_offset = 0 # Skip over CRLF as if it was a single character. if self.get(self.index) == '\r' \ @@ -41,7 +43,7 @@ def next(self): self.index += 1 return self.get(self.index) - def peek(self): + def peek(self) -> Union[str, None]: # Skip over CRLF as if it was a single character. if self.get(self.index + self.peek_offset) == '\r' \ and self.get(self.index + self.peek_offset + 1) == '\n': @@ -49,10 +51,10 @@ def peek(self): self.peek_offset += 1 return self.get(self.index + self.peek_offset) - def reset_peek(self, offset=0): + def reset_peek(self, offset: int = 0) -> None: self.peek_offset = offset - def skip_to_peek(self): + def skip_to_peek(self) -> None: self.index += self.peek_offset self.peek_offset = 0 @@ -64,18 +66,18 @@ def skip_to_peek(self): class FluentParserStream(ParserStream): - def peek_blank_inline(self): + def peek_blank_inline(self) -> str: start = self.index + self.peek_offset while self.current_peek == ' ': self.peek() return self.string[start:self.index + self.peek_offset] - def skip_blank_inline(self): + def skip_blank_inline(self) -> str: blank = self.peek_blank_inline() self.skip_to_peek() return blank - def peek_blank_block(self): + def peek_blank_block(self) -> str: blank = "" while True: line_start = self.peek_offset @@ -94,27 +96,27 @@ def peek_blank_block(self): self.reset_peek(line_start) return blank - def skip_blank_block(self): + def skip_blank_block(self) -> str: blank = self.peek_blank_block() self.skip_to_peek() return blank - def peek_blank(self): + def peek_blank(self) -> None: while self.current_peek in (" ", EOL): self.peek() - def skip_blank(self): + def skip_blank(self) -> None: self.peek_blank() self.skip_to_peek() - def expect_char(self, ch): + def expect_char(self, ch: str) -> Literal[True]: if self.current_char == ch: self.next() return True raise ParseError('E0003', ch) - def expect_line_end(self): + def expect_line_end(self) -> Literal[True]: if self.current_char is EOF: # EOF is a valid line end in Fluent. return True @@ -126,29 +128,29 @@ def expect_line_end(self): # Unicode Character 'SYMBOL FOR NEWLINE' (U+2424) raise ParseError('E0003', '\u2424') - def take_char(self, f): + def take_char(self, f: Callable[[str], bool]) -> Union[str, Literal[False], None]: ch = self.current_char - if ch is EOF: + if ch is None: return EOF if f(ch): self.next() return ch return False - def is_char_id_start(self, ch): - if ch is EOF: + def is_char_id_start(self, ch: Union[str, None]) -> bool: + if ch is None: return False cc = ord(ch) return (cc >= 97 and cc <= 122) or \ (cc >= 65 and cc <= 90) - def is_identifier_start(self): + def is_identifier_start(self) -> bool: return self.is_char_id_start(self.current_peek) - def is_number_start(self): + def is_number_start(self) -> bool: ch = self.peek() if self.current_char == '-' else self.current_char - if ch is EOF: + if ch is None: self.reset_peek() return False @@ -157,17 +159,17 @@ def is_number_start(self): self.reset_peek() return is_digit - def is_char_pattern_continuation(self, ch): + def is_char_pattern_continuation(self, ch: Union[str, None]) -> bool: if ch is EOF: return False return ch not in SPECIAL_LINE_START_CHARS - def is_value_start(self): + def is_value_start(self) -> bool: # Inline Patterns may start with any char. return self.current_peek is not EOF and self.current_peek != EOL - def is_value_continuation(self): + def is_value_continuation(self) -> bool: column1 = self.peek_offset self.peek_blank_inline() @@ -188,7 +190,7 @@ def is_value_continuation(self): # 0 - comment # 1 - group comment # 2 - resource comment - def is_next_line_comment(self, level=-1): + def is_next_line_comment(self, level: int = -1) -> bool: if self.current_peek != EOL: return False @@ -210,7 +212,7 @@ def is_next_line_comment(self, level=-1): self.reset_peek() return False - def is_variant_start(self): + def is_variant_start(self) -> bool: current_peek_offset = self.peek_offset if self.current_peek == '*': self.peek() @@ -221,10 +223,10 @@ def is_variant_start(self): self.reset_peek(current_peek_offset) return False - def is_attribute_start(self): + def is_attribute_start(self) -> bool: return self.current_peek == '.' - def skip_to_next_entry_start(self, junk_start): + def skip_to_next_entry_start(self, junk_start: int) -> None: last_newline = self.string.rfind(EOL, 0, self.index) if junk_start < last_newline: # Last seen newline is _after_ the junk start. It's safe to rewind @@ -248,7 +250,7 @@ def skip_to_next_entry_start(self, junk_start): if (first, peek) == ('/', '/') or (first, peek) == ('[', '['): break - def take_id_start(self): + def take_id_start(self) -> Union[str, None]: if self.is_char_id_start(self.current_char): ret = self.current_char self.next() @@ -256,8 +258,8 @@ def take_id_start(self): raise ParseError('E0004', 'a-zA-Z') - def take_id_char(self): - def closure(ch): + def take_id_char(self) -> Union[str, Literal[False], None]: + def closure(ch: str) -> bool: cc = ord(ch) return ((cc >= 97 and cc <= 122) or (cc >= 65 and cc <= 90) or @@ -265,14 +267,14 @@ def closure(ch): cc == 95 or cc == 45) return self.take_char(closure) - def take_digit(self): - def closure(ch): + def take_digit(self) -> Union[str, Literal[False], None]: + def closure(ch: str) -> bool: cc = ord(ch) return (cc >= 48 and cc <= 57) return self.take_char(closure) - def take_hex_digit(self): - def closure(ch): + def take_hex_digit(self) -> Union[str, Literal[False], None]: + def closure(ch: str) -> bool: cc = ord(ch) return ( (cc >= 48 and cc <= 57) # 0-9 diff --git a/fluent.syntax/fluent/syntax/visitor.py b/fluent.syntax/fluent/syntax/visitor.py index cae5baac..0df9f596 100644 --- a/fluent.syntax/fluent/syntax/visitor.py +++ b/fluent.syntax/fluent/syntax/visitor.py @@ -1,4 +1,5 @@ -from .ast import BaseNode +from typing import Any, List +from .ast import BaseNode, Node class Visitor: @@ -11,7 +12,8 @@ class Visitor: If you want to still descend into the children of the node, call `generic_visit` of the superclass. ''' - def visit(self, node): + + def visit(self, node: Any) -> None: if isinstance(node, list): for child in node: self.visit(child) @@ -22,8 +24,8 @@ def visit(self, node): visit = getattr(self, f'visit_{nodename}', self.generic_visit) visit(node) - def generic_visit(self, node): - for propname, propvalue in vars(node).items(): + def generic_visit(self, node: BaseNode) -> None: + for propvalue in vars(node).values(): self.visit(propvalue) @@ -35,7 +37,8 @@ class Transformer(Visitor): If you need to keep the original AST around, pass a `node.clone()` to the transformer. ''' - def visit(self, node): + + def visit(self, node: Any) -> Any: if not isinstance(node, BaseNode): return node @@ -43,10 +46,10 @@ def visit(self, node): visit = getattr(self, f'visit_{nodename}', self.generic_visit) return visit(node) - def generic_visit(self, node): + def generic_visit(self, node: Node) -> Node: # type: ignore for propname, propvalue in vars(node).items(): if isinstance(propvalue, list): - new_vals = [] + new_vals: List[Any] = [] for child in propvalue: new_val = self.visit(child) if new_val is not None: diff --git a/fluent.syntax/setup.cfg b/fluent.syntax/setup.cfg index 53b3d312..323007ae 100644 --- a/fluent.syntax/setup.cfg +++ b/fluent.syntax/setup.cfg @@ -12,3 +12,6 @@ max-line-length=120 line_length=120 skip_glob=.tox not_skip=__init__.py + +[mypy] +strict = True diff --git a/fluent.syntax/setup.py b/fluent.syntax/setup.py index a703ef40..23c17064 100644 --- a/fluent.syntax/setup.py +++ b/fluent.syntax/setup.py @@ -1,8 +1,8 @@ -from setuptools import setup, find_namespace_packages -import os +from os import path +from setuptools import setup -this_directory = os.path.abspath(os.path.dirname(__file__)) -with open(os.path.join(this_directory, 'README.rst'), 'rb') as f: +this_directory = path.abspath(path.dirname(__file__)) +with open(path.join(this_directory, 'README.rst'), 'rb') as f: long_description = f.read().decode('utf-8') setup(name='fluent.syntax', @@ -24,6 +24,10 @@ 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3 :: Only', ], - packages=find_namespace_packages(include=['fluent.*']), + packages=['fluent.syntax'], + package_data={'fluent.syntax': ['py.typed']}, + install_requires=[ + 'typing-extensions>=3.7,<5' + ], test_suite='tests.syntax' ) diff --git a/fluent.syntax/tox.ini b/fluent.syntax/tox.ini index 2ed33bdb..db0448bc 100644 --- a/fluent.syntax/tox.ini +++ b/fluent.syntax/tox.ini @@ -7,4 +7,6 @@ skipsdist=True [testenv] setenv = PYTHONPATH = {toxinidir} +deps = + typing-extensions==3.7 commands = ./runtests.py