diff --git a/end_to_end_tests/golden-record/my_test_api_client/models/__init__.py b/end_to_end_tests/golden-record/my_test_api_client/models/__init__.py index 5bbd77d7d..adc00cd2b 100644 --- a/end_to_end_tests/golden-record/my_test_api_client/models/__init__.py +++ b/end_to_end_tests/golden-record/my_test_api_client/models/__init__.py @@ -1,6 +1,9 @@ """ Contains all the data models used in inputs/outputs """ from .a_model import AModel +from .a_model_with_direct_self_reference_property import AModelWithDirectSelfReferenceProperty +from .a_model_with_indirect_reference_property import AModelWithIndirectReferenceProperty +from .a_model_with_indirect_self_reference_property import AModelWithIndirectSelfReferenceProperty from .all_of_sub_model import AllOfSubModel from .an_all_of_enum import AnAllOfEnum from .an_enum import AnEnum diff --git a/end_to_end_tests/golden-record/my_test_api_client/models/a_model_with_direct_self_reference_property.py b/end_to_end_tests/golden-record/my_test_api_client/models/a_model_with_direct_self_reference_property.py new file mode 100644 index 000000000..3f11569fd --- /dev/null +++ b/end_to_end_tests/golden-record/my_test_api_client/models/a_model_with_direct_self_reference_property.py @@ -0,0 +1,63 @@ +from typing import Any, Dict, List, Type, TypeVar, Union + +import attr + +from ..types import UNSET, Unset + +T = TypeVar("T", bound="AModelWithDirectSelfReferenceProperty") + + +@attr.s(auto_attribs=True) +class AModelWithDirectSelfReferenceProperty: + """ """ + + required_self_ref: AModelWithDirectSelfReferenceProperty + optional_self_ref: Union[Unset, AModelWithDirectSelfReferenceProperty] = UNSET + additional_properties: Dict[str, Any] = attr.ib(init=False, factory=dict) + + def to_dict(self) -> Dict[str, Any]: + required_self_ref = self.required_self_ref + optional_self_ref = self.optional_self_ref + + field_dict: Dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "required_self_ref": required_self_ref, + } + ) + if optional_self_ref is not UNSET: + field_dict["optional_self_ref"] = optional_self_ref + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + d = src_dict.copy() + required_self_ref = d.pop("required_self_ref") + + optional_self_ref = d.pop("optional_self_ref", UNSET) + + a_model_with_direct_self_reference_property = cls( + required_self_ref=required_self_ref, + optional_self_ref=optional_self_ref, + ) + + a_model_with_direct_self_reference_property.additional_properties = d + return a_model_with_direct_self_reference_property + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/end_to_end_tests/golden-record/my_test_api_client/models/a_model_with_indirect_reference_property.py b/end_to_end_tests/golden-record/my_test_api_client/models/a_model_with_indirect_reference_property.py new file mode 100644 index 000000000..14058761d --- /dev/null +++ b/end_to_end_tests/golden-record/my_test_api_client/models/a_model_with_indirect_reference_property.py @@ -0,0 +1,60 @@ +from typing import Any, Dict, List, Type, TypeVar, Union + +import attr + +from ..models.an_enum import AnEnum +from ..types import UNSET, Unset + +T = TypeVar("T", bound="AModelWithIndirectReferenceProperty") + + +@attr.s(auto_attribs=True) +class AModelWithIndirectReferenceProperty: + """ """ + + an_enum_indirect_ref: Union[Unset, AnEnum] = UNSET + additional_properties: Dict[str, Any] = attr.ib(init=False, factory=dict) + + def to_dict(self) -> Dict[str, Any]: + an_enum_indirect_ref: Union[Unset, str] = UNSET + if not isinstance(self.an_enum_indirect_ref, Unset): + an_enum_indirect_ref = self.an_enum_indirect_ref.value + + field_dict: Dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({}) + if an_enum_indirect_ref is not UNSET: + field_dict["an_enum_indirect_ref"] = an_enum_indirect_ref + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + d = src_dict.copy() + an_enum_indirect_ref: Union[Unset, AnEnum] = UNSET + _an_enum_indirect_ref = d.pop("an_enum_indirect_ref", UNSET) + if not isinstance(_an_enum_indirect_ref, Unset): + an_enum_indirect_ref = AnEnum(_an_enum_indirect_ref) + + a_model_with_indirect_reference_property = cls( + an_enum_indirect_ref=an_enum_indirect_ref, + ) + + a_model_with_indirect_reference_property.additional_properties = d + return a_model_with_indirect_reference_property + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/end_to_end_tests/golden-record/my_test_api_client/models/a_model_with_indirect_self_reference_property.py b/end_to_end_tests/golden-record/my_test_api_client/models/a_model_with_indirect_self_reference_property.py new file mode 100644 index 000000000..2c101f264 --- /dev/null +++ b/end_to_end_tests/golden-record/my_test_api_client/models/a_model_with_indirect_self_reference_property.py @@ -0,0 +1,77 @@ +from typing import Any, Dict, List, Type, TypeVar, Union + +import attr + +from ..models.an_enum import AnEnum +from ..types import UNSET, Unset + +T = TypeVar("T", bound="AModelWithIndirectSelfReferenceProperty") + + +@attr.s(auto_attribs=True) +class AModelWithIndirectSelfReferenceProperty: + """ """ + + required_self_ref: AModelWithIndirectSelfReferenceProperty + an_enum: Union[Unset, AnEnum] = UNSET + optional_self_ref: Union[Unset, AModelWithIndirectSelfReferenceProperty] = UNSET + additional_properties: Dict[str, Any] = attr.ib(init=False, factory=dict) + + def to_dict(self) -> Dict[str, Any]: + required_self_ref = self.required_self_ref + an_enum: Union[Unset, str] = UNSET + if not isinstance(self.an_enum, Unset): + an_enum = self.an_enum.value + + optional_self_ref = self.optional_self_ref + + field_dict: Dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "required_self_ref": required_self_ref, + } + ) + if an_enum is not UNSET: + field_dict["an_enum"] = an_enum + if optional_self_ref is not UNSET: + field_dict["optional_self_ref"] = optional_self_ref + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + d = src_dict.copy() + required_self_ref = d.pop("required_self_ref") + + an_enum: Union[Unset, AnEnum] = UNSET + _an_enum = d.pop("an_enum", UNSET) + if not isinstance(_an_enum, Unset): + an_enum = AnEnum(_an_enum) + + optional_self_ref = d.pop("optional_self_ref", UNSET) + + a_model_with_indirect_self_reference_property = cls( + required_self_ref=required_self_ref, + an_enum=an_enum, + optional_self_ref=optional_self_ref, + ) + + a_model_with_indirect_self_reference_property.additional_properties = d + return a_model_with_indirect_self_reference_property + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/end_to_end_tests/openapi.json b/end_to_end_tests/openapi.json index 585b97cde..0c0c953f6 100644 --- a/end_to_end_tests/openapi.json +++ b/end_to_end_tests/openapi.json @@ -1194,6 +1194,58 @@ "properties": { "inner": {"$ref": "#/components/schemas/model_reference_doesnt_match"} } + }, + "AModelWithIndirectReferenceProperty": { + "title": "AModelWithIndirectReferenceProperty", + "type": "object", + "properties": { + "an_enum_indirect_ref": { + "$ref": "#/components/schemas/AnEnumDeeperIndirectReference" + } + } + }, + "AnEnumDeeperIndirectReference": { + "$ref": "#/components/schemas/AnEnumIndirectReference" + }, + "AnEnumIndirectReference": { + "$ref": "#/components/schemas/AnEnum" + }, + "AModelWithDirectSelfReferenceProperty": { + "type": "object", + "properties": { + "required_self_ref": { + "$ref": "#/components/schemas/AModelWithDirectSelfReferenceProperty", + }, + "optional_self_ref": { + "$ref": "#/components/schemas/AModelWithDirectSelfReferenceProperty", + } + }, + "required": [ + "required_self_ref" + ] + }, + "AModelWithIndirectSelfReferenceProperty": { + "type": "object", + "properties": { + "an_enum": { + "$ref": "#/components/schemas/AnEnum" + }, + "required_self_ref": { + "$ref": "#/components/schemas/AnDeeperIndirectSelfReference" + }, + "optional_self_ref": { + "$ref": "#/components/schemas/AnDeeperIndirectSelfReference" + } + }, + "required": [ + "required_self_ref" + ] + }, + "AnDeeperIndirectSelfReference": { + "$ref": "#/components/schemas/AnIndirectSelfReference" + }, + "AnIndirectSelfReference": { + "$ref": "#/components/schemas/AModelWithIndirectSelfReferenceProperty" } } } diff --git a/openapi_python_client/parser/errors.py b/openapi_python_client/parser/errors.py index 562e54428..6855ef6bd 100644 --- a/openapi_python_client/parser/errors.py +++ b/openapi_python_client/parser/errors.py @@ -1,6 +1,6 @@ from dataclasses import dataclass from enum import Enum -from typing import Optional +from typing import Any, Optional __all__ = ["ErrorLevel", "GeneratorError", "ParseError", "PropertyError", "ValidationError"] @@ -39,5 +39,12 @@ class PropertyError(ParseError): header = "Problem creating a Property: " +@dataclass +class RecursiveReferenceInterupt(PropertyError): + """Error raised when a property have an recursive reference to itself""" + + schemas: Optional[Any] = None # TODO: shall not use Any here, shall be Schemas, to fix later + + class ValidationError(Exception): pass diff --git a/openapi_python_client/parser/properties/.property.py.swp b/openapi_python_client/parser/properties/.property.py.swp new file mode 100644 index 000000000..b8aa28d43 Binary files /dev/null and b/openapi_python_client/parser/properties/.property.py.swp differ diff --git a/openapi_python_client/parser/properties/__init__.py b/openapi_python_client/parser/properties/__init__.py index 908745a6a..3c9f53c82 100644 --- a/openapi_python_client/parser/properties/__init__.py +++ b/openapi_python_client/parser/properties/__init__.py @@ -17,12 +17,12 @@ from ... import Config from ... import schema as oai from ... import utils -from ..errors import ParseError, PropertyError, ValidationError +from ..errors import ParseError, PropertyError, RecursiveReferenceInterupt, ValidationError from .converter import convert, convert_chain from .enum_property import EnumProperty from .model_property import ModelProperty, build_model_property from .property import Property -from .schemas import Class, Schemas, parse_reference_path, update_schemas_with_data +from .schemas import Class, Schemas, _Holder, _ReferencePath, parse_reference_path, update_schemas_with @attr.s(auto_attribs=True, frozen=True) @@ -34,6 +34,59 @@ class NoneProperty(Property): template: ClassVar[Optional[str]] = "none_property.py.jinja" +@attr.s(auto_attribs=True, frozen=True) +class LazySelfReferenceProperty(Property): + """A property used to resolve recursive reference. + It proxyfy the required method call to its binded Property owner + """ + + owner: _Holder[Union[ModelProperty, EnumProperty, RecursiveReferenceInterupt]] + _resolved: bool = False + + def get_base_type_string(self) -> str: + self._ensure_resolved() + + prop = self.owner.data + assert isinstance(prop, Property) + return prop.get_base_type_string() + + def get_base_json_type_string(self) -> str: + self._ensure_resolved() + + prop = self.owner.data + assert isinstance(prop, Property) + return prop.get_base_json_type_string() + + def get_type_string(self, no_optional: bool = False, json: bool = False) -> str: + self._ensure_resolved() + + prop = self.owner.data + assert isinstance(prop, Property) + return prop.get_type_string(no_optional, json) + + def get_instance_type_string(self) -> str: + self._ensure_resolved() + return super().get_instance_type_string() + + def to_string(self) -> str: + self._ensure_resolved() + + if not self.required: + return f"{self.python_name}: Union[Unset, {self.get_type_string()}] = UNSET" + else: + return f"{self.python_name}: {self.get_type_string()}" + + def _ensure_resolved(self) -> None: + if self._resolved: + return + + if not isinstance(self.owner.data, Property): + raise RuntimeError(f"LazySelfReferenceProperty {self.name} owner shall have been resolved.") + else: + object.__setattr__(self, "_resolved", True) + object.__setattr__(self, "nullable", self.owner.data.nullable) + + @attr.s(auto_attribs=True, frozen=True) class StringProperty(Property): """A property of type str""" @@ -410,11 +463,18 @@ def _property_from_ref( ref_path = parse_reference_path(data.ref) if isinstance(ref_path, ParseError): return PropertyError(data=data, detail=ref_path.detail), schemas + existing = schemas.classes_by_reference.get(ref_path) - if not existing: - return PropertyError(data=data, detail="Could not find reference in parsed models or enums"), schemas + if not existing or not existing.data: + return PropertyError(data=data, detail=f"Could not find reference {ref_path} in parsed models or enums"), schemas - prop = attr.evolve(existing, required=required, name=name) + if isinstance(existing.data, RecursiveReferenceInterupt): + return ( + LazySelfReferenceProperty(required=required, name=name, nullable=False, default=None, owner=existing), + schemas, + ) + + prop = attr.evolve(existing.data, required=required, name=name) if parent: prop = attr.evolve(prop, nullable=parent.nullable) if isinstance(prop, EnumProperty): @@ -550,29 +610,46 @@ def build_schemas( to_process: Iterable[Tuple[str, Union[oai.Reference, oai.Schema]]] = components.items() still_making_progress = True errors: List[PropertyError] = [] + recursive_references_waiting_reprocess: Dict[str, Union[oai.Reference, oai.Schema]] = dict() + visited: Set[_ReferencePath] = set() + depth = 0 # References could have forward References so keep going as long as we are making progress while still_making_progress: still_making_progress = False errors = [] next_round = [] + # Only accumulate errors from the last round, since we might fix some along the way for name, data in to_process: - if isinstance(data, oai.Reference): - schemas.errors.append(PropertyError(data=data, detail="Reference schemas are not supported.")) - continue ref_path = parse_reference_path(f"#/components/schemas/{name}") if isinstance(ref_path, ParseError): schemas.errors.append(PropertyError(detail=ref_path.detail, data=data)) continue - schemas_or_err = update_schemas_with_data(ref_path=ref_path, data=data, schemas=schemas, config=config) + + schemas_or_err = update_schemas_with( + ref_path=ref_path, data=data, schemas=schemas, visited=(ref_path, visited), config=config + ) + visited.add(ref_path) if isinstance(schemas_or_err, PropertyError): - next_round.append((name, data)) - errors.append(schemas_or_err) - continue + if isinstance(schemas_or_err, RecursiveReferenceInterupt): + up_schemas = schemas_or_err.schemas + assert isinstance(up_schemas, Schemas) # TODO fix typedef in RecursiveReferenceInterupt + schemas_or_err = up_schemas + recursive_references_waiting_reprocess[name] = data + else: + visited.remove(ref_path) + next_round.append((name, data)) + errors.append(schemas_or_err) + continue + schemas = schemas_or_err still_making_progress = True + depth += 1 to_process = next_round + if len(recursive_references_waiting_reprocess.keys()): + schemas = build_schemas(components=recursive_references_waiting_reprocess, schemas=schemas, config=config) + schemas.errors.extend(errors) return schemas diff --git a/openapi_python_client/parser/properties/model_property.py b/openapi_python_client/parser/properties/model_property.py index a40460886..74ec76b8f 100644 --- a/openapi_python_client/parser/properties/model_property.py +++ b/openapi_python_client/parser/properties/model_property.py @@ -11,6 +11,11 @@ from .schemas import Class, Schemas, parse_reference_path +@attr.s(auto_attribs=True, frozen=True) +class RecusiveProperty(Property): + pass + + @attr.s(auto_attribs=True, frozen=True) class ModelProperty(Property): """A property which refers to another Schema""" @@ -93,9 +98,12 @@ def _check_existing(prop: Property) -> Union[Property, PropertyError]: ref_path = parse_reference_path(sub_prop.ref) if isinstance(ref_path, ParseError): return PropertyError(detail=ref_path.detail, data=sub_prop) - sub_model = schemas.classes_by_reference.get(ref_path) - if sub_model is None: + + sub_model_ref = schemas.classes_by_reference.get(ref_path) + if sub_model_ref is None or not isinstance(sub_model_ref.data, Property): return PropertyError(f"Reference {sub_prop.ref} not found") + + sub_model = sub_model_ref.data if not isinstance(sub_model, ModelProperty): return PropertyError("Cannot take allOf a non-object") for prop in chain(sub_model.required_properties, sub_model.optional_properties): diff --git a/openapi_python_client/parser/properties/schemas.py b/openapi_python_client/parser/properties/schemas.py index a81879dfc..12ceb8098 100644 --- a/openapi_python_client/parser/properties/schemas.py +++ b/openapi_python_client/parser/properties/schemas.py @@ -1,6 +1,6 @@ -__all__ = ["Class", "Schemas", "parse_reference_path", "update_schemas_with_data"] +__all__ = ["Class", "Schemas", "parse_reference_path", "update_schemas_with", "_ReferencePath"] -from typing import TYPE_CHECKING, Dict, List, NewType, Union, cast +from typing import TYPE_CHECKING, Dict, Generic, List, NewType, Optional, Set, Tuple, TypeVar, Union, cast from urllib.parse import urlparse import attr @@ -8,7 +8,7 @@ from ... import Config from ... import schema as oai from ... import utils -from ..errors import ParseError, PropertyError +from ..errors import ParseError, PropertyError, RecursiveReferenceInterupt if TYPE_CHECKING: # pragma: no cover from .enum_property import EnumProperty @@ -17,7 +17,7 @@ EnumProperty = "EnumProperty" ModelProperty = "ModelProperty" - +T = TypeVar("T") _ReferencePath = NewType("_ReferencePath", str) _ClassName = NewType("_ClassName", str) @@ -29,6 +29,11 @@ def parse_reference_path(ref_path_raw: str) -> Union[_ReferencePath, ParseError] return cast(_ReferencePath, parsed.fragment) +@attr.s(auto_attribs=True) +class _Holder(Generic[T]): + data: Optional[T] + + @attr.s(auto_attribs=True, frozen=True) class Class: """Represents Python class which will be generated from an OpenAPI schema""" @@ -58,13 +63,49 @@ def from_string(*, string: str, config: Config) -> "Class": class Schemas: """Structure for containing all defined, shareable, and reusable schemas (attr classes and Enums)""" - classes_by_reference: Dict[_ReferencePath, Union[EnumProperty, ModelProperty]] = attr.ib(factory=dict) - classes_by_name: Dict[_ClassName, Union[EnumProperty, ModelProperty]] = attr.ib(factory=dict) + classes_by_reference: Dict[ + _ReferencePath, _Holder[Union[EnumProperty, ModelProperty, RecursiveReferenceInterupt]] + ] = attr.ib(factory=dict) + classes_by_name: Dict[ + _ClassName, _Holder[Union[EnumProperty, ModelProperty, RecursiveReferenceInterupt]] + ] = attr.ib(factory=dict) errors: List[ParseError] = attr.ib(factory=list) -def update_schemas_with_data( - *, ref_path: _ReferencePath, data: oai.Schema, schemas: Schemas, config: Config +def update_schemas_with( + *, + ref_path: _ReferencePath, + data: Union[oai.Reference, oai.Schema], + schemas: Schemas, + visited: Tuple[_ReferencePath, Set[_ReferencePath]], + config: Config, +) -> Union[Schemas, PropertyError]: + if isinstance(data, oai.Reference): + return _update_schemas_with_reference( + ref_path=ref_path, data=data, schemas=schemas, visited=visited, config=config + ) + else: + return _update_schemas_with_data(ref_path=ref_path, data=data, schemas=schemas, visited=visited, config=config) + + +def _update_schemas_with_reference( + *, ref_path: _ReferencePath, data: oai.Reference, schemas: Schemas, visited: Tuple[_ReferencePath, Set[_ReferencePath]], config: Config +) -> Union[Schemas, PropertyError]: + reference_pointer = parse_reference_path(data.ref) + if isinstance(reference_pointer, ParseError): + return PropertyError(detail=reference_pointer.detail, data=data) + + curr, previous = visited + previous.add(reference_pointer) + resolved_reference = schemas.classes_by_reference.get(reference_pointer) + if resolved_reference: + return attr.evolve(schemas, classes_by_reference={ref_path: resolved_reference, **schemas.classes_by_reference}) + else: + return PropertyError(f"Reference {ref_path} could not be resolved", data=data) + + +def _update_schemas_with_data( + *, ref_path: _ReferencePath, data: oai.Schema, schemas: Schemas, visited: Tuple[_ReferencePath, Set[_ReferencePath]], config: Config ) -> Union[Schemas, PropertyError]: from . import build_enum_property, build_model_property @@ -77,7 +118,20 @@ def update_schemas_with_data( prop, schemas = build_model_property( data=data, name=ref_path, schemas=schemas, required=True, parent_name=None, config=config ) + + holder = schemas.classes_by_reference.get(ref_path) if isinstance(prop, PropertyError): + curr, previous = visited + if (ref_path in previous or curr in f"{prop.data}") and not holder: + holder = _Holder(data=RecursiveReferenceInterupt()) + schemas = attr.evolve(schemas, classes_by_reference={ref_path: holder, **schemas.classes_by_reference}) + return RecursiveReferenceInterupt(schemas=schemas) return prop - schemas = attr.evolve(schemas, classes_by_reference={ref_path: prop, **schemas.classes_by_reference}) + + if holder: + holder.data = prop + else: + schemas = attr.evolve( + schemas, classes_by_reference={ref_path: _Holder(data=prop), **schemas.classes_by_reference} + ) return schemas