From 806df1e04d1014fe3c669c5305f296f4ce08f10f Mon Sep 17 00:00:00 2001 From: p1-ra Date: Fri, 7 May 2021 23:43:34 +0200 Subject: [PATCH 1/5] parser / schemas / add indirect reference resolution --- .../parser/properties/__init__.py | 9 +++---- .../parser/properties/schemas.py | 27 +++++++++++++++++-- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/openapi_python_client/parser/properties/__init__.py b/openapi_python_client/parser/properties/__init__.py index 908745a6a..bddf0016e 100644 --- a/openapi_python_client/parser/properties/__init__.py +++ b/openapi_python_client/parser/properties/__init__.py @@ -22,7 +22,7 @@ from .enum_property import EnumProperty from .model_property import ModelProperty, build_model_property from .property import Property -from .schemas import Class, Schemas, parse_reference_path, update_schemas_with_data +from .schemas import Class, Schemas, parse_reference_path, update_schemas_with @attr.s(auto_attribs=True, frozen=True) @@ -558,18 +558,17 @@ def build_schemas( next_round = [] # Only accumulate errors from the last round, since we might fix some along the way for name, data in to_process: - if isinstance(data, oai.Reference): - schemas.errors.append(PropertyError(data=data, detail="Reference schemas are not supported.")) - continue ref_path = parse_reference_path(f"#/components/schemas/{name}") if isinstance(ref_path, ParseError): schemas.errors.append(PropertyError(detail=ref_path.detail, data=data)) continue - schemas_or_err = update_schemas_with_data(ref_path=ref_path, data=data, schemas=schemas, config=config) + + schemas_or_err = update_schemas_with(ref_path=ref_path, data=data, schemas=schemas, config=config) if isinstance(schemas_or_err, PropertyError): next_round.append((name, data)) errors.append(schemas_or_err) continue + schemas = schemas_or_err still_making_progress = True to_process = next_round diff --git a/openapi_python_client/parser/properties/schemas.py b/openapi_python_client/parser/properties/schemas.py index a81879dfc..710eeefde 100644 --- a/openapi_python_client/parser/properties/schemas.py +++ b/openapi_python_client/parser/properties/schemas.py @@ -1,4 +1,4 @@ -__all__ = ["Class", "Schemas", "parse_reference_path", "update_schemas_with_data"] +__all__ = ["Class", "Schemas", "parse_reference_path", "update_schemas_with"] from typing import TYPE_CHECKING, Dict, List, NewType, Union, cast from urllib.parse import urlparse @@ -63,7 +63,30 @@ class Schemas: errors: List[ParseError] = attr.ib(factory=list) -def update_schemas_with_data( +def update_schemas_with( + *, ref_path: _ReferencePath, data: Union[oai.Reference, oai.Schema], schemas: Schemas, config: Config +) -> Union[Schemas, PropertyError]: + if isinstance(data, oai.Reference): + return _update_schemas_with_reference(ref_path=ref_path, data=data, schemas=schemas, config=config) + else: + return _update_schemas_with_data(ref_path=ref_path, data=data, schemas=schemas, config=config) + + +def _update_schemas_with_reference( + *, ref_path: _ReferencePath, data: oai.Reference, schemas: Schemas, config: Config +) -> Union[Schemas, PropertyError]: + reference_pointer = parse_reference_path(data.ref) + if isinstance(reference_pointer, ParseError): + return PropertyError(detail=reference_pointer.detail, data=data) + + resolved_reference = schemas.classes_by_reference.get(reference_pointer) + if resolved_reference: + return attr.evolve(schemas, classes_by_reference={ref_path: resolved_reference, **schemas.classes_by_reference}) + else: + return PropertyError(f"Reference {ref_path} could not be resolved", data=data) + + +def _update_schemas_with_data( *, ref_path: _ReferencePath, data: oai.Schema, schemas: Schemas, config: Config ) -> Union[Schemas, PropertyError]: from . import build_enum_property, build_model_property From 5696079ff781fa173e97dd29c3b9ed9545b5c3de Mon Sep 17 00:00:00 2001 From: p1-ra Date: Sat, 8 May 2021 01:34:56 +0200 Subject: [PATCH 2/5] e2e / update `openapi.json` and golden-record --- .../my_test_api_client/models/__init__.py | 1 + ..._model_with_indirect_reference_property.py | 60 +++++++++++++++++++ end_to_end_tests/openapi.json | 15 +++++ 3 files changed, 76 insertions(+) create mode 100644 end_to_end_tests/golden-record/my_test_api_client/models/a_model_with_indirect_reference_property.py diff --git a/end_to_end_tests/golden-record/my_test_api_client/models/__init__.py b/end_to_end_tests/golden-record/my_test_api_client/models/__init__.py index 5bbd77d7d..a1ccb9c63 100644 --- a/end_to_end_tests/golden-record/my_test_api_client/models/__init__.py +++ b/end_to_end_tests/golden-record/my_test_api_client/models/__init__.py @@ -1,6 +1,7 @@ """ Contains all the data models used in inputs/outputs """ from .a_model import AModel +from .a_model_with_indirect_reference_property import AModelWithIndirectReferenceProperty from .all_of_sub_model import AllOfSubModel from .an_all_of_enum import AnAllOfEnum from .an_enum import AnEnum diff --git a/end_to_end_tests/golden-record/my_test_api_client/models/a_model_with_indirect_reference_property.py b/end_to_end_tests/golden-record/my_test_api_client/models/a_model_with_indirect_reference_property.py new file mode 100644 index 000000000..14058761d --- /dev/null +++ b/end_to_end_tests/golden-record/my_test_api_client/models/a_model_with_indirect_reference_property.py @@ -0,0 +1,60 @@ +from typing import Any, Dict, List, Type, TypeVar, Union + +import attr + +from ..models.an_enum import AnEnum +from ..types import UNSET, Unset + +T = TypeVar("T", bound="AModelWithIndirectReferenceProperty") + + +@attr.s(auto_attribs=True) +class AModelWithIndirectReferenceProperty: + """ """ + + an_enum_indirect_ref: Union[Unset, AnEnum] = UNSET + additional_properties: Dict[str, Any] = attr.ib(init=False, factory=dict) + + def to_dict(self) -> Dict[str, Any]: + an_enum_indirect_ref: Union[Unset, str] = UNSET + if not isinstance(self.an_enum_indirect_ref, Unset): + an_enum_indirect_ref = self.an_enum_indirect_ref.value + + field_dict: Dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({}) + if an_enum_indirect_ref is not UNSET: + field_dict["an_enum_indirect_ref"] = an_enum_indirect_ref + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + d = src_dict.copy() + an_enum_indirect_ref: Union[Unset, AnEnum] = UNSET + _an_enum_indirect_ref = d.pop("an_enum_indirect_ref", UNSET) + if not isinstance(_an_enum_indirect_ref, Unset): + an_enum_indirect_ref = AnEnum(_an_enum_indirect_ref) + + a_model_with_indirect_reference_property = cls( + an_enum_indirect_ref=an_enum_indirect_ref, + ) + + a_model_with_indirect_reference_property.additional_properties = d + return a_model_with_indirect_reference_property + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/end_to_end_tests/openapi.json b/end_to_end_tests/openapi.json index 585b97cde..5b8927724 100644 --- a/end_to_end_tests/openapi.json +++ b/end_to_end_tests/openapi.json @@ -1194,6 +1194,21 @@ "properties": { "inner": {"$ref": "#/components/schemas/model_reference_doesnt_match"} } + }, + "AModelWithIndirectReferenceProperty": { + "title": "AModelWithIndirectReferenceProperty", + "type": "object", + "properties": { + "an_enum_indirect_ref": { + "$ref": "#/components/schemas/AnEnumDeeperIndirectReference" + } + } + }, + "AnEnumDeeperIndirectReference": { + "$ref": "#/components/schemas/AnEnumIndirectReference" + }, + "AnEnumIndirectReference": { + "$ref": "#/components/schemas/AnEnum" } } } From c7da89e201c66caf123961afd76cd5a38b2fec88 Mon Sep 17 00:00:00 2001 From: p1-ra Date: Sat, 8 May 2021 03:32:15 +0200 Subject: [PATCH 3/5] parser / property / add recursive reference resoltion --- openapi_python_client/parser/errors.py | 9 +- .../parser/properties/.property.py.swp | Bin 0 -> 16384 bytes .../parser/properties/__init__.py | 94 ++++++++++++++++-- .../parser/properties/model_property.py | 12 ++- .../parser/properties/schemas.py | 52 +++++++--- 5 files changed, 143 insertions(+), 24 deletions(-) create mode 100644 openapi_python_client/parser/properties/.property.py.swp diff --git a/openapi_python_client/parser/errors.py b/openapi_python_client/parser/errors.py index 562e54428..6855ef6bd 100644 --- a/openapi_python_client/parser/errors.py +++ b/openapi_python_client/parser/errors.py @@ -1,6 +1,6 @@ from dataclasses import dataclass from enum import Enum -from typing import Optional +from typing import Any, Optional __all__ = ["ErrorLevel", "GeneratorError", "ParseError", "PropertyError", "ValidationError"] @@ -39,5 +39,12 @@ class PropertyError(ParseError): header = "Problem creating a Property: " +@dataclass +class RecursiveReferenceInterupt(PropertyError): + """Error raised when a property have an recursive reference to itself""" + + schemas: Optional[Any] = None # TODO: shall not use Any here, shall be Schemas, to fix later + + class ValidationError(Exception): pass diff --git a/openapi_python_client/parser/properties/.property.py.swp b/openapi_python_client/parser/properties/.property.py.swp new file mode 100644 index 0000000000000000000000000000000000000000..b8aa28d437fb65f735ae9a452ad527368b3b38a0 GIT binary patch literal 16384 zcmeHNNslDO6%N3F&0-__#wn<=>?%~9GlL}ym)s-T>}Zx2-95H68f8aiMNJ2j6RRHx67aE=Kb5y~v2HteUO?`(#;>($`fL z%ZvBEw?t%i4v(FE_!Pf8ImU3^#@LU4^4#vTA6>)P4;a&OBDF0vbzSW`xv{x@W^--p z(XB_mvw8CP%ArpMfk#Jj|7bClT1?g2bY}Moy_V$)i-p#-ZcTmbRFp=RN2etwl~zd; zzRr^AY~^~fBQg*fXftpt`_^5@U8*1H+`+&0)tz={{2m#I3`7PZ1CfEqKx7~?5E+OJ z{GT&mr`NI{!p3X8otgi{699LG&+Gi}@HtxQ_dnxDWFRsS8Hfx-1|kEIfyh8)ATkgc zhzvvqA_M<}3<%uFe;ayzNH_ZU|9`mufA1@dy$k#S_&x9|;0@q);Ag-APypWpHh|;6 zU%t%P_kj;@W$Y#3Mc^6WIM4&S!0o_|z$dpb_GiEXr+^M{2k^m;on%W5B03GWK`iH^2+P zMPMDc8F=Rg#(oPt1N;)mfPY*M9)RBguK*et0@nZx_}dp4dkT06_%85u;3nW5JTUn+ z@FU;^@J-+wzzx8k@X+No;OD>?cmU`Ew*&uzKmG}jUp^q;{8tzEmm2et9E&QqTp6wl z%TE_t4j2mq=rfF1cb!d_%ne4;fN;$Rc6Ls0-M`&op-1LeiNqV3kCRe9Ua3-!R?T*U zys$ zzcel7SUqvs&Zg4%0o+m6=R4X++v%+oQA@x~Xwyv9Ft#=4v&n8Nb(qA!-=a|G0A1y| z80IbCdtrPX9VZLGfv%y*E`mz%FL_xY8VWFr?-!(&rLgco{aR1L+;IP==4!w< zc>RqeOe+h|jf+xpp=Z35xv*+q@@`&aBJc4@F{*OOwU|hg@a;u^(~`QH3}s0QBqZ;G zw9T3ulb- zx=Y!t7g(Y=gC418(?wdJ-`V$O4`@{vd0W!Du&&eR2=Yp4GnE-dJl~w{Xj!4)o?bgTXUgVDLH?*v48Qq^z`66Uj;5G9Aud_=xW+ z0}ZXvncU^sp3E*TB$qSV_b({6BG)J`yglR24u$?Xmo$BTsb$vX^X>ApT&fPAPsQ(Q zQVz>21R%UhW$ntuSkN@2)VbAKh8hXr)uL@4llTXYHF_&-BjeDdao7u zqJV*y>y*tQbCq zTA8FNYPzttOjB=s$G=-jyGEzaE30yoVCRQ#)hN7pszxY(YOG}0tp^w_F3L1RM!@3- zDNXM6YX6e0SR_oPuE4V&IWOu<=PqaJVRI9=Kx25Ck_jfA76vz}gE6Q`^&hx>?PNEmKI~q6m*h4i5hlEnRORC z4H5Nh+6w$rA` zZBLU5dP9yCIGNjnf=@(N7G^+gh=nS{$qbvX4LT(J-lk(o9p+YSEjr+*;lmI>VO>5} zI5u`s)N!or5oa*RmGA|A5`KzQgSE!u3`%znZh;NNz8Y3m1`pjQO@N_+$hqGMUo`p zjOwZrWB$ZSQ~emk5=DmZ2&JIkY;b*!116*U|GRPTz6;R(|8@R8lkWFl_U$S4#Vaxp z8Hfx-1|kEIfyh8)ATkgchzvvqA_I|ul?>ofL~}L&jJV_;`?VgbT^{hdZTWvTz005L SzZ?IFKs;Hy str: + self._ensure_resolved() + + prop = self.owner.data + assert isinstance(prop, Property) + return prop.get_base_type_string() + + def get_base_json_type_string(self) -> str: + self._ensure_resolved() + + prop = self.owner.data + assert isinstance(prop, Property) + return prop.get_base_json_type_string() + + def get_type_string(self, no_optional: bool = False, json: bool = False) -> str: + self._ensure_resolved() + + prop = self.owner.data + assert isinstance(prop, Property) + return prop.get_type_string(no_optional, json) + + def get_instance_type_string(self) -> str: + self._ensure_resolved() + return super().get_instance_type_string() + + def to_string(self) -> str: + self._ensure_resolved() + + if not self.required: + return f"{self.python_name}: Union[Unset, {self.get_type_string()}] = UNSET" + else: + return f"{self.python_name}: {self.get_type_string()}" + + def _ensure_resolved(self) -> None: + if self._resolved: + return + + if not isinstance(self.owner.data, Property): + raise RuntimeError(f"LazySelfReferenceProperty {self.name} owner shall have been resolved.") + else: + object.__setattr__(self, "_resolved", True) + object.__setattr__(self, "nullable", self.owner.data.nullable) + + @attr.s(auto_attribs=True, frozen=True) class StringProperty(Property): """A property of type str""" @@ -410,11 +463,18 @@ def _property_from_ref( ref_path = parse_reference_path(data.ref) if isinstance(ref_path, ParseError): return PropertyError(data=data, detail=ref_path.detail), schemas + existing = schemas.classes_by_reference.get(ref_path) - if not existing: + if not existing or not existing.data: return PropertyError(data=data, detail="Could not find reference in parsed models or enums"), schemas - prop = attr.evolve(existing, required=required, name=name) + if isinstance(existing.data, RecursiveReferenceInterupt): + return ( + LazySelfReferenceProperty(required=required, name=name, nullable=False, default=None, owner=existing), + schemas, + ) + + prop = attr.evolve(existing.data, required=required, name=name) if parent: prop = attr.evolve(prop, nullable=parent.nullable) if isinstance(prop, EnumProperty): @@ -550,12 +610,15 @@ def build_schemas( to_process: Iterable[Tuple[str, Union[oai.Reference, oai.Schema]]] = components.items() still_making_progress = True errors: List[PropertyError] = [] - + recursive_references_waiting_reprocess: Dict[str, Union[oai.Reference, oai.Schema]] = dict() + visited: Set[_ReferencePath] = set() + depth = 0 # References could have forward References so keep going as long as we are making progress while still_making_progress: still_making_progress = False errors = [] next_round = [] + # Only accumulate errors from the last round, since we might fix some along the way for name, data in to_process: ref_path = parse_reference_path(f"#/components/schemas/{name}") @@ -563,15 +626,28 @@ def build_schemas( schemas.errors.append(PropertyError(detail=ref_path.detail, data=data)) continue - schemas_or_err = update_schemas_with(ref_path=ref_path, data=data, schemas=schemas, config=config) + visited.add(ref_path) + schemas_or_err = update_schemas_with( + ref_path=ref_path, data=data, schemas=schemas, visited=visited, config=config + ) if isinstance(schemas_or_err, PropertyError): - next_round.append((name, data)) - errors.append(schemas_or_err) - continue + if isinstance(schemas_or_err, RecursiveReferenceInterupt): + up_schemas = schemas_or_err.schemas + assert isinstance(up_schemas, Schemas) # TODO fix typedef in RecursiveReferenceInterupt + schemas_or_err = up_schemas + recursive_references_waiting_reprocess[name] = data + else: + next_round.append((name, data)) + errors.append(schemas_or_err) + continue schemas = schemas_or_err still_making_progress = True + depth += 1 to_process = next_round + if len(recursive_references_waiting_reprocess.keys()): + schemas = build_schemas(components=recursive_references_waiting_reprocess, schemas=schemas, config=config) + schemas.errors.extend(errors) return schemas diff --git a/openapi_python_client/parser/properties/model_property.py b/openapi_python_client/parser/properties/model_property.py index a40460886..74ec76b8f 100644 --- a/openapi_python_client/parser/properties/model_property.py +++ b/openapi_python_client/parser/properties/model_property.py @@ -11,6 +11,11 @@ from .schemas import Class, Schemas, parse_reference_path +@attr.s(auto_attribs=True, frozen=True) +class RecusiveProperty(Property): + pass + + @attr.s(auto_attribs=True, frozen=True) class ModelProperty(Property): """A property which refers to another Schema""" @@ -93,9 +98,12 @@ def _check_existing(prop: Property) -> Union[Property, PropertyError]: ref_path = parse_reference_path(sub_prop.ref) if isinstance(ref_path, ParseError): return PropertyError(detail=ref_path.detail, data=sub_prop) - sub_model = schemas.classes_by_reference.get(ref_path) - if sub_model is None: + + sub_model_ref = schemas.classes_by_reference.get(ref_path) + if sub_model_ref is None or not isinstance(sub_model_ref.data, Property): return PropertyError(f"Reference {sub_prop.ref} not found") + + sub_model = sub_model_ref.data if not isinstance(sub_model, ModelProperty): return PropertyError("Cannot take allOf a non-object") for prop in chain(sub_model.required_properties, sub_model.optional_properties): diff --git a/openapi_python_client/parser/properties/schemas.py b/openapi_python_client/parser/properties/schemas.py index 710eeefde..ff579a9ea 100644 --- a/openapi_python_client/parser/properties/schemas.py +++ b/openapi_python_client/parser/properties/schemas.py @@ -1,6 +1,6 @@ -__all__ = ["Class", "Schemas", "parse_reference_path", "update_schemas_with"] +__all__ = ["Class", "Schemas", "parse_reference_path", "update_schemas_with", "_ReferencePath"] -from typing import TYPE_CHECKING, Dict, List, NewType, Union, cast +from typing import TYPE_CHECKING, Dict, Generic, List, NewType, Optional, Set, TypeVar, Union, cast from urllib.parse import urlparse import attr @@ -8,7 +8,7 @@ from ... import Config from ... import schema as oai from ... import utils -from ..errors import ParseError, PropertyError +from ..errors import ParseError, PropertyError, RecursiveReferenceInterupt if TYPE_CHECKING: # pragma: no cover from .enum_property import EnumProperty @@ -17,7 +17,7 @@ EnumProperty = "EnumProperty" ModelProperty = "ModelProperty" - +T = TypeVar("T") _ReferencePath = NewType("_ReferencePath", str) _ClassName = NewType("_ClassName", str) @@ -29,6 +29,11 @@ def parse_reference_path(ref_path_raw: str) -> Union[_ReferencePath, ParseError] return cast(_ReferencePath, parsed.fragment) +@attr.s(auto_attribs=True) +class _Holder(Generic[T]): + data: Optional[T] + + @attr.s(auto_attribs=True, frozen=True) class Class: """Represents Python class which will be generated from an OpenAPI schema""" @@ -58,22 +63,33 @@ def from_string(*, string: str, config: Config) -> "Class": class Schemas: """Structure for containing all defined, shareable, and reusable schemas (attr classes and Enums)""" - classes_by_reference: Dict[_ReferencePath, Union[EnumProperty, ModelProperty]] = attr.ib(factory=dict) - classes_by_name: Dict[_ClassName, Union[EnumProperty, ModelProperty]] = attr.ib(factory=dict) + classes_by_reference: Dict[ + _ReferencePath, _Holder[Union[EnumProperty, ModelProperty, RecursiveReferenceInterupt]] + ] = attr.ib(factory=dict) + classes_by_name: Dict[ + _ClassName, _Holder[Union[EnumProperty, ModelProperty, RecursiveReferenceInterupt]] + ] = attr.ib(factory=dict) errors: List[ParseError] = attr.ib(factory=list) def update_schemas_with( - *, ref_path: _ReferencePath, data: Union[oai.Reference, oai.Schema], schemas: Schemas, config: Config + *, + ref_path: _ReferencePath, + data: Union[oai.Reference, oai.Schema], + schemas: Schemas, + visited: Set[_ReferencePath], + config: Config, ) -> Union[Schemas, PropertyError]: if isinstance(data, oai.Reference): - return _update_schemas_with_reference(ref_path=ref_path, data=data, schemas=schemas, config=config) + return _update_schemas_with_reference( + ref_path=ref_path, data=data, schemas=schemas, visited=visited, config=config + ) else: - return _update_schemas_with_data(ref_path=ref_path, data=data, schemas=schemas, config=config) + return _update_schemas_with_data(ref_path=ref_path, data=data, schemas=schemas, visited=visited, config=config) def _update_schemas_with_reference( - *, ref_path: _ReferencePath, data: oai.Reference, schemas: Schemas, config: Config + *, ref_path: _ReferencePath, data: oai.Reference, schemas: Schemas, visited: Set[_ReferencePath], config: Config ) -> Union[Schemas, PropertyError]: reference_pointer = parse_reference_path(data.ref) if isinstance(reference_pointer, ParseError): @@ -87,7 +103,7 @@ def _update_schemas_with_reference( def _update_schemas_with_data( - *, ref_path: _ReferencePath, data: oai.Schema, schemas: Schemas, config: Config + *, ref_path: _ReferencePath, data: oai.Schema, schemas: Schemas, visited: Set[_ReferencePath], config: Config ) -> Union[Schemas, PropertyError]: from . import build_enum_property, build_model_property @@ -100,7 +116,19 @@ def _update_schemas_with_data( prop, schemas = build_model_property( data=data, name=ref_path, schemas=schemas, required=True, parent_name=None, config=config ) + + holder = schemas.classes_by_reference.get(ref_path) if isinstance(prop, PropertyError): + if ref_path in visited and not holder: + holder = _Holder(data=RecursiveReferenceInterupt()) + schemas = attr.evolve(schemas, classes_by_reference={ref_path: holder, **schemas.classes_by_reference}) + return RecursiveReferenceInterupt(schemas=schemas) return prop - schemas = attr.evolve(schemas, classes_by_reference={ref_path: prop, **schemas.classes_by_reference}) + + if holder: + holder.data = prop + else: + schemas = attr.evolve( + schemas, classes_by_reference={ref_path: _Holder(data=prop), **schemas.classes_by_reference} + ) return schemas From 47adea9c997aadd423281ee57e56576030ba7212 Mon Sep 17 00:00:00 2001 From: p1-ra Date: Sat, 8 May 2021 15:38:20 +0200 Subject: [PATCH 4/5] e2e / update `openapi.json` and golden-record --- .../my_test_api_client/models/__init__.py | 1 + ...l_with_indirect_self_reference_property.py | 77 +++++++++++++++++++ end_to_end_tests/openapi.json | 23 ++++++ 3 files changed, 101 insertions(+) create mode 100644 end_to_end_tests/golden-record/my_test_api_client/models/a_model_with_indirect_self_reference_property.py diff --git a/end_to_end_tests/golden-record/my_test_api_client/models/__init__.py b/end_to_end_tests/golden-record/my_test_api_client/models/__init__.py index a1ccb9c63..3d16518a8 100644 --- a/end_to_end_tests/golden-record/my_test_api_client/models/__init__.py +++ b/end_to_end_tests/golden-record/my_test_api_client/models/__init__.py @@ -2,6 +2,7 @@ from .a_model import AModel from .a_model_with_indirect_reference_property import AModelWithIndirectReferenceProperty +from .a_model_with_indirect_self_reference_property import AModelWithIndirectSelfReferenceProperty from .all_of_sub_model import AllOfSubModel from .an_all_of_enum import AnAllOfEnum from .an_enum import AnEnum diff --git a/end_to_end_tests/golden-record/my_test_api_client/models/a_model_with_indirect_self_reference_property.py b/end_to_end_tests/golden-record/my_test_api_client/models/a_model_with_indirect_self_reference_property.py new file mode 100644 index 000000000..2c101f264 --- /dev/null +++ b/end_to_end_tests/golden-record/my_test_api_client/models/a_model_with_indirect_self_reference_property.py @@ -0,0 +1,77 @@ +from typing import Any, Dict, List, Type, TypeVar, Union + +import attr + +from ..models.an_enum import AnEnum +from ..types import UNSET, Unset + +T = TypeVar("T", bound="AModelWithIndirectSelfReferenceProperty") + + +@attr.s(auto_attribs=True) +class AModelWithIndirectSelfReferenceProperty: + """ """ + + required_self_ref: AModelWithIndirectSelfReferenceProperty + an_enum: Union[Unset, AnEnum] = UNSET + optional_self_ref: Union[Unset, AModelWithIndirectSelfReferenceProperty] = UNSET + additional_properties: Dict[str, Any] = attr.ib(init=False, factory=dict) + + def to_dict(self) -> Dict[str, Any]: + required_self_ref = self.required_self_ref + an_enum: Union[Unset, str] = UNSET + if not isinstance(self.an_enum, Unset): + an_enum = self.an_enum.value + + optional_self_ref = self.optional_self_ref + + field_dict: Dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "required_self_ref": required_self_ref, + } + ) + if an_enum is not UNSET: + field_dict["an_enum"] = an_enum + if optional_self_ref is not UNSET: + field_dict["optional_self_ref"] = optional_self_ref + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + d = src_dict.copy() + required_self_ref = d.pop("required_self_ref") + + an_enum: Union[Unset, AnEnum] = UNSET + _an_enum = d.pop("an_enum", UNSET) + if not isinstance(_an_enum, Unset): + an_enum = AnEnum(_an_enum) + + optional_self_ref = d.pop("optional_self_ref", UNSET) + + a_model_with_indirect_self_reference_property = cls( + required_self_ref=required_self_ref, + an_enum=an_enum, + optional_self_ref=optional_self_ref, + ) + + a_model_with_indirect_self_reference_property.additional_properties = d + return a_model_with_indirect_self_reference_property + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/end_to_end_tests/openapi.json b/end_to_end_tests/openapi.json index 5b8927724..89314cd7d 100644 --- a/end_to_end_tests/openapi.json +++ b/end_to_end_tests/openapi.json @@ -1209,6 +1209,29 @@ }, "AnEnumIndirectReference": { "$ref": "#/components/schemas/AnEnum" + }, + "AModelWithIndirectSelfReferenceProperty": { + "type": "object", + "properties": { + "an_enum": { + "$ref": "#/components/schemas/AnEnum" + }, + "required_self_ref": { + "$ref": "#/components/schemas/AnDeeperIndirectSelfReference" + }, + "optional_self_ref": { + "$ref": "#/components/schemas/AnDeeperIndirectSelfReference" + } + }, + "required": [ + "required_self_ref" + ] + }, + "AnDeeperIndirectSelfReference": { + "$ref": "#/components/schemas/AnIndirectSelfReference" + }, + "AnIndirectSelfReference": { + "$ref": "#/components/schemas/AModelWithIndirectSelfReferenceProperty" } } } From 5980501bdc91cf8a69934bf8c8842b315db1725b Mon Sep 17 00:00:00 2001 From: p1-ra Date: Tue, 18 May 2021 17:03:53 +0200 Subject: [PATCH 5/5] parser / correct recursive {direct,indirect} ref detection logic | todo: cleanup --- .../my_test_api_client/models/__init__.py | 1 + ...del_with_direct_self_reference_property.py | 63 +++++++++++++++++++ end_to_end_tests/openapi.json | 14 +++++ .../parser/properties/__init__.py | 8 ++- .../parser/properties/schemas.py | 13 ++-- 5 files changed, 91 insertions(+), 8 deletions(-) create mode 100644 end_to_end_tests/golden-record/my_test_api_client/models/a_model_with_direct_self_reference_property.py diff --git a/end_to_end_tests/golden-record/my_test_api_client/models/__init__.py b/end_to_end_tests/golden-record/my_test_api_client/models/__init__.py index 3d16518a8..adc00cd2b 100644 --- a/end_to_end_tests/golden-record/my_test_api_client/models/__init__.py +++ b/end_to_end_tests/golden-record/my_test_api_client/models/__init__.py @@ -1,6 +1,7 @@ """ Contains all the data models used in inputs/outputs """ from .a_model import AModel +from .a_model_with_direct_self_reference_property import AModelWithDirectSelfReferenceProperty from .a_model_with_indirect_reference_property import AModelWithIndirectReferenceProperty from .a_model_with_indirect_self_reference_property import AModelWithIndirectSelfReferenceProperty from .all_of_sub_model import AllOfSubModel diff --git a/end_to_end_tests/golden-record/my_test_api_client/models/a_model_with_direct_self_reference_property.py b/end_to_end_tests/golden-record/my_test_api_client/models/a_model_with_direct_self_reference_property.py new file mode 100644 index 000000000..3f11569fd --- /dev/null +++ b/end_to_end_tests/golden-record/my_test_api_client/models/a_model_with_direct_self_reference_property.py @@ -0,0 +1,63 @@ +from typing import Any, Dict, List, Type, TypeVar, Union + +import attr + +from ..types import UNSET, Unset + +T = TypeVar("T", bound="AModelWithDirectSelfReferenceProperty") + + +@attr.s(auto_attribs=True) +class AModelWithDirectSelfReferenceProperty: + """ """ + + required_self_ref: AModelWithDirectSelfReferenceProperty + optional_self_ref: Union[Unset, AModelWithDirectSelfReferenceProperty] = UNSET + additional_properties: Dict[str, Any] = attr.ib(init=False, factory=dict) + + def to_dict(self) -> Dict[str, Any]: + required_self_ref = self.required_self_ref + optional_self_ref = self.optional_self_ref + + field_dict: Dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "required_self_ref": required_self_ref, + } + ) + if optional_self_ref is not UNSET: + field_dict["optional_self_ref"] = optional_self_ref + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + d = src_dict.copy() + required_self_ref = d.pop("required_self_ref") + + optional_self_ref = d.pop("optional_self_ref", UNSET) + + a_model_with_direct_self_reference_property = cls( + required_self_ref=required_self_ref, + optional_self_ref=optional_self_ref, + ) + + a_model_with_direct_self_reference_property.additional_properties = d + return a_model_with_direct_self_reference_property + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/end_to_end_tests/openapi.json b/end_to_end_tests/openapi.json index 89314cd7d..0c0c953f6 100644 --- a/end_to_end_tests/openapi.json +++ b/end_to_end_tests/openapi.json @@ -1210,6 +1210,20 @@ "AnEnumIndirectReference": { "$ref": "#/components/schemas/AnEnum" }, + "AModelWithDirectSelfReferenceProperty": { + "type": "object", + "properties": { + "required_self_ref": { + "$ref": "#/components/schemas/AModelWithDirectSelfReferenceProperty", + }, + "optional_self_ref": { + "$ref": "#/components/schemas/AModelWithDirectSelfReferenceProperty", + } + }, + "required": [ + "required_self_ref" + ] + }, "AModelWithIndirectSelfReferenceProperty": { "type": "object", "properties": { diff --git a/openapi_python_client/parser/properties/__init__.py b/openapi_python_client/parser/properties/__init__.py index c59a07919..3c9f53c82 100644 --- a/openapi_python_client/parser/properties/__init__.py +++ b/openapi_python_client/parser/properties/__init__.py @@ -466,7 +466,7 @@ def _property_from_ref( existing = schemas.classes_by_reference.get(ref_path) if not existing or not existing.data: - return PropertyError(data=data, detail="Could not find reference in parsed models or enums"), schemas + return PropertyError(data=data, detail=f"Could not find reference {ref_path} in parsed models or enums"), schemas if isinstance(existing.data, RecursiveReferenceInterupt): return ( @@ -613,6 +613,7 @@ def build_schemas( recursive_references_waiting_reprocess: Dict[str, Union[oai.Reference, oai.Schema]] = dict() visited: Set[_ReferencePath] = set() depth = 0 + # References could have forward References so keep going as long as we are making progress while still_making_progress: still_making_progress = False @@ -626,10 +627,10 @@ def build_schemas( schemas.errors.append(PropertyError(detail=ref_path.detail, data=data)) continue - visited.add(ref_path) schemas_or_err = update_schemas_with( - ref_path=ref_path, data=data, schemas=schemas, visited=visited, config=config + ref_path=ref_path, data=data, schemas=schemas, visited=(ref_path, visited), config=config ) + visited.add(ref_path) if isinstance(schemas_or_err, PropertyError): if isinstance(schemas_or_err, RecursiveReferenceInterupt): up_schemas = schemas_or_err.schemas @@ -637,6 +638,7 @@ def build_schemas( schemas_or_err = up_schemas recursive_references_waiting_reprocess[name] = data else: + visited.remove(ref_path) next_round.append((name, data)) errors.append(schemas_or_err) continue diff --git a/openapi_python_client/parser/properties/schemas.py b/openapi_python_client/parser/properties/schemas.py index ff579a9ea..12ceb8098 100644 --- a/openapi_python_client/parser/properties/schemas.py +++ b/openapi_python_client/parser/properties/schemas.py @@ -1,6 +1,6 @@ __all__ = ["Class", "Schemas", "parse_reference_path", "update_schemas_with", "_ReferencePath"] -from typing import TYPE_CHECKING, Dict, Generic, List, NewType, Optional, Set, TypeVar, Union, cast +from typing import TYPE_CHECKING, Dict, Generic, List, NewType, Optional, Set, Tuple, TypeVar, Union, cast from urllib.parse import urlparse import attr @@ -77,7 +77,7 @@ def update_schemas_with( ref_path: _ReferencePath, data: Union[oai.Reference, oai.Schema], schemas: Schemas, - visited: Set[_ReferencePath], + visited: Tuple[_ReferencePath, Set[_ReferencePath]], config: Config, ) -> Union[Schemas, PropertyError]: if isinstance(data, oai.Reference): @@ -89,12 +89,14 @@ def update_schemas_with( def _update_schemas_with_reference( - *, ref_path: _ReferencePath, data: oai.Reference, schemas: Schemas, visited: Set[_ReferencePath], config: Config + *, ref_path: _ReferencePath, data: oai.Reference, schemas: Schemas, visited: Tuple[_ReferencePath, Set[_ReferencePath]], config: Config ) -> Union[Schemas, PropertyError]: reference_pointer = parse_reference_path(data.ref) if isinstance(reference_pointer, ParseError): return PropertyError(detail=reference_pointer.detail, data=data) + curr, previous = visited + previous.add(reference_pointer) resolved_reference = schemas.classes_by_reference.get(reference_pointer) if resolved_reference: return attr.evolve(schemas, classes_by_reference={ref_path: resolved_reference, **schemas.classes_by_reference}) @@ -103,7 +105,7 @@ def _update_schemas_with_reference( def _update_schemas_with_data( - *, ref_path: _ReferencePath, data: oai.Schema, schemas: Schemas, visited: Set[_ReferencePath], config: Config + *, ref_path: _ReferencePath, data: oai.Schema, schemas: Schemas, visited: Tuple[_ReferencePath, Set[_ReferencePath]], config: Config ) -> Union[Schemas, PropertyError]: from . import build_enum_property, build_model_property @@ -119,7 +121,8 @@ def _update_schemas_with_data( holder = schemas.classes_by_reference.get(ref_path) if isinstance(prop, PropertyError): - if ref_path in visited and not holder: + curr, previous = visited + if (ref_path in previous or curr in f"{prop.data}") and not holder: holder = _Holder(data=RecursiveReferenceInterupt()) schemas = attr.evolve(schemas, classes_by_reference={ref_path: holder, **schemas.classes_by_reference}) return RecursiveReferenceInterupt(schemas=schemas)