[python-nextgen] fix circular reference import (#15070)

* fix ciruclar reference import in python nextgen

* update samples

Author: wing328

modules/openapi-generator/src/main/java/org/openapitools/codegen/languages/PythonNextgenClientCodegen.java

@@ -64,6 +64,11 @@ public class PythonNextgenClientCodegen extends AbstractPythonCodegen implements
 
     private String testFolder;
 
+    // map of set (model imports)
+    private HashMap<String, HashSet<String>> circularImports = new HashMap<>();
+    // map of codegen models
+    private HashMap<String, CodegenModel> codegenModelMap = new HashMap<>();
+
     public PythonNextgenClientCodegen() {
         super();
 
@@ -404,14 +409,17 @@ public String getTypeDeclaration(Schema p) {
      * @param typingImports typing imports
      * @param pydantic pydantic imports
      * @param datetimeImports datetime imports
+     * @param modelImports model imports
+     * @param classname class name
      * @return pydantic type
      *
      */
     private String getPydanticType(CodegenParameter cp,
                                    Set<String> typingImports,
                                    Set<String> pydanticImports,
                                    Set<String> datetimeImports,
-                                   Set<String> modelImports) {
+                                   Set<String> modelImports,
+                                   String classname) {
         if (cp == null) {
             // if codegen parameter (e.g. map/dict of undefined type) is null, default to string
             LOGGER.warn("Codegen property is null (e.g. map/dict of undefined type). Default to typing.Any.");
@@ -432,11 +440,12 @@ private String getPydanticType(CodegenParameter cp,
             }
             pydanticImports.add("conlist");
             return String.format(Locale.ROOT, "conlist(%s%s)",
-                    getPydanticType(cp.items, typingImports, pydanticImports, datetimeImports, modelImports),
+                    getPydanticType(cp.items, typingImports, pydanticImports, datetimeImports, modelImports, classname),
                     constraints);
         } else if (cp.isMap) {
             typingImports.add("Dict");
-            return String.format(Locale.ROOT, "Dict[str, %s]", getPydanticType(cp.items, typingImports, pydanticImports, datetimeImports, modelImports));
+            return String.format(Locale.ROOT, "Dict[str, %s]",
+                    getPydanticType(cp.items, typingImports, pydanticImports, datetimeImports, modelImports, classname));
         } else if (cp.isString || cp.isBinary || cp.isByteArray) {
             if (cp.hasValidation) {
                 List<String> fieldCustomization = new ArrayList<>();
@@ -612,7 +621,7 @@ private String getPydanticType(CodegenParameter cp,
                 CodegenMediaType cmt = contents.get(key);
                 // TODO process the first one only at the moment
                 if (cmt != null)
-                    return getPydanticType(cmt.getSchema(), typingImports, pydanticImports, datetimeImports, modelImports);
+                    return getPydanticType(cmt.getSchema(), typingImports, pydanticImports, datetimeImports, modelImports, classname);
             }
             throw new RuntimeException("Error! Failed to process getPydanticType when getting the content: " + cp);
         } else {
@@ -627,14 +636,17 @@ private String getPydanticType(CodegenParameter cp,
      * @param typingImports typing imports
      * @param pydantic pydantic imports
      * @param datetimeImports datetime imports
+     * @param modelImports model imports
+     * @param classname class name
      * @return pydantic type
      *
      */
     private String getPydanticType(CodegenProperty cp,
                                    Set<String> typingImports,
                                    Set<String> pydanticImports,
                                    Set<String> datetimeImports,
-                                   Set<String> modelImports) {
+                                   Set<String> modelImports,
+                                   String classname) {
         if (cp == null) {
             // if codegen property (e.g. map/dict of undefined type) is null, default to string
             LOGGER.warn("Codegen property is null (e.g. map/dict of undefined type). Default to typing.Any.");
@@ -674,11 +686,11 @@ private String getPydanticType(CodegenProperty cp,
             pydanticImports.add("conlist");
             typingImports.add("List"); // for return type
             return String.format(Locale.ROOT, "conlist(%s%s)",
-                    getPydanticType(cp.items, typingImports, pydanticImports, datetimeImports, modelImports),
+                    getPydanticType(cp.items, typingImports, pydanticImports, datetimeImports, modelImports, classname),
                     constraints);
         } else if (cp.isMap) {
             typingImports.add("Dict");
-            return String.format(Locale.ROOT, "Dict[str, %s]", getPydanticType(cp.items, typingImports, pydanticImports, datetimeImports, modelImports));
+            return String.format(Locale.ROOT, "Dict[str, %s]", getPydanticType(cp.items, typingImports, pydanticImports, datetimeImports, modelImports, classname));
         } else if (cp.isString) {
             if (cp.hasValidation) {
                 List<String> fieldCustomization = new ArrayList<>();
@@ -846,10 +858,24 @@ private String getPydanticType(CodegenProperty cp,
             typingImports.add("Any");
             return "Dict[str, Any]";
         } else if (!cp.isPrimitiveType || cp.isModel) { // model
-            if (!cp.isCircularReference) {
-                // skip import if it's a circular reference
+            // skip import if it's a circular reference
+            if (classname == null) {
+                // for parameter model, import directly
                 hasModelsToImport = true;
                 modelImports.add(cp.dataType);
+            } else {
+                if (circularImports.containsKey(cp.dataType)) {
+                    if (circularImports.get(cp.dataType).contains(classname)) {
+                        // cp.dataType import map of set contains this model (classname), don't import
+                        LOGGER.debug("Skipped importing {} in {} due to circular import.", cp.dataType, classname);
+                    } else {
+                        // not circular import, so ok to import it
+                        hasModelsToImport = true;
+                        modelImports.add(cp.dataType);
+                    }
+                } else {
+                    LOGGER.error("Failed to look up {} from the imports (map of set) of models.", cp.dataType);
+                }
             }
             return cp.dataType;
         } else {
@@ -871,7 +897,7 @@ public OperationsMap postProcessOperationsWithModels(OperationsMap objs, List<Mo
 
             List<CodegenParameter> params = operation.allParams;
             for (CodegenParameter param : params) {
-                String typing = getPydanticType(param, typingImports, pydanticImports, datetimeImports, modelImports);
+                String typing = getPydanticType(param, typingImports, pydanticImports, datetimeImports, modelImports, null);
                 List<String> fields = new ArrayList<>();
                 String firstField = "";
 
@@ -923,7 +949,7 @@ public OperationsMap postProcessOperationsWithModels(OperationsMap objs, List<Mo
             // update typing import for operation return type
             if (!StringUtils.isEmpty(operation.returnType)) {
                 String typing = getPydanticType(operation.returnProperty, typingImports,
-                        new TreeSet<>() /* skip pydantic import for return type */, datetimeImports, modelImports);
+                        new TreeSet<>() /* skip pydantic import for return type */, datetimeImports, modelImports, null);
             }
 
         }
@@ -983,13 +1009,118 @@ public OperationsMap postProcessOperationsWithModels(OperationsMap objs, List<Mo
     @Override
     public Map<String, ModelsMap> postProcessAllModels(Map<String, ModelsMap> objs) {
         final Map<String, ModelsMap> processed = super.postProcessAllModels(objs);
+
+        for (Map.Entry<String, ModelsMap> entry : objs.entrySet()) {
+            // create hash map of codegen model
+            CodegenModel cm = ModelUtils.getModelByName(entry.getKey(), objs);
+            codegenModelMap.put(cm.classname, ModelUtils.getModelByName(entry.getKey(), objs));
+        }
+
+        // create circular import
+        for (String m : codegenModelMap.keySet()) {
+            createImportMapOfSet(m, codegenModelMap);
+        }
+
         for (Map.Entry<String, ModelsMap> entry : processed.entrySet()) {
             entry.setValue(postProcessModelsMap(entry.getValue()));
         }
 
         return processed;
     }
 
+    /**
+     * Update circularImports with the model name (key) and its imports gathered recursively
+     *
+     * @param modelName       model name
+     * @param codegenModelMap a map of CodegenModel
+     */
+    void createImportMapOfSet(String modelName, Map<String, CodegenModel> codegenModelMap) {
+        HashSet<String> imports = new HashSet<>();
+        circularImports.put(modelName, imports);
+
+        CodegenModel cm = codegenModelMap.get(modelName);
+
+        if (cm == null) {
+            LOGGER.warn("Failed to lookup model in createImportMapOfSet: " + modelName);
+            return;
+        }
+
+        List<CodegenProperty> codegenProperties = null;
+        if (cm.oneOf != null && !cm.oneOf.isEmpty()) { // oneOf
+            codegenProperties = cm.getComposedSchemas().getOneOf();
+        } else if (cm.anyOf != null && !cm.anyOf.isEmpty()) { // anyOF
+            codegenProperties = cm.getComposedSchemas().getAnyOf();
+        } else { // typical model
+            codegenProperties = cm.vars;
+        }
+
+        for (CodegenProperty cp : codegenProperties) {
+            String modelNameFromDataType = getModelNameFromDataType(cp);
+            if (modelNameFromDataType != null) { // model
+                imports.add(modelNameFromDataType); // update import
+                // go through properties or sub-schemas of the model recursively to identify more (model) import if any
+                updateImportsFromCodegenModel(modelNameFromDataType, codegenModelMap.get(modelNameFromDataType), imports);
+            }
+        }
+    }
+
+    /**
+     * Update set of imports from codegen model recursivly
+     *
+     * @param modelName model name
+     * @param cm        codegen model
+     * @param imports   set of imports
+     */
+    public void updateImportsFromCodegenModel(String modelName, CodegenModel cm, Set<String> imports) {
+        if (cm == null) {
+            LOGGER.warn("Failed to lookup model in createImportMapOfSet " + modelName);
+            return;
+        }
+
+        List<CodegenProperty> codegenProperties = null;
+        if (cm.oneOf != null && !cm.oneOf.isEmpty()) { // oneOfValidationError
+            codegenProperties = cm.getComposedSchemas().getOneOf();
+        } else if (cm.anyOf != null && !cm.anyOf.isEmpty()) { // anyOF
+            codegenProperties = cm.getComposedSchemas().getAnyOf();
+        } else { // typical model
+            codegenProperties = cm.vars;
+        }
+
+        for (CodegenProperty cp : codegenProperties) {
+            String modelNameFromDataType = getModelNameFromDataType(cp);
+            if (modelNameFromDataType != null) { // model
+                if (modelName.equals(modelNameFromDataType)) { // self referencing
+                    continue;
+                } else if (imports.contains(modelNameFromDataType)) { // circular import
+                    continue;
+                } else {
+                    imports.add(modelNameFromDataType); // update import
+                    // go through properties of the model recursively to identify more (model) import if any
+                    updateImportsFromCodegenModel(modelNameFromDataType, codegenModelMap.get(modelNameFromDataType), imports);
+                }
+            }
+        }
+    }
+
+    /**
+     * Returns the model name (if any) from data type of codegen property.
+     * Returns null if it's not a model.
+     *
+     * @param cp Codegen property
+     * @return model name
+     */
+    private String getModelNameFromDataType(CodegenProperty cp) {
+        if (cp.isArray) {
+            return getModelNameFromDataType(cp.items);
+        } else if (cp.isMap) {
+            return getModelNameFromDataType(cp.items);
+        } else if (!cp.isPrimitiveType || cp.isModel) {
+            return cp.dataType;
+        } else {
+            return null;
+        }
+    }
+
     private ModelsMap postProcessModelsMap(ModelsMap objs) {
         // process enum in models
         objs = postProcessModelsEnum(objs);
@@ -1044,7 +1175,7 @@ private ModelsMap postProcessModelsMap(ModelsMap objs) {
 
             //loop through properties/schemas to set up typing, pydantic
             for (CodegenProperty cp : codegenProperties) {
-                String typing = getPydanticType(cp, typingImports, pydanticImports, datetimeImports, modelImports);
+                String typing = getPydanticType(cp, typingImports, pydanticImports, datetimeImports, modelImports, model.classname);
                 List<String> fields = new ArrayList<>();
                 String firstField = "";
 

modules/openapi-generator/src/test/resources/3_0/python/petstore-with-fake-endpoints-models-for-testing.yaml

@@ -2126,3 +2126,24 @@ components:
       properties:
         optionalDict:
           $ref: "#/components/schemas/DictWithAdditionalProperties"
+    Circular-Reference-Model:
+      type: object
+      properties:
+        size:
+          type: integer
+        nested:
+          $ref: '#/components/schemas/FirstRef'
+    FirstRef:
+      type: object
+      properties:
+        category:
+          type: string
+        self_ref:
+          $ref: '#/components/schemas/SecondRef'
+    SecondRef:
+      type: object
+      properties:
+        category:
+          type: string
+        circular_ref:
+          $ref: '#/components/schemas/Circular-Reference-Model'

samples/openapi3/client/petstore/python-nextgen-aiohttp/.openapi-generator/FILES

@@ -18,6 +18,7 @@ docs/Capitalization.md
 docs/Cat.md
 docs/CatAllOf.md
 docs/Category.md
+docs/CircularReferenceModel.md
 docs/ClassModel.md
 docs/Client.md
 docs/Color.md
@@ -34,6 +35,7 @@ docs/FakeApi.md
 docs/FakeClassnameTags123Api.md
 docs/File.md
 docs/FileSchemaTestClass.md
+docs/FirstRef.md
 docs/Foo.md
 docs/FooGetDefaultResponse.md
 docs/FormatTest.md
@@ -61,6 +63,7 @@ docs/Pet.md
 docs/PetApi.md
 docs/Pig.md
 docs/ReadOnlyFirst.md
+docs/SecondRef.md
 docs/SelfReferenceModel.md
 docs/SingleRefType.md
 docs/SpecialCharacterEnum.md
@@ -99,6 +102,7 @@ petstore_api/models/capitalization.py
 petstore_api/models/cat.py
 petstore_api/models/cat_all_of.py
 petstore_api/models/category.py
+petstore_api/models/circular_reference_model.py
 petstore_api/models/class_model.py
 petstore_api/models/client.py
 petstore_api/models/color.py
@@ -112,6 +116,7 @@ petstore_api/models/enum_class.py
 petstore_api/models/enum_test.py
 petstore_api/models/file.py
 petstore_api/models/file_schema_test_class.py
+petstore_api/models/first_ref.py
 petstore_api/models/foo.py
 petstore_api/models/foo_get_default_response.py
 petstore_api/models/format_test.py
@@ -138,6 +143,7 @@ petstore_api/models/parent_with_optional_dict.py
 petstore_api/models/pet.py
 petstore_api/models/pig.py
 petstore_api/models/read_only_first.py
+petstore_api/models/second_ref.py
 petstore_api/models/self_reference_model.py
 petstore_api/models/single_ref_type.py
 petstore_api/models/special_character_enum.py

samples/openapi3/client/petstore/python-nextgen-aiohttp/README.md

@@ -146,6 +146,7 @@ Class | Method | HTTP request | Description
  - [Cat](docs/Cat.md)
  - [CatAllOf](docs/CatAllOf.md)
  - [Category](docs/Category.md)
+ - [CircularReferenceModel](docs/CircularReferenceModel.md)
  - [ClassModel](docs/ClassModel.md)
  - [Client](docs/Client.md)
  - [Color](docs/Color.md)
@@ -159,6 +160,7 @@ Class | Method | HTTP request | Description
  - [EnumTest](docs/EnumTest.md)
  - [File](docs/File.md)
  - [FileSchemaTestClass](docs/FileSchemaTestClass.md)
+ - [FirstRef](docs/FirstRef.md)
  - [Foo](docs/Foo.md)
  - [FooGetDefaultResponse](docs/FooGetDefaultResponse.md)
  - [FormatTest](docs/FormatTest.md)
@@ -185,6 +187,7 @@ Class | Method | HTTP request | Description
  - [Pet](docs/Pet.md)
  - [Pig](docs/Pig.md)
  - [ReadOnlyFirst](docs/ReadOnlyFirst.md)
+ - [SecondRef](docs/SecondRef.md)
  - [SelfReferenceModel](docs/SelfReferenceModel.md)
  - [SingleRefType](docs/SingleRefType.md)
  - [SpecialCharacterEnum](docs/SpecialCharacterEnum.md)

samples/openapi3/client/petstore/python-nextgen-aiohttp/docs/CircularReferenceModel.md

@@ -0,0 +1,29 @@
+# CircularReferenceModel
+
+
+## Properties
+Name | Type | Description | Notes
+------------ | ------------- | ------------- | -------------
+**size** | **int** |  | [optional] 
+**nested** | [**FirstRef**](FirstRef.md) |  | [optional] 
+
+## Example
+
+```python
+from petstore_api.models.circular_reference_model import CircularReferenceModel
+
+# TODO update the JSON string below
+json = "{}"
+# create an instance of CircularReferenceModel from a JSON string
+circular_reference_model_instance = CircularReferenceModel.from_json(json)
+# print the JSON string representation of the object
+print CircularReferenceModel.to_json()
+
+# convert the object into a dict
+circular_reference_model_dict = circular_reference_model_instance.to_dict()
+# create an instance of CircularReferenceModel from a dict
+circular_reference_model_form_dict = circular_reference_model.from_dict(circular_reference_model_dict)
+```
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+
+