Skip to content

refactor(python): use built-in type hinting generics #22925

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
timonrieger wants to merge 1 commit into
base: master
Choose a base branch
from
+11,294 −11,392
Open

refactor(python): use built-in type hinting generics #22925

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
2 changes: 0 additions & 2 deletions docs/generators/python-aiohttp.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,6 @@ These options may be applied as additional-properties (cli) or configOptions (pl
## LANGUAGE PRIMITIVES

<ul class="column-ul">
<li>Dict</li>
<li>List</li>
<li>UUID</li>
<li>bool</li>
<li>bytes</li>
Expand Down
2 changes: 0 additions & 2 deletions docs/generators/python-blueplanet.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,6 @@ These options may be applied as additional-properties (cli) or configOptions (pl
## LANGUAGE PRIMITIVES

<ul class="column-ul">
<li>Dict</li>
<li>List</li>
<li>UUID</li>
<li>bool</li>
<li>bytes</li>
Expand Down
2 changes: 0 additions & 2 deletions docs/generators/python-fastapi.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,6 @@ These options may be applied as additional-properties (cli) or configOptions (pl
## LANGUAGE PRIMITIVES

<ul class="column-ul">
<li>Dict</li>
<li>List</li>
<li>UUID</li>
<li>bool</li>
<li>bytes</li>
Expand Down
2 changes: 0 additions & 2 deletions docs/generators/python-flask.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,6 @@ These options may be applied as additional-properties (cli) or configOptions (pl
## LANGUAGE PRIMITIVES

<ul class="column-ul">
<li>Dict</li>
<li>List</li>
<li>UUID</li>
<li>bool</li>
<li>bytes</li>
Expand Down
2 changes: 0 additions & 2 deletions docs/generators/python-pydantic-v1.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,6 @@ These options may be applied as additional-properties (cli) or configOptions (pl
## LANGUAGE PRIMITIVES

<ul class="column-ul">
<li>Dict</li>
<li>List</li>
<li>bool</li>
<li>bytearray</li>
<li>bytes</li>
Expand Down
2 changes: 0 additions & 2 deletions docs/generators/python.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,6 @@ These options may be applied as additional-properties (cli) or configOptions (pl
## LANGUAGE PRIMITIVES

<ul class="column-ul">
<li>Dict</li>
<li>List</li>
<li>UUID</li>
<li>bool</li>
<li>bytes</li>
Expand Down
17 changes: 15 additions & 2 deletions modules/openapi-generator/src/main/java/org/openapitools/codegen/DefaultCodegen.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1771,7 +1771,8 @@ public DefaultCodegen() {
.build();

defaultIncludes = new HashSet<>(
Arrays.asList("double",
Arrays.asList(
"double",
"int",
"long",
"short",
Expand All @@ -1784,7 +1785,19 @@ public DefaultCodegen() {
"Void",
"Integer",
"Long",
"Float")
"Float",
// OpenAPI primitive/container keywords (these are schema-level concepts, not language types)
// and should never produce language import statements.
"string",
"number",
"integer",
"boolean",
"null",
"object",
"array",
"map",
"set"
)
);

typeMapping = new HashMap<>();
Expand Down
61 changes: 39 additions & 22 deletions ...api-generator/src/main/java/org/openapitools/codegen/languages/AbstractPythonCodegen.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,24 @@ public abstract class AbstractPythonCodegen extends DefaultCodegen implements Co

public static final String MAP_NUMBER_TO = "mapNumberTo";

/**
* Names which should never be treated as model imports for Python.
* <p>
* These can leak into {@code CodegenModel.imports} from generic container/import collection logic
* (e.g. schema keywords like {@code array} or typing/builtin names), and then later get
* incorrectly converted into model import statements.
*/
private static final Set<String> PYTHON_NON_MODEL_IMPORTS = Collections.unmodifiableSet(new HashSet<>(
Arrays.asList(
// OpenAPI schema keywords (should never be imported)
"array", "map", "set", "object",
// Python builtins
"list", "dict", "tuple", "set", "type",
// typing names (capitalized)
"List", "Dict", "Tuple", "Set", "Type"
)
));

protected String packageName = "openapi_client";
@Setter protected String packageVersion = "1.0.0";
@Setter protected String projectName; // for setup.py, e.g. petstore-api
Expand Down Expand Up @@ -101,8 +119,6 @@ public AbstractPythonCodegen() {
languageSpecificPrimitives.add("float");
languageSpecificPrimitives.add("list");
languageSpecificPrimitives.add("dict");
languageSpecificPrimitives.add("List");
languageSpecificPrimitives.add("Dict");
languageSpecificPrimitives.add("bool");
languageSpecificPrimitives.add("str");
languageSpecificPrimitives.add("datetime");
Expand Down Expand Up @@ -146,6 +162,21 @@ public AbstractPythonCodegen() {
regexModifiers.put('x', "VERBOSE");
}

@Override
protected boolean shouldAddImport(String type) {
if (type == null) {
return false;
}
// Reject type expressions (e.g. list[str], dict[str, Foo]) and other non-symbol entries early.
if (type.indexOf('[') >= 0 || type.indexOf(']') >= 0 || type.indexOf('<') >= 0 || type.indexOf('>') >= 0) {
return false;
}
if (PYTHON_NON_MODEL_IMPORTS.contains(type)) {
return false;
}
return super.shouldAddImport(type);
}

@Override
public void processOpts() {
super.processOpts();
Expand Down Expand Up @@ -896,7 +927,6 @@ private ModelsMap postProcessModelsMap(ModelsMap objs) {
if (!model.oneOf.isEmpty()) { // oneOfValidationError
codegenProperties = model.getComposedSchemas().getOneOf();
moduleImports.add("typing", "Any");
moduleImports.add("typing", "List");
moduleImports.add("pydantic", "Field");
moduleImports.add("pydantic", "StrictStr");
moduleImports.add("pydantic", "ValidationError");
Expand Down Expand Up @@ -932,11 +962,7 @@ private ModelsMap postProcessModelsMap(ModelsMap objs) {
// if model_generic.mustache is used
if (model.oneOf.isEmpty() && model.anyOf.isEmpty() && !model.isEnum) {
moduleImports.add("typing", "ClassVar");
moduleImports.add("typing", "Dict");
moduleImports.add("typing", "Any");
if (this.disallowAdditionalPropertiesIfNotPresent || model.isAdditionalPropertiesTrue) {
moduleImports.add("typing", "List");
}
}

// if pydantic model
Expand Down Expand Up @@ -1571,7 +1597,7 @@ public PythonType addTypeParam(PythonType typeParam) {
* The Python / Pydantic type can be as expressive as needed:
*
* - it could simply be `str`
* - or something more complex like `Optional[List[Dict[str, List[int]]]]`.
* - or something more complex like `Optional[list[dict[str, list[int]]]]`.
*
* Note that the default value (if available) and/or the metadata about
* the field / variable being defined are *not* part of the
Expand Down Expand Up @@ -1773,13 +1799,9 @@ private PythonType arrayType(IJsonSchemaValidationProperties cp) {
// Also, having a set instead of list creates complications:
// random JSON serialization order, unable to easily serialize
// to JSON, etc.
//pt.setType("Set");
//moduleImports.add("typing", "Set");
pt.setType("List");
moduleImports.add("typing", "List");
pt.setType("list");
} else {
pt.setType("List");
moduleImports.add("typing", "List");
pt.setType("list");
}
pt.addTypeParam(collectionItemType(cp.getItems()));
return pt;
Expand Down Expand Up @@ -1828,8 +1850,7 @@ private PythonType stringType(IJsonSchemaValidationProperties cp) {
}

private PythonType mapType(IJsonSchemaValidationProperties cp) {
moduleImports.add("typing", "Dict");
PythonType pt = new PythonType("Dict");
PythonType pt = new PythonType("dict");
pt.addTypeParam(new PythonType("str"));
pt.addTypeParam(collectionItemType(cp.getItems()));
return pt;
Expand Down Expand Up @@ -1954,9 +1975,7 @@ private PythonType binaryType(IJsonSchemaValidationProperties cp) {
pt.addTypeParam(strt);

if (cp.getIsBinary()) {
moduleImports.add("typing", "Tuple");

PythonType tt = new PythonType("Tuple");
PythonType tt = new PythonType("tuple");
// this string is a filename, not a validated value
tt.addTypeParam(new PythonType("str"));
tt.addTypeParam(bytest);
Expand All @@ -1976,9 +1995,7 @@ private PythonType binaryType(IJsonSchemaValidationProperties cp) {
pt.addTypeParam(new PythonType("StrictStr"));

if (cp.getIsBinary()) {
moduleImports.add("typing", "Tuple");

PythonType tt = new PythonType("Tuple");
PythonType tt = new PythonType("tuple");
tt.addTypeParam(new PythonType("StrictStr"));
tt.addTypeParam(new PythonType("StrictBytes"));

Expand Down
6 changes: 0 additions & 6 deletions ...rc/main/java/org/openapitools/codegen/languages/AbstractPythonConnexionServerCodegen.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,12 +99,6 @@ public AbstractPythonConnexionServerCodegen(String templateDirectory, boolean fi
simpleModule.addSerializer(Boolean.class, new PythonBooleanSerializer());
MAPPER.registerModule(simpleModule);

// TODO may remove these later to default to the setting in abstract python base class instead
languageSpecificPrimitives.add("List");
languageSpecificPrimitives.add("Dict");
typeMapping.put("array", "List");
typeMapping.put("map", "Dict");

// set the output folder here
outputFolder = "generated-code" + File.separatorChar + "connexion";

Expand Down
50 changes: 37 additions & 13 deletions ...tor/src/main/java/org/openapitools/codegen/languages/AbstractPythonPydanticV1Codegen.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,24 @@ public abstract class AbstractPythonPydanticV1Codegen extends DefaultCodegen imp

public static final String MAP_NUMBER_TO = "mapNumberTo";

/**
* Names which should never be treated as model imports for Python.
* <p>
* These can leak into {@code CodegenModel.imports} from generic container/import collection logic
* (e.g. schema keywords like {@code array} or typing/builtin names), and then later get
* incorrectly converted into model import statements.
*/
private static final Set<String> PYTHON_NON_MODEL_IMPORTS = Collections.unmodifiableSet(new HashSet<>(
Arrays.asList(
// OpenAPI schema keywords (should never be imported)
"array", "map", "set", "object",
// Python builtins
"list", "dict", "tuple", "set", "type",
// typing names (capitalized)
"List", "Dict", "Tuple", "Set", "Type"
)
));

protected String packageName = "openapi_client";
@Setter protected String packageVersion = "1.0.0";
@Setter protected String projectName; // for setup.py, e.g. petstore-api
Expand Down Expand Up @@ -91,8 +109,6 @@ public AbstractPythonPydanticV1Codegen() {
languageSpecificPrimitives.add("float");
languageSpecificPrimitives.add("list");
languageSpecificPrimitives.add("dict");
languageSpecificPrimitives.add("List");
languageSpecificPrimitives.add("Dict");
languageSpecificPrimitives.add("bool");
languageSpecificPrimitives.add("str");
languageSpecificPrimitives.add("datetime");
Expand Down Expand Up @@ -136,6 +152,21 @@ public AbstractPythonPydanticV1Codegen() {
regexModifiers.put('x', "VERBOSE");
}

@Override
protected boolean shouldAddImport(String type) {
if (type == null) {
return false;
}
// Reject type expressions (e.g. list[str], dict[str, Foo]) and other non-symbol entries early.
if (type.indexOf('[') >= 0 || type.indexOf(']') >= 0 || type.indexOf('<') >= 0 || type.indexOf('>') >= 0) {
return false;
}
if (PYTHON_NON_MODEL_IMPORTS.contains(type)) {
return false;
}
return super.shouldAddImport(type);
}

@Override
public void processOpts() {
super.processOpts();
Expand Down Expand Up @@ -843,7 +874,6 @@ private ModelsMap postProcessModelsMap(ModelsMap objs) {
if (!model.oneOf.isEmpty()) { // oneOfValidationError
codegenProperties = model.getComposedSchemas().getOneOf();
typingImports.add("Any");
typingImports.add("List");
pydanticImports.add("Field");
pydanticImports.add("StrictStr");
pydanticImports.add("ValidationError");
Expand Down Expand Up @@ -879,7 +909,6 @@ private ModelsMap postProcessModelsMap(ModelsMap objs) {
if (model.oneOf.isEmpty() && model.anyOf.isEmpty()
&& !model.isEnum
&& !this.disallowAdditionalPropertiesIfNotPresent) {
typingImports.add("Dict");
typingImports.add("Any");
}

Expand Down Expand Up @@ -1070,8 +1099,7 @@ private String getPydanticType(CodegenParameter cp,
getPydanticType(cp.items, typingImports, pydanticImports, datetimeImports, modelImports, exampleImports, postponedModelImports, postponedExampleImports, classname),
constraints);
} else if (cp.isMap) {
typingImports.add("Dict");
return String.format(Locale.ROOT, "Dict[str, %s]",
return String.format(Locale.ROOT, "dict[str, %s]",
getPydanticType(cp.items, typingImports, pydanticImports, datetimeImports, modelImports, exampleImports, postponedModelImports, postponedExampleImports, classname));
} else if (cp.isString) {
if (cp.hasValidation) {
Expand Down Expand Up @@ -1266,9 +1294,8 @@ private String getPydanticType(CodegenParameter cp,
} else if (cp.isUuid) {
return cp.dataType;
} else if (cp.isFreeFormObject) { // type: object
typingImports.add("Dict");
typingImports.add("Any");
return "Dict[str, Any]";
return "dict[str, Any]";
} else if (!cp.isPrimitiveType) {
// add model prefix
hasModelsToImport = true;
Expand Down Expand Up @@ -1351,13 +1378,11 @@ private String getPydanticType(CodegenProperty cp,
constraints += ", unique_items=True";
}
pydanticImports.add("conlist");
typingImports.add("List"); // for return type
return String.format(Locale.ROOT, "conlist(%s%s)",
getPydanticType(cp.items, typingImports, pydanticImports, datetimeImports, modelImports, exampleImports, postponedModelImports, postponedExampleImports, classname),
constraints);
} else if (cp.isMap) {
typingImports.add("Dict");
return String.format(Locale.ROOT, "Dict[str, %s]", getPydanticType(cp.items, typingImports, pydanticImports, datetimeImports, modelImports, exampleImports, postponedModelImports, postponedExampleImports, classname));
return String.format(Locale.ROOT, "dict[str, %s]", getPydanticType(cp.items, typingImports, pydanticImports, datetimeImports, modelImports, exampleImports, postponedModelImports, postponedExampleImports, classname));
} else if (cp.isString) {
if (cp.hasValidation) {
List<String> fieldCustomization = new ArrayList<>();
Expand Down Expand Up @@ -1550,9 +1575,8 @@ private String getPydanticType(CodegenProperty cp,
} else if (cp.isUuid) {
return cp.dataType;
} else if (cp.isFreeFormObject) { // type: object
typingImports.add("Dict");
typingImports.add("Any");
return "Dict[str, Any]";
return "dict[str, Any]";
} else if (!cp.isPrimitiveType || cp.isModel) { // model
// skip import if it's a circular reference
if (classname == null) {
Expand Down
5 changes: 1 addition & 4 deletions ...enapi-generator/src/main/java/org/openapitools/codegen/languages/PythonClientCodegen.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,10 +99,7 @@ public PythonClientCodegen() {
// at the moment
importMapping.clear();

// override type mapping in abstract python codegen
typeMapping.put("array", "List");
typeMapping.put("set", "List");
typeMapping.put("map", "Dict");
// extend type mapping in abstract python codegen
typeMapping.put("decimal", "decimal.Decimal");
typeMapping.put("file", "bytes");
typeMapping.put("binary", "bytes");
Expand Down
5 changes: 0 additions & 5 deletions ...enerator/src/main/java/org/openapitools/codegen/languages/PythonFastAPIServerCodegen.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,11 +118,6 @@ public PythonFastAPIServerCodegen() {
additionalProperties.put(CodegenConstants.PACKAGE_NAME, DEFAULT_PACKAGE_NAME);
additionalProperties.put(CodegenConstants.FASTAPI_IMPLEMENTATION_PACKAGE, DEFAULT_IMPL_FOLDER);

languageSpecificPrimitives.add("List");
languageSpecificPrimitives.add("Dict");
typeMapping.put("array", "List");
typeMapping.put("map", "Dict");

outputFolder = "generated-code" + File.separator + NAME;
modelTemplateFiles.put("model.mustache", ".py");
apiTemplateFiles.put("api.mustache", ".py");
Expand Down
5 changes: 1 addition & 4 deletions ...rator/src/main/java/org/openapitools/codegen/languages/PythonPydanticV1ClientCodegen.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,10 +93,7 @@ public PythonPydanticV1ClientCodegen() {
// at the moment
importMapping.clear();

// override type mapping in abstract python codegen
typeMapping.put("array", "List");
typeMapping.put("set", "List");
typeMapping.put("map", "Dict");
// extend type mapping in abstract python codegen
typeMapping.put("decimal", "decimal.Decimal");
typeMapping.put("file", "bytearray");
typeMapping.put("binary", "bytearray");
Expand Down
Loading
Loading