[ENG-1067] Use Dataclasses in Python instead of TypedDict for return …

…values (#323) * save * docs(changeset): update description of pythonResponseType * _raw * implement pydantic response type * rename to python-pydantic-responses * generating pydantic files now TODO: fix pydantic models to make test pass * test working for pydantic responses * regenerate SDKs w/ updated generator * add type hint dependencies to pydantic file * python-pydantic-responses working * fix leap-workflows-sdks * regenerate all integration test SDKs * fix setup.py in Python SDK * regenerate e2e SDKs * fix duplicate key from license * fix duplicate key from license * rename configurations to pydantic in python-pydantic-responses test * add test_typed_dict_response * test works with class_ and two tags * fix handling of reserved word + multiple tags * handle prstv1 * regenerate e2e sdks * disable generation of pydantic classes if prstv1 * regenerate e2e SDKs run-it
konfig-dev · Nov 1, 2023 · 962ab77 · 962ab77
1 parent 021b71e
commit 962ab77
Show file tree

Hide file tree

Showing 135 changed files with 2,439 additions and 366 deletions.
diff --git a/generator/konfig-dash/.changeset/mighty-laws-applaud.md b/generator/konfig-dash/.changeset/mighty-laws-applaud.md
@@ -0,0 +1,5 @@
+---
+'konfig-lib': minor
+---
+
+update description of pythonResponseType
diff --git a/generator/konfig-dash/packages/konfig-lib/src/KonfigYaml.ts b/generator/konfig-dash/packages/konfig-lib/src/KonfigYaml.ts
@@ -188,7 +188,7 @@ const pythonResponseTypeVersion1 = z
 const pythonResponseTypeVersion2 = z
   .literal('2')
   .describe(
-    'Responses are DataClass instances and do not include all HTTP response fields in the response object. To get raw HTTP repsonse fields, use the _with_http_info version of the method.'
+    'Responses are Pydantic instances and do not include all HTTP response fields in the response object. To get raw HTTP repsonse fields, use the .raw.[method] version of the method.'
   )
 
 export const pythonResponseTypeVersion = z

diff --git a/generator/konfig-generator-api/src/main/java/org/openapitools/codegen/CodegenConfig.java b/generator/konfig-generator-api/src/main/java/org/openapitools/codegen/CodegenConfig.java
@@ -78,6 +78,8 @@ public interface CodegenConfig {
 
     String typeFileFolder();
 
+    String additionalModelFileFolder();
+
     String modelTestFileFolder();
 
     String modelDocFileFolder();
@@ -86,6 +88,8 @@ public interface CodegenConfig {
 
     String typePackage();
 
+    String additionalModelPackage();
+
     String toApiName(String name);
 
     String toApiVarName(String name);
@@ -160,6 +164,8 @@ public interface CodegenConfig {
 
     Map<String, String> typeTemplateFiles();
 
+    Map<String, String> additionalModelTemplateFiles();
+
     Map<String, String> apiTestTemplateFiles();
 
     Map<String, String> modelTestTemplateFiles();
@@ -224,6 +230,8 @@ public interface CodegenConfig {
 
     String typeFilename(String templateName, String modelName);
 
+    String additionalModelFilename(String templateName, String modelName);
+
     String apiFilename(String templateName, String tag);
 
     String apiTestFilename(String templateName, String tag);

diff --git a/generator/konfig-generator-api/src/main/java/org/openapitools/codegen/CodegenModel.java b/generator/konfig-generator-api/src/main/java/org/openapitools/codegen/CodegenModel.java
@@ -92,6 +92,11 @@ public class CodegenModel implements IJsonSchemaValidationProperties {
 
     public Set<String> imports = new TreeSet<>();
     public Set<String> typeImports = new TreeSet<>();
+    public Set<String> additionalModelImports = new TreeSet<>();
+
+    // In Python, we need a modified version of Import without the "as [Model]Pydantic" suffix so I added this
+    public Set<String> additionalModelImportsModified = new TreeSet<>();
+
     public boolean hasVars, emptyVars, hasMoreModels, hasEnums, isEnum, hasValidation;
     /**
      * Indicates the OAS schema specifies "nullable: true".

diff --git a/generator/konfig-generator-api/src/main/java/org/openapitools/codegen/CodegenOperation.java b/generator/konfig-generator-api/src/main/java/org/openapitools/codegen/CodegenOperation.java
@@ -76,6 +76,7 @@ public class CodegenOperation {
     // type imports
     public Set<String> schemaImports = new HashSet<String>();
     public Set<String> typeImports = new HashSet<String>();
+    public Set<String> additionalModelImports = new HashSet<String>();
     public List<Map<String, String>> examples;
     public List<Map<String, String>> requestBodyExamples;
     public ExternalDocumentation externalDocs;

diff --git a/generator/konfig-generator-api/src/main/java/org/openapitools/codegen/CodegenProperty.java b/generator/konfig-generator-api/src/main/java/org/openapitools/codegen/CodegenProperty.java
@@ -53,6 +53,7 @@ public class CodegenProperty implements Cloneable, IJsonSchemaValidationProperti
      * The name of this property in the OpenAPI schema.
      */
     public String name;
+    public boolean hasProblematicName = false;
     public String min; // TODO: is this really used?
     public String max; // TODO: is this really used?
     public String defaultValue;
@@ -185,6 +186,7 @@ public class CodegenProperty implements Cloneable, IJsonSchemaValidationProperti
     public String nameInCamelCase; // property name in camel case
     public String nameInCamelCaseLowerFirst; // property name in camel case with lowercase first character
     public String nameInSnakeCase; // property name in upper snake case
+    public String nameInSnakeCaseLower; // property name in lower snake case
     // enum name based on the property name, usually use as a prefix (e.g. VAR_NAME) for enum name (e.g. VAR_NAME_VALUE1)
     public String enumName;
     public Integer maxItems;

diff --git a/generator/konfig-generator-api/src/main/java/org/openapitools/codegen/DefaultCodegen.java b/generator/konfig-generator-api/src/main/java/org/openapitools/codegen/DefaultCodegen.java
@@ -181,6 +181,7 @@ public class DefaultCodegen implements CodegenConfig {
     protected Map<String, String> inlineSchemaNameDefault = new HashMap<>();
     protected String modelPackage = "", apiPackage = "", fileSuffix;
     protected String typePackage = "";
+    protected String additionalModelPackage = "";
     protected String modelNamePrefix = "", modelNameSuffix = "";
     protected String apiNamePrefix = "", apiNameSuffix = "Api";
     protected String testPackage = "";
@@ -194,6 +195,7 @@ public class DefaultCodegen implements CodegenConfig {
     protected Map<String, String> apiTemplateFiles = new HashMap<>();
     protected Map<String, String> modelTemplateFiles = new HashMap<>();
     protected Map<String, String> typeTemplateFiles = new HashMap<>();
+    protected Map<String, String> additionalModelTemplateFiles = new HashMap<>();
     protected Map<String, String> apiTestTemplateFiles = new HashMap<>();
     protected Map<String, String> modelTestTemplateFiles = new HashMap<>();
     protected Map<String, String> apiDocTemplateFiles = new HashMap<>();
@@ -1177,6 +1179,11 @@ public String typePackage() {
         return typePackage;
     }
 
+    @Override
+    public String additionalModelPackage() {
+        return additionalModelPackage;
+    }
+
     @Override
     public String apiPackage() {
         return apiPackage;
@@ -1241,6 +1248,11 @@ public Map<String, String> typeTemplateFiles() {
         return typeTemplateFiles;
     }
 
+    @Override
+    public Map<String, String> additionalModelTemplateFiles() {
+        return additionalModelTemplateFiles;
+    }
+
     @Override
     public String apiFileFolder() {
         return outputFolder + File.separator + apiPackage().replace('.', File.separatorChar);
@@ -1256,6 +1268,11 @@ public String typeFileFolder() {
         return outputFolder + File.separator + typePackage().replace('.', File.separatorChar);
     }
 
+    @Override
+    public String additionalModelFileFolder() {
+        return outputFolder + File.separator + additionalModelPackage().replace('.', File.separatorChar);
+    }
+
     @Override
     public String apiTestFileFolder() {
         return outputFolder + File.separator + testPackage().replace('.', File.separatorChar);
@@ -3924,6 +3941,7 @@ public CodegenProperty fromProperty(String name, Schema p, boolean required,
         property.nameInCamelCase = camelize(property.name);
         property.nameInCamelCaseLowerFirst = camelize(property.name, CamelizeOption.LOWERCASE_FIRST_LETTER);
         property.nameInSnakeCase = CaseFormat.UPPER_CAMEL.to(CaseFormat.UPPER_UNDERSCORE, property.nameInCamelCase);
+        property.nameInSnakeCaseLower = CaseFormat.LOWER_CAMEL.to(CaseFormat.LOWER_UNDERSCORE, property.baseName);
         property.description = escapeText(p.getDescription());
         property.unescapedDescription = p.getDescription();
         property.title = p.getTitle();
@@ -6172,6 +6190,12 @@ public String typeFilename(String templateName, String modelName) {
         return typeFileFolder() + File.separator + toModelFilename(modelName) + suffix;
     }
 
+    @Override
+    public String additionalModelFilename(String templateName, String modelName) {
+        String suffix = additionalModelTemplateFiles().get(templateName);
+        return additionalModelFileFolder() + File.separator + toModelFilename(modelName) + suffix;
+    }
+
     /**
      * Return the full path and API documentation file
      *

diff --git a/generator/konfig-generator-api/src/main/java/org/openapitools/codegen/DefaultGenerator.java b/generator/konfig-generator-api/src/main/java/org/openapitools/codegen/DefaultGenerator.java
@@ -413,7 +413,20 @@ private void generateType(List<File> files, Map<String, Object> models, String m
             if (written != null) {
                 files.add(written);
                 if (config.isEnablePostProcessFile() && !dryRun) {
-                    config.postProcessFile(written, "model");
+                    config.postProcessFile(written, "type");
+                }
+            }
+        }
+    }
+
+    private void generateAdditionalModel(List<File> files, Map<String, Object> models, String modelName) throws IOException {
+        for (String templateName : config.additionalModelTemplateFiles().keySet()) {
+            String filename = config.additionalModelFilename(templateName, modelName);
+            File written = processTemplateToFile(models, templateName, filename, generateModels, CodegenConstants.MODELS);
+            if (written != null) {
+                files.add(written);
+                if (config.isEnablePostProcessFile() && !dryRun) {
+                    config.postProcessFile(written, "additionalModel");
                 }
             }
         }
@@ -571,6 +584,9 @@ void generateModels(List<File> files, List<ModelMap> allModels, List<String> unu
                 // to generate type files
                 generateType(files, models, modelName);
 
+                // to generate additional files for model (in Python, this is used for Pydantic models)
+                generateAdditionalModel(files, models, modelName);
+
                 // to generate model test files
                 generateModelTests(files, models, modelName);
 

diff --git a/...generator-api/src/main/java/org/openapitools/codegen/languages/AbstractPythonCodegen.java b/...generator-api/src/main/java/org/openapitools/codegen/languages/AbstractPythonCodegen.java
@@ -691,6 +691,10 @@ public String toApiFilename(String name) {
         return underscore(toApiName(name));
     }
 
+    public String toApiFilenameRaw(String name) {
+        return toApiFilename(name) + "_raw";
+    }
+
     @Override
     public String toClientApiName(String name) {
         return underscore(name);

diff --git a/...g-generator-api/src/main/java/org/openapitools/codegen/languages/PythonClientCodegen.java b/...g-generator-api/src/main/java/org/openapitools/codegen/languages/PythonClientCodegen.java
@@ -33,6 +33,8 @@
 import io.swagger.v3.oas.models.tags.Tag;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.commons.lang3.tuple.Triple;
 import org.apache.commons.text.StringEscapeUtils;
 import org.openapitools.codegen.*;
 import org.openapitools.codegen.CodegenDiscriminator.MappedModel;
@@ -43,10 +45,7 @@
 import org.openapitools.codegen.meta.GeneratorMetadata;
 import org.openapitools.codegen.meta.Stability;
 import org.openapitools.codegen.meta.features.*;
-import org.openapitools.codegen.model.ModelMap;
-import org.openapitools.codegen.model.ModelsMap;
-import org.openapitools.codegen.model.OperationMap;
-import org.openapitools.codegen.model.OperationsMap;
+import org.openapitools.codegen.model.*;
 import org.openapitools.codegen.templating.CommonTemplateContentLocator;
 import org.openapitools.codegen.templating.GeneratorTemplateContentLocator;
 import org.openapitools.codegen.templating.HandlebarsEngineAdapter;
@@ -172,6 +171,7 @@ public PythonClientCodegen() {
 
         modelPackage = "model";
         typePackage = "type";
+        additionalModelPackage = "pydantic";
         apiPackage = "apis";
         outputFolder = "generated-code" + File.separatorChar + "python";
 
@@ -182,6 +182,25 @@ public PythonClientCodegen() {
         // default HIDE_GENERATION_TIMESTAMP to true
         hideGenerationTimestamp = Boolean.TRUE;
 
+        ArrayList<PythonDependency> dependencies = new ArrayList<>();
+        dependencies.add(new PythonDependency("certifi", "2023.7.22", ">=", ">="));
+        dependencies.add(new PythonDependency("python-dateutil", "2.8.2", "~=", "^"));
+        dependencies.add(new PythonDependency("typing_extensions", "4.3.0", "~=", "^"));
+        dependencies.add(new PythonDependency("urllib3", "1.26.18", "~=", "^"));
+        dependencies.add(new PythonDependency("frozendict", "2.3.4", "~=", "^"));
+        dependencies.add(new PythonDependency("aiohttp", "3.8.4", "~=", "^"));
+        dependencies.add(new PythonDependency("pydantic", "2.4.2", "~=", "^"));
+        ArrayList<PythonDependency> poetryDependencies = new ArrayList<>();
+        poetryDependencies.add(new PythonDependency("python", "3.7", "N/A", "^"));
+        poetryDependencies.addAll(dependencies);
+
+
+        // join dependencies with newline
+        additionalProperties.put("poetryDependencies", String.join("\n", poetryDependencies.stream().map(PythonDependency::poetry).collect(Collectors.toList())));
+
+        // join dependencies with ",\n"
+        additionalProperties.put("setupRequirements", String.join(",\n    ", dependencies.stream().map(PythonDependency::setupPy).collect(Collectors.toList())));
+
         // from https://docs.python.org/3/reference/lexical_analysis.html#keywords
         setReservedWordsLowerCase(
                 Arrays.asList(
@@ -313,6 +332,10 @@ public void processOpts() {
         typeTemplateFiles.put("type." + templateExtension, ".py");
 
         apiTemplateFiles.put("api." + templateExtension, ".py");
+        if (additionalProperties.get("prstv2") != null && additionalProperties.get("prstv2").equals(true)) {
+            additionalModelTemplateFiles.put("pydantic." + templateExtension, ".py");
+            apiTemplateFiles.put("api_raw." + templateExtension, ".py");
+        }
         modelTestTemplateFiles.put("model_test." + templateExtension, ".py");
 
         // Commented these out as we now generate all docs in the top-level Python SDK's README.md
@@ -469,6 +492,7 @@ public void processOpts() {
         supportingFiles.add(new SupportingFile("__init__models." + templateExtension, packagePath() + File.separatorChar + "models", "__init__.py"));
         supportingFiles.add(new SupportingFile("__init__model." + templateExtension, packagePath() + File.separatorChar + modelPackage, "__init__.py"));
         supportingFiles.add(new SupportingFile("__init__type." + templateExtension, packagePath() + File.separatorChar + typePackage, "__init__.py"));
+        supportingFiles.add(new SupportingFile("__init__pydantic." + templateExtension, packagePath() + File.separatorChar + additionalModelPackage, "__init__.py"));
         supportingFiles.add(new SupportingFile("__init__apis." + templateExtension, packagePath() + File.separatorChar + apiPackage, "__init__.py"));
         // Generate the 'signing.py' module, but only if the 'HTTP signature' security scheme is specified in the OAS.
         Map<String, SecurityScheme> securitySchemeMap = openAPI != null ?
@@ -524,7 +548,8 @@ protected File processTemplateToFile(Map<String, Object> templateData, String te
     @Override
     public String apiFilename(String templateName, String tag) {
         String suffix = apiTemplateFiles().get(templateName);
-        return apiFileFolder() + File.separator + toApiFilename(tag) + suffix;
+        String filename = templateName.contains("_raw") ? toApiFilenameRaw(tag) : toApiFilename(tag);
+        return apiFileFolder() + File.separator + filename + suffix;
     }
 
     private void generateFiles(List<List<Object>> processTemplateToFileInfos, boolean shouldGenerate, String skippedByOption) {
@@ -609,6 +634,7 @@ protected void generateEndpoints(OperationsMap objs) {
             endpointMap.put("imports", co.imports);
             endpointMap.put("schemaImports", co.schemaImports);
             endpointMap.put("typeImports", co.typeImports);
+            endpointMap.put("additionalModelImports", co.additionalModelImports);
             endpointMap.put("packageName", packageName);
             endpointMap.put("operations", operations);
             ((HashMap<String, Object>) operations.get("additionalProperties")).entrySet().forEach((entry) -> {
@@ -951,6 +977,14 @@ public String toTypeImport(String name) {
         return "from " + packagePath() + "." +  typePackage() + "." + toModelFilename(name) + " import " + toModelName(name);
     }
 
+    public String toPydanticImport(String name) {
+        return toPydanticImportBase(name) + " as " + toModelName(name) + "Pydantic";
+    }
+
+    public String toPydanticImportBase(String name) {
+        return "from " + packagePath() + "." +  "pydantic" + "." + toModelFilename(name) + " import " + toModelName(name);
+    }
+
     @Override
     @SuppressWarnings("static-method")
     public OperationsMap postProcessOperationsWithModels(OperationsMap objs, List<ModelMap> allModels) {
@@ -975,6 +1009,9 @@ public OperationsMap postProcessOperationsWithModels(OperationsMap objs, List<Mo
             for (String modelName : modelNames) {
                 operation.typeImports.add(toTypeImport(modelName));
             }
+            for (String modelName : modelNames) {
+                operation.additionalModelImports.add(toPydanticImport(modelName));
+            }
         }
         generateEndpoints(objs);
         return objs;
@@ -1019,6 +1056,10 @@ public Map<String, ModelsMap> postProcessAllModels(Map<String, ModelsMap> objs)
                 for (String importModelName : importModelNames) {
                     cm.typeImports.add(toTypeImport(importModelName));
                 }
+                for (String importModelName : importModelNames) {
+                    cm.additionalModelImports.add(toPydanticImport(importModelName));
+                    cm.additionalModelImportsModified.add(toPydanticImportBase(importModelName));
+                }
             }
         }
         boolean testFolderSet = testFolder != null;
@@ -1121,6 +1162,7 @@ public CodegenProperty fromProperty(String name, Schema p, boolean required, boo
         // templates use its presence to handle these badly named variables / keys
         if ((isReservedWord(cp.baseName) || !isValidPythonVarOrClassName(cp.baseName)) && !cp.baseName.equals(cp.name)) {
             cp.nameInSnakeCase = cp.name;
+            cp.hasProblematicName = true;
         } else {
             cp.nameInSnakeCase = null;
         }
@@ -2828,6 +2870,11 @@ public String typeFileFolder() {
         return outputFolder + File.separatorChar + packagePath() + File.separatorChar +  typePackage();
     }
 
+    @Override
+    public String additionalModelFileFolder() {
+        return outputFolder + File.separatorChar + packagePath() + File.separatorChar +  additionalModelPackage();
+    }
+
     @Override
     public String apiTestFileFolder() {
         return outputFolder + File.separatorChar + testFolder;

diff --git a/...r/konfig-generator-api/src/main/java/org/openapitools/codegen/model/PythonDependency.java b/...r/konfig-generator-api/src/main/java/org/openapitools/codegen/model/PythonDependency.java
@@ -0,0 +1,25 @@
+package org.openapitools.codegen.model;
+
+public class PythonDependency {
+    String name;
+    String version;
+    String setupPyModifier;
+    String poetryModifier;
+
+    public PythonDependency(String name, String version, String setupPyModifier, String poetryModifier) {
+        this.name = name;
+        this.version = version;
+        this.setupPyModifier = setupPyModifier;
+        this.poetryModifier = poetryModifier;
+    }
+
+    // example: certifi = ">=2023.7.22"
+    public String poetry() {
+        return name + " = " + "\"" + poetryModifier + version + "\"";
+    }
+
+    // example: certifi = ">=2023.7.22"
+    public String setupPy() {
+        return "\"" + name + " " + setupPyModifier + " " + version + "\"";
+    }
+}
diff --git a/...thon_dataclass_responses/type/__init__.py → ...ources/python/__init__pydantic.handlebars b/...thon_dataclass_responses/type/__init__.py → ...ources/python/__init__pydantic.handlebars