From 2106c3145e06bab51dc6f5d7ada80ba931ec7de0 Mon Sep 17 00:00:00 2001 From: gtema Date: Wed, 7 Feb 2024 16:41:41 +0100 Subject: [PATCH] feat: process identity.federation resource - fix identity.federation resources - add possibility to drop unsused types from generated cli code - remove "pub" for most cli internal types --- codegenerator/common/__init__.py | 3 + codegenerator/common/rust.py | 100 ++++++- codegenerator/metadata.py | 41 ++- codegenerator/model.py | 3 + codegenerator/openapi/keystone.py | 166 ++++++++++- codegenerator/openapi/keystone_schemas.py | 208 ++++++++++++++ codegenerator/rust_cli.py | 72 +++-- .../templates/rust_cli/path_parameters.j2 | 2 +- .../templates/rust_cli/query_parameters.j2 | 2 +- .../templates/rust_cli/response_struct.j2 | 17 +- .../templates/rust_cli/set_body_parameters.j2 | 2 +- codegenerator/tests/unit/test_rust_cli.py | 24 +- metadata/compute_metadata.yaml | 68 ----- metadata/identity_metadata.yaml | 96 ++++--- metadata/image_metadata.yaml | 12 - metadata/network_metadata.yaml | 260 ------------------ metadata/object-store_metadata.yaml | 139 ++++++++++ tools/generate_rust_identity.sh | 1 + 18 files changed, 791 insertions(+), 425 deletions(-) create mode 100644 metadata/object-store_metadata.yaml diff --git a/codegenerator/common/__init__.py b/codegenerator/common/__init__.py index 8c8a052..e24e218 100644 --- a/codegenerator/common/__init__.py +++ b/codegenerator/common/__init__.py @@ -154,6 +154,9 @@ def find_resource_schema( return (props[resource_name]["items"], resource_name) return (props[resource_name], resource_name) for name, item in props.items(): + if name == "additionalProperties" and isinstance(item, bool): + # Some schemas are broken + continue (r, path) = find_resource_schema(item, name, resource_name) if r: return (r, path) diff --git a/codegenerator/common/rust.py b/codegenerator/common/rust.py index 0a69158..86a1f2a 100644 --- a/codegenerator/common/rust.py +++ b/codegenerator/common/rust.py @@ -240,9 +240,9 @@ class Struct(BaseCompoundType): base_type: str = "struct" fields: dict[str, StructField] = {} field_type_class_: Type[StructField] | StructField = StructField - additional_fields_type: BasePrimitiveType | BaseCombinedType | BaseCompoundType | None = ( - None - ) + additional_fields_type: ( + BasePrimitiveType | BaseCombinedType | BaseCompoundType | None + ) = None @property def type_hint(self): @@ -436,6 +436,8 @@ class TypeManager: option_type_class: Type[Option] | Option = Option string_enum_class: Type[StringEnum] | StringEnum = StringEnum + ignored_models: list[model.Reference] = [] + def __init__(self): self.models = [] self.refs = {} @@ -778,6 +780,7 @@ def set_models(self, models): """Process (translate) ADT models into Rust SDK style""" self.models = models self.refs = {} + self.ignored_models = [] unique_model_names: set[str] = set() for model_ in models: model_data_type = self.convert_model(model_) @@ -791,13 +794,47 @@ def set_models(self, models): # New name is still unused model_data_type.name = new_name unique_model_names.add(new_name) + elif isinstance(model_data_type, Struct): + # This is already an exceptional case (identity.mapping + # with remote being oneOf with multiple structs) + # Try to make a name consisting of props + props = model_data_type.fields.keys() + new_new_name = name + "".join( + x.title() for x in props + ).replace("_", "") + if new_new_name not in unique_model_names: + for other_ref, other_model in self.refs.items(): + other_name = getattr(other_model, "name", None) + if not other_name: + continue + if other_name in [ + name, + new_name, + ] and isinstance(other_model, Struct): + # rename first occurence to the same scheme + props = other_model.fields.keys() + new_other_name = name + "".join( + x.title() for x in props + ).replace("_", "") + other_model.name = new_other_name + unique_model_names.add(new_other_name) + + model_data_type.name = new_new_name + unique_model_names.add(new_new_name) + else: + raise RuntimeError( + "Model name %s is already present" % new_name + ) else: raise RuntimeError( - "Model name %s is already present" % name + "Model name %s is already present" % new_name ) elif name: unique_model_names.add(name) + for ignore_model in self.ignored_models: + self.discard_model(ignore_model) + def get_subtypes(self): """Get all subtypes excluding TLA""" for k, v in self.refs.items(): @@ -903,6 +940,61 @@ def get_parameters( if v.location == location: yield (k, v) + def discard_model( + self, + type_model: model.PrimitiveType | model.ADT | model.Reference, + ): + """Discard model from the manager""" + logging.debug(f"Request to discard {type_model}") + if isinstance(type_model, model.Reference): + type_model = self._get_adt_by_reference(type_model) + if not hasattr(type_model, "reference"): + return + for ref, data in list(self.refs.items()): + if ref == type_model.reference: + sub_ref: model.Reference | None = None + if ref.type == model.Struct: + logging.debug( + "Element is a struct. Purging also field types" + ) + # For struct type we cascadely discard all field types as + # well + for v in type_model.fields.values(): + if isinstance(v.data_type, model.Reference): + sub_ref = v.data_type + else: + sub_ref = getattr(v.data_type, "reference", None) + if sub_ref: + logging.debug(f"Need to purge also {sub_ref}") + self.discard_model(sub_ref) + elif ref.type == model.OneOfType: + logging.debug( + "Element is a OneOf. Purging also kinds types" + ) + for v in type_model.kinds: + if isinstance(v, model.Reference): + sub_ref = v + else: + sub_ref = getattr(v, "reference", None) + if sub_ref: + logging.debug(f"Need to purge also {sub_ref}") + self.discard_model(sub_ref) + elif ref.type == model.Array: + logging.debug( + f"Element is a Array. Purging also item type {type_model.item_type}" + ) + if isinstance(type_model.item_type, model.Reference): + sub_ref = type_model.item_type + else: + sub_ref = getattr( + type_model.item_type, "reference", None + ) + if sub_ref: + logging.debug(f"Need to purge also {sub_ref}") + self.discard_model(sub_ref) + logging.debug(f"Purging {ref} from models") + self.refs.pop(ref, None) + def sanitize_rust_docstrings(doc: str | None) -> str | None: """Sanitize the string to be a valid rust docstring""" diff --git a/codegenerator/metadata.py b/codegenerator/metadata.py index 9c296bd..bea7d75 100644 --- a/codegenerator/metadata.py +++ b/codegenerator/metadata.py @@ -280,13 +280,28 @@ def generate( f"Cannot identify op name for {path}:{method}" ) + # Next hacks + if args.service_type == "identity" and resource_name in [ + "OS_FEDERATION/identity_provider", + "OS_FEDERATION/identity_provider/protocol", + "OS_FEDERATION/mapping", + "OS_FEDERATION/service_provider", + ]: + if method == "put": + operation_key = "create" + elif method == "patch": + operation_key = "update" + if operation_key in resource_model: raise RuntimeError("Operation name conflict") else: if ( operation_key == "action" and args.service_type - in ["compute", "block-storage"] + in [ + "compute", + "block-storage", + ] ): # For action we actually have multiple independent operations try: @@ -302,7 +317,7 @@ def generate( ).get("discriminator") if discriminator != "action": raise RuntimeError( - "Cannot generate metadata for %s since requet body is not having action discriminator" + "Cannot generate metadata for %s since request body is not having action discriminator" % path ) for body in bodies: @@ -317,7 +332,11 @@ def generate( if ( resource_name == "flavor" and action_name - in ["update", "create", "delete"] + in [ + "update", + "create", + "delete", + ] ): # Flavor update/create/delete # operations are exposed ALSO as wsgi @@ -467,6 +486,12 @@ def generate( if "id" not in response_schema.get("properties", {}).keys(): # Resource has no ID in show method => find impossible continue + elif ( + "name" not in response_schema.get("properties", {}).keys() + and res_name != "floatingip" + ): + # Resource has no NAME => find useless + continue list_op_ = list_detailed_op or list_op if not list_op_: @@ -623,6 +648,10 @@ def post_process_operation( operation = post_process_compute_operation( resource_name, operation_name, operation ) + elif service_type == "identity": + operation = post_process_identity_operation( + resource_name, operation_name, operation + ) elif service_type == "image": operation = post_process_image_operation( resource_name, operation_name, operation @@ -666,6 +695,12 @@ def post_process_compute_operation( return operation +def post_process_identity_operation( + resource_name: str, operation_name: str, operation +): + return operation + + def post_process_image_operation( resource_name: str, operation_name: str, operation ): diff --git a/codegenerator/model.py b/codegenerator/model.py index e3bb4ce..60983fe 100644 --- a/codegenerator/model.py +++ b/codegenerator/model.py @@ -303,6 +303,9 @@ def parse_object( if properties: obj = Struct() for k, v in properties.items(): + if k == "additionalProperties" and isinstance(v, bool): + # Some schemas (in keystone) are Broken + continue if ignore_read_only and v.get("readOnly", False): continue data_type = self.parse_schema( diff --git a/codegenerator/openapi/keystone.py b/codegenerator/openapi/keystone.py index a92b011..a0cd805 100644 --- a/codegenerator/openapi/keystone.py +++ b/codegenerator/openapi/keystone.py @@ -476,6 +476,26 @@ def _post_process_operation_hook(self, openapi_spec, operation_spec): if ref not in [x.ref for x in operation_spec.parameters]: operation_spec.parameters.append(ParameterSchema(ref=ref)) + elif operationId == "OS-FEDERATION/identity_providers:get": + for ( + key, + val, + ) in keystone_schemas.IDENTITY_PROVIDERS_LIST_PARAMETERS.items(): + openapi_spec.components.parameters.setdefault( + key, ParameterSchema(**val) + ) + ref = f"#/components/parameters/{key}" + if ref not in [x.ref for x in operation_spec.parameters]: + operation_spec.parameters.append(ParameterSchema(ref=ref)) + + elif operationId in [ + "OS-FEDERATION/projects:get", + "OS-FEDERATION/projects:head", + "OS-FEDERATION/domains:get", + "OS-FEDERATION/domains:head", + ]: + operation_spec.deprecated = True + def _get_schema_ref( self, openapi_spec, @@ -855,7 +875,7 @@ def _get_schema_ref( ), ) ref = f"#/components/schemas/{name}" - elif name in "UsersApplication_CredentialsPostRequest": + elif name == "UsersApplication_CredentialsPostRequest": openapi_spec.components.schemas.setdefault( name, TypeSchema( @@ -871,6 +891,150 @@ def _get_schema_ref( ), ) ref = f"#/components/schemas/{name}" + # ### Identity provider + elif name == "Os_FederationIdentity_ProvidersGetResponse": + openapi_spec.components.schemas.setdefault( + name, + TypeSchema(**keystone_schemas.IDENTITY_PROVIDERS_SCHEMA), + ) + ref = f"#/components/schemas/{name}" + elif name in [ + "Os_FederationIdentity_ProviderGetResponse", + "Os_FederationIdentity_ProviderPutResponse", + "Os_FederationIdentity_ProviderPatchResponse", + ]: + openapi_spec.components.schemas.setdefault( + name, + TypeSchema( + **keystone_schemas.IDENTITY_PROVIDER_CONTAINER_SCHEMA + ), + ) + ref = f"#/components/schemas/{name}" + elif name == "Os_FederationIdentity_ProviderPutRequest": + openapi_spec.components.schemas.setdefault( + name, + TypeSchema(**keystone_schemas.IDENTITY_PROVIDER_CREATE_SCHEMA), + ) + ref = f"#/components/schemas/{name}" + elif name == "Os_FederationIdentity_ProviderPatchRequest": + openapi_spec.components.schemas.setdefault( + name, + TypeSchema(**keystone_schemas.IDENTITY_PROVIDER_UPDATE_SCHEMA), + ) + ref = f"#/components/schemas/{name}" + # ### Identity provider protocols + elif name == "Os_FederationIdentity_ProvidersProtocolsGetResponse": + openapi_spec.components.schemas.setdefault( + name, + TypeSchema( + **keystone_schemas.IDENTITY_PROVIDER_PROTOCOLS_SCHEMA + ), + ) + ref = f"#/components/schemas/{name}" + elif name in [ + "Os_FederationIdentity_ProvidersProtocolGetResponse", + "Os_FederationIdentity_ProvidersProtocolPutResponse", + "Os_FederationIdentity_ProvidersProtocolPatchResponse", + ]: + openapi_spec.components.schemas.setdefault( + name, + TypeSchema( + **keystone_schemas.IDENTITY_PROVIDER_PROTOCOL_CONTAINER_SCHEMA + ), + ) + ref = f"#/components/schemas/{name}" + elif name == "Os_FederationIdentity_ProvidersProtocolPutRequest": + openapi_spec.components.schemas.setdefault( + name, + TypeSchema( + **keystone_schemas.IDENTITY_PROVIDER_PROTOCOL_CREATE_SCHEMA + ), + ) + ref = f"#/components/schemas/{name}" + elif name == "Os_FederationIdentity_ProvidersProtocolPatchRequest": + openapi_spec.components.schemas.setdefault( + name, + TypeSchema( + **keystone_schemas.IDENTITY_PROVIDER_PROTOCOL_UPDATE_SCHEMA + ), + ) + ref = f"#/components/schemas/{name}" + # ### Identity provider mapping + elif name == "Os_FederationMappingsGetResponse": + openapi_spec.components.schemas.setdefault( + name, + TypeSchema(**keystone_schemas.MAPPINGS_SCHEMA), + ) + ref = f"#/components/schemas/{name}" + elif name in [ + "Os_FederationMappingGetResponse", + "Os_FederationMappingPutResponse", + "Os_FederationMappingPatchResponse", + ]: + openapi_spec.components.schemas.setdefault( + name, + TypeSchema(**keystone_schemas.MAPPING_CONTAINER_SCHEMA), + ) + ref = f"#/components/schemas/{name}" + elif name in [ + "Os_FederationMappingPutRequest", + "Os_FederationMappingPatchRequest", + ]: + openapi_spec.components.schemas.setdefault( + name, + TypeSchema(**keystone_schemas.MAPPING_CREATE_SCHEMA), + ) + ref = f"#/components/schemas/{name}" + # ### Identity provider service provider + elif name == "Os_FederationService_ProvidersGetResponse": + openapi_spec.components.schemas.setdefault( + name, + TypeSchema( + **keystone_schemas.FEDERATION_SERVICE_PROVIDERS_SCHEMA + ), + ) + ref = f"#/components/schemas/{name}" + elif name in [ + "Os_FederationService_ProviderGetResponse", + "Os_FederationService_ProviderPutResponse", + "Os_FederationService_ProviderPatchResponse", + ]: + openapi_spec.components.schemas.setdefault( + name, + TypeSchema( + **keystone_schemas.FEDERATION_SERVICE_PROVIDER_CONTAINER_SCHEMA + ), + ) + ref = f"#/components/schemas/{name}" + elif name == "Os_FederationService_ProviderPutRequest": + openapi_spec.components.schemas.setdefault( + name, + TypeSchema( + **keystone_schemas.FEDERATION_SERVICE_PROVIDER_CREATE_SCHEMA + ), + ) + ref = f"#/components/schemas/{name}" + elif name == "Os_FederationService_ProviderPatchRequest": + openapi_spec.components.schemas.setdefault( + name, + TypeSchema( + **keystone_schemas.FEDERATION_SERVICE_PROVIDER_UPDATE_SCHEMA + ), + ) + ref = f"#/components/schemas/{name}" + # SAML2 Metadata + elif name == "Os_FederationSaml2MetadataGetResponse": + mime_type = "text/xml" + openapi_spec.components.schemas.setdefault( + name, + TypeSchema( + type="string", + format="xml", + descripion="Identity Provider metadata information in XML format", + ), + ) + ref = f"#/components/schemas/{name}" + # Default else: (ref, mime_type) = super()._get_schema_ref( openapi_spec, name, description, action_name=action_name diff --git a/codegenerator/openapi/keystone_schemas.py b/codegenerator/openapi/keystone_schemas.py index e49746f..db499f5 100644 --- a/codegenerator/openapi/keystone_schemas.py +++ b/codegenerator/openapi/keystone_schemas.py @@ -14,11 +14,15 @@ from typing import Any +from jsonref import replace_refs + from keystone.application_credential import ( schema as application_credential_schema, ) from keystone.assignment import schema as assignment_schema from keystone.identity import schema as identity_schema +from keystone.federation import schema as federation_schema +from keystone.federation import utils as federation_mapping_schema from keystone.resource import schema as ks_schema @@ -1028,3 +1032,207 @@ "schema": {"type": "string"}, }, } + +IDENTITY_PROVIDER_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "The Identity Provider unique ID", + }, + "description": { + "type": "string", + "description": "The Identity Provider description", + }, + "domain_id": { + "type": "string", + "format": "uuid", + "description": "The ID of a domain that is associated with the Identity Provider.", + }, + "authorization_ttl": { + "type": "integer", + "description": "The length of validity in minutes for group memberships carried over through mapping and persisted in the database.", + }, + "enabled": { + "type": "boolean", + "description": "Whether the Identity Provider is enabled or not", + }, + "remote_ids": { + "type": "array", + "description": "List of the unique Identity Provider’s remote IDs", + "items": {"type": "string"}, + }, + }, +} + +IDENTITY_PROVIDER_CONTAINER_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": {"identity_provider": IDENTITY_PROVIDER_SCHEMA}, +} + +IDENTITY_PROVIDER_CREATE_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "identity_provider": federation_schema.identity_provider_create + }, +} + +IDENTITY_PROVIDER_UPDATE_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "identity_provider": federation_schema.identity_provider_update + }, +} + +IDENTITY_PROVIDERS_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "identity_providers": { + "type": "array", + "items": IDENTITY_PROVIDER_SCHEMA, + } + }, +} + +IDENTITY_PROVIDERS_LIST_PARAMETERS: dict[str, Any] = { + "idp_id": { + "in": "query", + "name": "id", + "description": "Filter for Identity Providers’ ID attribute", + "schema": {"type": "string"}, + }, + "idp_enabled": { + "in": "query", + "name": "enabled", + "description": "Filter for Identity Providers’ enabled attribute", + "schema": {"type": "boolean"}, + }, +} + +IDENTITY_PROVIDER_PROTOCOL_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "id": { + "type": "string", + "format": "uuid", + "description": "The federation protocol ID", + }, + "mapping_id": {"type": "string"}, + "remote_id_attribute": {"type": "string", "maxLength": 64}, + }, +} + +IDENTITY_PROVIDER_PROTOCOL_CONTAINER_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": {"protocol": IDENTITY_PROVIDER_PROTOCOL_SCHEMA}, +} + +IDENTITY_PROVIDER_PROTOCOLS_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "protocols": { + "type": "array", + "items": IDENTITY_PROVIDER_PROTOCOL_SCHEMA, + } + }, +} + +IDENTITY_PROVIDER_PROTOCOL_CREATE_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": {"protocol": federation_schema.protocol_create}, +} + +IDENTITY_PROVIDER_PROTOCOL_UPDATE_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": {"protocol": federation_schema.protocol_update}, +} + +MAPPING_PROPERTIES = replace_refs( + federation_mapping_schema.MAPPING_SCHEMA, proxies=False +) +MAPPING_PROPERTIES.pop("definitions", None) +MAPPING_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "The Federation Mapping unique ID", + }, + **MAPPING_PROPERTIES["properties"], + }, +} + +MAPPING_CONTAINER_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": {"mapping": MAPPING_SCHEMA}, +} + +MAPPINGS_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": {"mappings": {"type": "array", "items": MAPPING_SCHEMA}}, +} + +MAPPING_CREATE_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": {"mapping": MAPPING_PROPERTIES}, +} + +FEDERATION_SERVICE_PROVIDER_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "auth_url": { + "type": "string", + "description": "The URL to authenticate against", + }, + "description": { + "type": ["string", "null"], + "description": "The description of the Service Provider", + }, + "id": { + "type": "string", + "description": "The Service Provider unique ID", + }, + "enabled": { + "type": "boolean", + "description": "Whether the Service Provider is enabled or not", + }, + "relay_state_prefix": { + "type": ["string", "null"], + "description": "The prefix of the RelayState SAML attribute", + }, + "sp_url": { + "type": "string", + "description": "The Service Provider’s URL", + }, + }, + "required": ["auth_url", "sp_url"], +} + +FEDERATION_SERVICE_PROVIDER_CONTAINER_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": {"service_provider": FEDERATION_SERVICE_PROVIDER_SCHEMA}, +} + +FEDERATION_SERVICE_PROVIDERS_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "service_providers": { + "type": "array", + "items": FEDERATION_SERVICE_PROVIDER_SCHEMA, + } + }, +} + +FEDERATION_SERVICE_PROVIDER_CREATE_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "service_provider": federation_schema.service_provider_create + }, +} + +FEDERATION_SERVICE_PROVIDER_UPDATE_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "service_provider": federation_schema.service_provider_update + }, +} diff --git a/codegenerator/rust_cli.py b/codegenerator/rust_cli.py index 8002e07..dfb4752 100644 --- a/codegenerator/rust_cli.py +++ b/codegenerator/rust_cli.py @@ -322,9 +322,12 @@ class StringEnum(common_rust.StringEnum): class ArrayInput(common_rust.Array): - original_data_type: common_rust.BaseCompoundType | common_rust.BaseCombinedType | common_rust.BasePrimitiveType | None = ( - None - ) + original_data_type: ( + common_rust.BaseCompoundType + | common_rust.BaseCombinedType + | common_rust.BasePrimitiveType + | None + ) = None @property def clap_macros(self): @@ -536,9 +539,10 @@ def convert_model( # input conversion possible original_data_type = self.convert_model(item_type) # We are not interested to see unused data in the submodels - if not item_type.reference: - raise NotImplementedError - self.refs.pop(item_type.reference, None) + self.ignored_models.append(item_type) + # self.ignored_models.extend( + # x.data_type for x in item_type.fields.values() + # ) typ = self.data_type_mapping[model.Array]( description=common_rust.sanitize_rust_docstrings( type_model.description @@ -549,8 +553,6 @@ def convert_model( elif isinstance(item_type, model.Array) and isinstance( item_type.item_type, model.ConstraintString ): - if item_type.reference: - self.refs.pop(item_type.reference, None) original_data_type = self.convert_model(item_type) typ = self.data_type_mapping[model.Array]( description=common_rust.sanitize_rust_docstrings( @@ -582,7 +584,8 @@ def _get_struct_type(self, type_model: model.Struct) -> common_rust.Struct: is_nullable: bool = False field_data_type = self.convert_model(field.data_type) if isinstance(field_data_type, self.option_type_class): - # Unwrap Option into "is_nullable" NOTE: but perhaps + # Unwrap Option into "is_nullable" + # NOTE: but perhaps # Option