feat: isolate openapi generators with multiprocessing

Every service relies on oslo_config and oslo_policy. This leads to
conflicts during generation, testing, etc. In order to save hairs on my
head isolate them with multiprocessing.

.github/workflows/docs.yml

@@ -45,6 +45,7 @@ jobs:
 
   # Deployment job
   deploy:
+    if: github.ref == "refs/heads/main"
     permissions:
       contents: read
       pages: write

codegenerator/openapi/cinder.py

@@ -10,31 +10,50 @@
 #   License for the specific language governing permissions and limitations
 #   under the License.
 #
-
+from multiprocessing import Process
 from pathlib import Path
 
-from cinder import objects, rpc
-from cinder.api.openstack import api_version_request
-from cinder.common import config
-from cinder.tests.unit.test import Database as db_fixture
+from ruamel.yaml.scalarstring import LiteralScalarString
+
 from codegenerator.common.schema import SpecSchema
 from codegenerator.common.schema import TypeSchema
 from codegenerator.openapi.base import OpenStackServerSourceBase
 from codegenerator.openapi.utils import merge_api_ref_doc
-from ruamel.yaml.scalarstring import LiteralScalarString
-
-CONF = config.CONF
 
 
 class CinderV3Generator(OpenStackServerSourceBase):
     URL_TAG_MAP = {
         "/versions": "version",
     }
 
-    def __init__(self):
+    def _api_ver_major(self, ver):
+        return ver._ver_major
+
+    def _api_ver_minor(self, ver):
+        return ver._ver_minor
+
+    def _api_ver(self, ver):
+        return (ver._ver_major, ver._ver_minor)
+
+    def generate(self, target_dir, args):
+        proc = Process(target=self._generate, args=[target_dir, args])
+        proc.start()
+        proc.join()
+        if proc.exitcode != 0:
+            raise RuntimeError("Error generating Cinder OpenAPI schma")
+        return Path(target_dir, "openapi_specs", "block-storage", "v3.yaml")
+
+    def _generate(self, target_dir, args, *pargs, **kwargs):
+        from cinder import objects, rpc
+        from cinder.api.openstack import api_version_request
+        from cinder.common import config
+        from cinder.tests.unit.test import Database as db_fixture
+
         # Register all Cinder objects
         objects.register_all()
 
+        CONF = config.CONF
+
         self.api_version = api_version_request._MAX_API_VERSION
         self.min_api_version = api_version_request._MIN_API_VERSION
 
@@ -49,16 +68,6 @@ def __init__(self):
 
         self.router = router.APIRouter()
 
-    def _api_ver_major(self, ver):
-        return ver._ver_major
-
-    def _api_ver_minor(self, ver):
-        return ver._ver_minor
-
-    def _api_ver(self, ver):
-        return (ver._ver_major, ver._ver_minor)
-
-    def generate(self, target_dir, args, *pargs, **kwargs):
         work_dir = Path(target_dir)
         work_dir.mkdir(parents=True, exist_ok=True)
 

codegenerator/openapi/glance.py

@@ -11,26 +11,16 @@
 #   under the License.
 #
 import copy
+from multiprocessing import Process
 from pathlib import Path
 
+from jsonref import replace_refs
 import routes
+from ruamel.yaml.scalarstring import LiteralScalarString
+
 from codegenerator.common.schema import SpecSchema, TypeSchema
 from codegenerator.openapi.base import OpenStackServerSourceBase
 from codegenerator.openapi.utils import merge_api_ref_doc
-from glance.api.v2 import image_members
-from glance.api.v2 import images
-from glance.api.v2 import metadef_namespaces
-from glance.api.v2 import metadef_objects
-from glance.api.v2 import metadef_properties
-from glance.api.v2 import metadef_resource_types
-from glance.api.v2 import metadef_tags
-from glance.api.v2 import router
-from glance.api.v2 import tasks
-from glance.common import config
-from glance import schema as glance_schema
-from jsonref import replace_refs
-from oslo_config import fixture as cfg_fixture
-from ruamel.yaml.scalarstring import LiteralScalarString
 
 
 class GlanceGenerator(OpenStackServerSourceBase):
@@ -42,12 +32,6 @@ def __init__(self):
         self.api_version = "2.16"
         self.min_api_version = None
 
-        self._config_fixture = self.useFixture(cfg_fixture.Config())
-
-        config.parse_args(args=[])
-
-        self.router = router.API(routes.Mapper())
-
     def _api_ver_major(self, ver):
         return ver.ver_major
 
@@ -58,6 +42,24 @@ def _api_ver(self, ver):
         return (ver.ver_major, ver.ver_minor)
 
     def generate(self, target_dir, args):
+        proc = Process(target=self._generate, args=[target_dir, args])
+        proc.start()
+        proc.join()
+        if proc.exitcode != 0:
+            raise RuntimeError("Error generating Glance OpenAPI schma")
+        return Path(target_dir, "openapi_specs", "image", "v2.yaml")
+
+    def _generate(self, target_dir, args):
+        from glance.api.v2 import router
+        from glance.common import config
+        from oslo_config import fixture as cfg_fixture
+
+        self._config_fixture = self.useFixture(cfg_fixture.Config())
+
+        config.parse_args(args=[])
+
+        self.router = router.API(routes.Mapper())
+
         work_dir = Path(target_dir)
         work_dir.mkdir(parents=True, exist_ok=True)
 
@@ -109,6 +111,16 @@ def _get_schema_ref(
         schema_def=None,
         action_name=None,
     ):
+        from glance.api.v2 import image_members
+        from glance.api.v2 import images
+        from glance.api.v2 import metadef_namespaces
+        from glance.api.v2 import metadef_objects
+        from glance.api.v2 import metadef_properties
+        from glance.api.v2 import metadef_resource_types
+        from glance.api.v2 import metadef_tags
+        from glance.api.v2 import tasks
+        from glance import schema as glance_schema
+
         if name == "TasksListResponse":
             openapi_spec.components.schemas.setdefault(
                 name,

codegenerator/openapi/keystone.py

@@ -11,21 +11,24 @@
 #   under the License.
 #
 import inspect
+from multiprocessing import Process
 import logging
 from pathlib import Path
 
+from ruamel.yaml.scalarstring import LiteralScalarString
+
+from keystone.assignment import schema as assignment_schema
+from keystone.auth import schema as auth_schema
+from keystone.identity import schema as identity_schema
+from keystone.resource import schema as ks_schema
+
 from codegenerator.common.schema import ParameterSchema
 from codegenerator.common.schema import PathSchema
 from codegenerator.common.schema import SpecSchema
 from codegenerator.common.schema import TypeSchema
 from codegenerator.openapi.base import OpenStackServerSourceBase
 from codegenerator.openapi.utils import merge_api_ref_doc
-from keystone.assignment import schema as assignment_schema
-from keystone.auth import schema as auth_schema
-from keystone.identity import schema as identity_schema
-from keystone.resource import schema as ks_schema
-from keystone.server.flask import application
-from ruamel.yaml.scalarstring import LiteralScalarString
+
 
 PROJECT_SCHEMA = TypeSchema(
     type="object",
@@ -221,9 +224,6 @@ def __init__(self):
         self.api_version = "3.0"
         self.min_api_version = "3.14"
 
-        self.app = application.application_factory()
-        self.router = self.app.url_map
-
     def _api_ver_major(self, ver):
         return ver._ver_major
 
@@ -233,7 +233,20 @@ def _api_ver_minor(self, ver):
     def _api_ver(self, ver):
         return (ver._ver_major, ver._ver_minor)
 
-    def generate(self, target_dir, args, *pargs, **kwargs):
+    def generate(self, target_dir, args):
+        proc = Process(target=self._generate, args=[target_dir, args])
+        proc.start()
+        proc.join()
+        if proc.exitcode != 0:
+            raise RuntimeError("Error generating Keystone OpenAPI schma")
+        return Path(target_dir, "openapi_specs", "identity", "v3.yaml")
+
+    def _generate(self, target_dir, args, *pargs, **kwargs):
+        from keystone.server.flask import application
+
+        self.app = application.application_factory()
+        self.router = self.app.url_map
+
         work_dir = Path(target_dir)
         work_dir.mkdir(parents=True, exist_ok=True)
 

codegenerator/openapi/neutron.py

@@ -11,13 +11,12 @@
 #   under the License.
 #
 import logging
+from multiprocessing import Process, Manager
 from pathlib import Path
 import re
 import tempfile
 from typing import Any
 
-from multiprocessing import Process, Manager
-
 from routes.base import Route
 from ruamel.yaml.scalarstring import LiteralScalarString
 
@@ -286,11 +285,12 @@ def generate(self, target_dir, args):
         # NOTE(gtema): call me paranoic or stupid, but I just gave up fighting
         # agains oslo_config and oslo_policy with their global state. It is
         # just too painful and takes too much precious time. On multiple
-        # invokation with different config there are plenty of things remaining
+        # invocation with different config there are plenty of things remaining
         # in the old state. In order to workaroung this just process in
         # different processes.
         with Manager() as manager:
-            # Since we may process same route multiple times we need to have a shared state
+            # Since we may process same route multiple times we need to have a
+            # shared state
             processed_routes = manager.dict()
             # Base Neutron
             p = Process(
@@ -299,6 +299,8 @@ def generate(self, target_dir, args):
             )
             p.start()
             p.join()
+            if p.exitcode != 0:
+                raise RuntimeError("Error generating Neutron OpenAPI schma")
 
             # VPNaaS
             p = Process(
@@ -307,6 +309,8 @@ def generate(self, target_dir, args):
             )
             p.start()
             p.join()
+            if p.exitcode != 0:
+                raise RuntimeError("Error generating Neutron OpenAPI schma")
 
         (impl_path, openapi_spec) = self._read_spec(work_dir)
 

codegenerator/openapi/nova.py

@@ -10,8 +10,11 @@
 #   License for the specific language governing permissions and limitations
 #   under the License.
 #
+from multiprocessing import Process
 from pathlib import Path
 
+from ruamel.yaml.scalarstring import LiteralScalarString
+
 from codegenerator.common.schema import (
     SpecSchema,
     TypeSchema,
@@ -21,11 +24,6 @@
 from codegenerator.openapi.base import OpenStackServerSourceBase
 from codegenerator.openapi import nova_schemas
 from codegenerator.openapi.utils import merge_api_ref_doc
-from nova.api.openstack import api_version_request
-from nova.api.openstack.compute import routes
-from nova.api.openstack.compute.schemas import flavor_manage
-from nova.tests import fixtures as nova_fixtures
-from ruamel.yaml.scalarstring import LiteralScalarString
 
 
 class NovaGenerator(OpenStackServerSourceBase):
@@ -39,14 +37,6 @@ class NovaGenerator(OpenStackServerSourceBase):
         "/servers/{server_id}/tags": "server-tags",
     }
 
-    def __init__(self):
-        pass
-        self.useFixture(nova_fixtures.RPCFixture("nova.test"))
-        self.api_version = api_version_request._MAX_API_VERSION
-        self.min_api_version = api_version_request._MIN_API_VERSION
-
-        self.router = routes.APIRouterV21()
-
     def _api_ver_major(self, ver):
         return ver.ver_major
 
@@ -56,7 +46,17 @@ def _api_ver_minor(self, ver):
     def _api_ver(self, ver):
         return (ver.ver_major, ver.ver_minor)
 
-    def generate(self, target_dir, args):
+    def _generate(self, target_dir, args):
+        from nova.api.openstack import api_version_request
+        from nova.api.openstack.compute import routes
+        from nova.tests import fixtures as nova_fixtures
+
+        self.api_version = api_version_request._MAX_API_VERSION
+        self.min_api_version = api_version_request._MIN_API_VERSION
+
+        self.useFixture(nova_fixtures.RPCFixture("nova.test"))
+        self.router = routes.APIRouterV21()
+
         work_dir = Path(target_dir)
         work_dir.mkdir(parents=True, exist_ok=True)
 
@@ -107,6 +107,14 @@ def generate(self, target_dir, args):
 
         return impl_path
 
+    def generate(self, target_dir, args):
+        proc = Process(target=self._generate, args=[target_dir, args])
+        proc.start()
+        proc.join()
+        if proc.exitcode != 0:
+            raise RuntimeError("Error generating Compute OpenAPI schma")
+        return Path(target_dir, "openapi_specs", "compute", "v2.yaml")
+
     def _get_param_ref(
         self,
         openapi_spec,
@@ -145,6 +153,8 @@ def _get_schema_ref(
         schema_def=None,
         action_name=None,
     ):
+        from nova.api.openstack.compute.schemas import flavor_manage
+
         schema = None
         # NOTE(gtema): This must go away once scemas are merged directly to
         # Nova