Compatibility check with dependencies support

.gitignore

@@ -16,3 +16,4 @@ __pycache__/
 /kafka_*/
 venv
 /karapace/version.py
+.run

karapace/dependency.py

@@ -0,0 +1,24 @@
+from typing import Optional, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from karapace.schema_models import ValidatedTypedSchema
+
+
+class Dependency:
+    def __init__(self, name: str, subject: str, version: int, schema: "ValidatedTypedSchema") -> None:
+        self.name = name
+        self.subject = subject
+        self.version = version
+        self.schema = schema
+
+    def get_schema(self) -> "ValidatedTypedSchema":
+        return self.schema
+
+    def identifier(self) -> str:
+        return self.name + "_" + self.subject + "_" + str(self.version)
+
+
+class DependencyVerifierResult:
+    def __init__(self, result: bool, message: Optional[str] = ""):
+        self.result = result
+        self.message = message

karapace/errors.py

@@ -1,3 +1,6 @@
+from typing import List, Union
+
+
 class VersionNotFoundException(Exception):
     pass
 
@@ -18,6 +21,14 @@ class InvalidSchemaType(Exception):
     pass
 
 
+class InvalidReferences(Exception):
+    pass
+
+
+class ReferencesNotSupportedException(Exception):
+    pass
+
+
 class SchemasNotFoundException(Exception):
     pass
 
@@ -38,6 +49,13 @@ class SubjectNotSoftDeletedException(Exception):
     pass
 
 
+class ReferenceExistsException(Exception):
+    def __init__(self, referenced_by: List, version: Union[int, str]):
+        super().__init__()
+        self.version = version
+        self.referenced_by = referenced_by
+
+
 class SubjectSoftDeletedException(Exception):
     pass
 

karapace/protobuf/compare_type_lists.py

@@ -0,0 +1,68 @@
+from itertools import chain
+from karapace.protobuf.compare_result import CompareResult, Modification
+from karapace.protobuf.compare_type_storage import CompareTypes
+from karapace.protobuf.enum_element import EnumElement
+from karapace.protobuf.exception import IllegalStateException
+from karapace.protobuf.message_element import MessageElement
+from karapace.protobuf.type_element import TypeElement
+from typing import List
+
+
+def compare_type_lists(
+    self_types_list: List[TypeElement],
+    other_types_list: List[TypeElement],
+    result: CompareResult,
+    compare_types: CompareTypes,
+) -> CompareResult:
+    self_types = {}
+    other_types = {}
+    self_indexes = {}
+    other_indexes = {}
+
+    type_: TypeElement
+    for i, type_ in enumerate(self_types_list):
+        self_types[type_.name] = type_
+        self_indexes[type_.name] = i
+        compare_types.add_self_type(compare_types.self_package_name, type_)
+
+    for i, type_ in enumerate(other_types_list):
+        other_types[type_.name] = type_
+        other_indexes[type_.name] = i
+        compare_types.add_other_type(compare_types.other_package_name, type_)
+
+    for name in chain(self_types.keys(), other_types.keys() - self_types.keys()):
+
+        result.push_path(str(name), True)
+
+        if self_types.get(name) is None and other_types.get(name) is not None:
+            if isinstance(other_types[name], MessageElement):
+                result.add_modification(Modification.MESSAGE_ADD)
+            elif isinstance(other_types[name], EnumElement):
+                result.add_modification(Modification.ENUM_ADD)
+            else:
+                raise IllegalStateException("Instance of element is not applicable")
+        elif self_types.get(name) is not None and other_types.get(name) is None:
+            if isinstance(self_types[name], MessageElement):
+                result.add_modification(Modification.MESSAGE_DROP)
+            elif isinstance(self_types[name], EnumElement):
+                result.add_modification(Modification.ENUM_DROP)
+            else:
+                raise IllegalStateException("Instance of element is not applicable")
+        else:
+            if other_indexes[name] != self_indexes[name]:
+                if isinstance(self_types[name], MessageElement):
+                    # incompatible type
+                    result.add_modification(Modification.MESSAGE_MOVE)
+                else:
+                    raise IllegalStateException("Instance of element is not applicable")
+            else:
+                if isinstance(self_types[name], MessageElement) and isinstance(other_types[name], MessageElement):
+                    self_types[name].compare(other_types[name], result, compare_types)
+                elif isinstance(self_types[name], EnumElement) and isinstance(other_types[name], EnumElement):
+                    self_types[name].compare(other_types[name], result, compare_types)
+                else:
+                    # incompatible type
+                    result.add_modification(Modification.TYPE_ALTER)
+        result.pop_path(True)
+
+    return result

karapace/protobuf/compare_type_storage.py

@@ -2,12 +2,12 @@
 from karapace.protobuf.compare_result import CompareResult
 from karapace.protobuf.exception import IllegalArgumentException
 from karapace.protobuf.proto_type import ProtoType
-from karapace.protobuf.type_element import TypeElement
 from typing import Dict, List, Optional, TYPE_CHECKING, Union
 
 if TYPE_CHECKING:
     from karapace.protobuf.field_element import FieldElement
     from karapace.protobuf.message_element import MessageElement
+    from karapace.protobuf.type_element import TypeElement
 
 
 def compute_name(t: ProtoType, result_path: List[str], package_name: str, types: dict) -> Optional[str]:
@@ -35,15 +35,15 @@ def compute_name(t: ProtoType, result_path: List[str], package_name: str, types:
 class CompareTypes:
     def __init__(self, self_package_name: str, other_package_name: str, result: CompareResult) -> None:
 
-        self.self_package_name = self_package_name
-        self.other_package_name = other_package_name
+        self.self_package_name = self_package_name or ""
+        self.other_package_name = other_package_name or ""
         self.self_types: Dict[str, Union[TypeRecord, TypeRecordMap]] = {}
         self.other_types: Dict[str, Union[TypeRecord, TypeRecordMap]] = {}
         self.locked_messages: List["MessageElement"] = []
         self.environment: List["MessageElement"] = []
         self.result = result
 
-    def add_a_type(self, prefix: str, package_name: str, type_element: TypeElement, types: dict) -> None:
+    def add_a_type(self, prefix: str, package_name: str, type_element: "TypeElement", types: dict) -> None:
         name: str
         if prefix:
             name = prefix + "." + type_element.name
@@ -65,10 +65,10 @@ def add_a_type(self, prefix: str, package_name: str, type_element: TypeElement,
         for t in type_element.nested_types:
             self.add_a_type(name, package_name, t, types)
 
-    def add_self_type(self, package_name: str, type_element: TypeElement) -> None:
+    def add_self_type(self, package_name: str, type_element: "TypeElement") -> None:
         self.add_a_type(package_name, package_name, type_element, self.self_types)
 
-    def add_other_type(self, package_name: str, type_element: TypeElement) -> None:
+    def add_other_type(self, package_name: str, type_element: "TypeElement") -> None:
         self.add_a_type(package_name, package_name, type_element, self.other_types)
 
     def get_self_type(self, t: ProtoType) -> Union[None, "TypeRecord", "TypeRecordMap"]:
@@ -119,12 +119,12 @@ def unlock_message(self, message: "MessageElement") -> bool:
 @dataclass
 class TypeRecord:
     package_name: str
-    type_element: TypeElement
+    type_element: "TypeElement"
 
 
 class TypeRecordMap(TypeRecord):
     def __init__(
-        self, package_name: str, type_element: TypeElement, key: Optional["FieldElement"], value: Optional["FieldElement"]
+        self, package_name: str, type_element: "TypeElement", key: Optional["FieldElement"], value: Optional["FieldElement"]
     ) -> None:
         super().__init__(package_name, type_element)
         self.key = key

karapace/protobuf/dependency.py

@@ -0,0 +1,56 @@
+from karapace.dependency import DependencyVerifierResult
+from karapace.protobuf.known_dependency import DependenciesHardcoded, KnownDependency
+from karapace.protobuf.one_of_element import OneOfElement
+from typing import List
+
+
+class ProtobufDependencyVerifier:
+    def __init__(self) -> None:
+        self.declared_types: List[str] = []
+        self.used_types: List[str] = []
+        self.import_path: List[str] = []
+
+    def add_declared_type(self, full_name: str) -> None:
+        self.declared_types.append(full_name)
+
+    def add_used_type(self, parent: str, element_type: str) -> None:
+        if element_type.find("map<") == 0:
+            end = element_type.find(">")
+            virgule = element_type.find(",")
+            key = element_type[4:virgule]
+            value = element_type[virgule + 1 : end]
+            value = value.strip()
+            self.used_types.append(parent + ";" + key)
+            self.used_types.append(parent + ";" + value)
+        else:
+            self.used_types.append(parent + ";" + element_type)
+
+    def add_import(self, import_name: str) -> None:
+        self.import_path.append(import_name)
+
+    def verify(self) -> DependencyVerifierResult:
+        declared_index = set(self.declared_types)
+        for used_type in self.used_types:
+            delimiter = used_type.rfind(";")
+            used_type_with_scope = ""
+            if delimiter != -1:
+                used_type_with_scope = used_type[:delimiter] + "." + used_type[delimiter + 1 :]
+                used_type = used_type[delimiter + 1 :]
+
+            if not (
+                used_type in DependenciesHardcoded.index
+                or KnownDependency.index_simple.get(used_type) is not None
+                or KnownDependency.index.get(used_type) is not None
+                or used_type in declared_index
+                or (delimiter != -1 and used_type_with_scope in declared_index)
+                or "." + used_type in declared_index
+            ):
+                return DependencyVerifierResult(False, f"type {used_type} is not defined")
+
+        return DependencyVerifierResult(True)
+
+
+def _process_one_of(verifier: ProtobufDependencyVerifier, package_name: str, parent_name: str, one_of: OneOfElement) -> None:
+    parent = package_name + "." + parent_name
+    for field in one_of.fields:
+        verifier.add_used_type(parent, field.element_type)

karapace/protobuf/exception.py

@@ -25,6 +25,10 @@ class ProtobufTypeException(Error):
     """Generic Protobuf type error."""
 
 
+class ProtobufUnresolvedDependencyException(ProtobufException):
+    """a Protobuf schema has unresolved dependency"""
+
+
 class SchemaParseException(ProtobufException):
     """Error while parsing a Protobuf schema descriptor."""
 

karapace/protobuf/field_element.py

@@ -1,5 +1,6 @@
 # Ported from square/wire:
 # wire-library/wire-schema/src/commonMain/kotlin/com/squareup/wire/schema/internal/parser/FieldElement.kt
+from karapace.errors import InvalidSchema
 from karapace.protobuf.compare_result import CompareResult, Modification
 from karapace.protobuf.compare_type_storage import TypeRecordMap
 from karapace.protobuf.field import Field
@@ -143,6 +144,11 @@ def compare_message(
 
         self_type_record = types.get_self_type(self_type)
         other_type_record = types.get_other_type(other_type)
+        if self_type_record is None:
+            raise InvalidSchema(f"Cannot resolve type {self_type}")
+        if other_type_record is None:
+            raise InvalidSchema(f"Cannot resolve type {other_type}")
+
         self_type_element: MessageElement = self_type_record.type_element
         other_type_element: MessageElement = other_type_record.type_element