Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/update computational asset definition #363

Open
wants to merge 16 commits into
base: develop
Choose a base branch
from
Open
3 changes: 3 additions & 0 deletions src/database/model/agent/location.py
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,9 @@ class LocationORM(LocationBase, table=True): # type: ignore [call-arg]
event_identifier: int | None = Field(
sa_column=Column(Integer, ForeignKey("event.identifier", ondelete="CASCADE"))
)
computational_asset_identifier: int | None = Field(
sa_column=Column(Integer, ForeignKey("computational_asset.identifier", ondelete="CASCADE"))
)
Comment on lines +121 to +123
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For changes to tables which already exist in the database, we need to add a database schema migration script. Please see this documentation for more information.

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Perfect, thanks for the reference!


class RelationshipConfig:
address: Optional[Address] = OneToOne(deserializer=CastDeserializer(AddressORM))
Expand Down
48 changes: 48 additions & 0 deletions src/database/model/computational_asset/accelerator.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
from typing import TYPE_CHECKING
from database.model.field_length import NORMAL
from sqlmodel import Field, SQLModel, Relationship

if TYPE_CHECKING: # avoid circular imports; only import while type checking
from database.model.computational_asset.computational_asset import ComputationalAsset


class AcceleratorBase(SQLModel):
vendor: str | None = Field(
description="The manufacturer of the Accelerator.",
max_length=NORMAL,
schema_extra={"example": "NVIDIA"},
)
type: str | None = Field(
description="Accelerator type.",
max_length=NORMAL,
schema_extra={"example": "GPU"},
)
model_name: str | None = Field(
description="The name of the Accelerator model.",
max_length=NORMAL,
schema_extra={"example": "A100"},
)
architecture: str | None = Field(
description="The accelerator architecture.",
max_length=NORMAL,
schema_extra={"example": "Ampere"},
)
cores: int | None = Field(
description="The number of cores used by the Accelerator.",
schema_extra={"example": 8},
)
memory: float | None = Field(
description="The Accelerator memory (GB).",
schema_extra={"example": 64},
)


class AcceleratorORM(AcceleratorBase, table=True): # type: ignore [call-arg]
__tablename__ = "accelerator"
identifier: int = Field(default=None, primary_key=True)
computational_asset_identifier: int | None = Field(foreign_key="computational_asset.identifier")
computational_asset: "ComputationalAsset" = Relationship(back_populates="accelerator")


class Accelerator(AcceleratorBase):
pass
130 changes: 124 additions & 6 deletions src/database/model/computational_asset/computational_asset.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,21 @@
from typing import Optional

from sqlmodel import Field, Relationship
from typing import Optional
from database.model.field_length import NORMAL
from database.model.relationships import ManyToOne, OneToMany
from database.model.serializers import (
AttributeSerializer,
CastDeserializerList,
FindByNameDeserializer,
)

from database.model.ai_asset.ai_asset import AIAssetBase, AIAsset
from database.model.computational_asset.computational_asset_type import ComputationalAssetType
from database.model.field_length import NORMAL
from database.model.relationships import ManyToOne
from database.model.serializers import AttributeSerializer, FindByNameDeserializer

from database.model.computational_asset.cpu import Cpu, CpuORM
from database.model.computational_asset.memory import Memory, MemoryORM
from database.model.computational_asset.accelerator import Accelerator, AcceleratorORM
from database.model.computational_asset.storage import Storage, StorageORM
from database.model.agent.location import LocationORM, Location


class ComputationalAssetBase(AIAssetBase):
Expand All @@ -15,6 +24,25 @@ class ComputationalAssetBase(AIAssetBase):
max_length=NORMAL,
schema_extra={"example": "https://www.example.com/cluster-status"},
)
os: str | None = Field(
description="The operating system installed in the Computational Asset.",
max_length=NORMAL,
schema_extra={"example": "Redhat"},
)
kernel: str | None = Field(
description="TBC",
max_length=NORMAL,
schema_extra={"example": "Linux"},
)
pricing_scheme: str | None = Field(
description="A text describing the pricing scheme, or a URL to such text.",
schema_extra={
"example": "Academic use: Free for researchers and students with valid"
"institutional credentials. Commercial use: Based on resource consumption"
" - contact [email protected] or visit https://example.com/pricing for "
"current rates."
},
)


class ComputationalAsset(ComputationalAssetBase, AIAsset, table=True): # type: ignore [call-arg]
Expand All @@ -28,11 +56,18 @@ class ComputationalAsset(ComputationalAssetBase, AIAsset, table=True): # type:
"""

__tablename__ = "computational_asset"

identifier: int = Field(default=None, primary_key=True)
type_identifier: int | None = Field(
foreign_key=ComputationalAssetType.__tablename__ + ".identifier"
)
type: Optional[ComputationalAssetType] = Relationship()
cpu: list[CpuORM] = Relationship(sa_relationship_kwargs={"cascade": "all, delete"})
memory: list[MemoryORM] = Relationship(sa_relationship_kwargs={"cascade": "all, delete"})
accelerator: list[AcceleratorORM] = Relationship(
sa_relationship_kwargs={"cascade": "all, delete"}
)
storage: list[StorageORM] = Relationship(sa_relationship_kwargs={"cascade": "all, delete"})
location: list[LocationORM] = Relationship(sa_relationship_kwargs={"cascade": "all, delete"})

class RelationshipConfig(AIAsset.RelationshipConfig):
type: Optional[str] = ManyToOne(
Expand All @@ -42,3 +77,86 @@ class RelationshipConfig(AIAsset.RelationshipConfig):
deserializer=FindByNameDeserializer(ComputationalAssetType),
example="storage",
)

cpu: list[Cpu] | None = OneToMany(
default_factory_pydantic=list, # no deletion trigger: cascading delete is used
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would it make sense for the components (accelerator, cpu, etc.) to be many-to-many? E.g., one CPU model may be used in different computational assets. Perhaps this is not common enough to avoid duplication in the database?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@hbonell-iti @soniasantiago It's marked ready to (re)review but I would like to still know if what's the thought process here. It doesn't look like Cpu is intended to specify one particular unit, but rather a model. In which case I assume it can occur in different computational assets?

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Apologies for not answering to your comment before.
As you say, it's true that one CPU model could be used in different computational assets. However, due to the different attributes of the CPU model and its possible values, we believe it won't happen too often. This is the main reason why the relation is defined one-to-many.

However, if you think this is not the best approach, let us know and we will do in the other way.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am fine going with your judgement, I just wanted to ensure it wasn't an oversight.

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Perfect. I just wanted to make sure that this is done in the right way. Thanks for checking this!

description="The CPU of the Computational Asset.",
deserializer=CastDeserializerList(CpuORM),
example=[
{
"num_cpu_cores": 8,
"architecture": "x86_64",
"vendor": "CPU_AMD",
"model_name": "Ryzen 7 5800X",
"cpu_family": "Zen 3",
"clock_speed": 3.8,
}
],
)

memory: list[Memory] | None = OneToMany(
default_factory_pydantic=list, # no deletion trigger: cascading delete is used
description="The Memory of Computational Asset.",
deserializer=CastDeserializerList(MemoryORM),
example=[
{
"type": "DDR5",
"amount_gb": 32.0,
"read_bandwidth": 38400,
"write_bandwidth": 38400,
"rdma": "InfiniBand",
},
],
)
accelerator: list[Accelerator] | None = OneToMany(
default_factory_pydantic=list, # no deletion trigger: cascading delete is used
description="The Accelerator integrated into the Computational Asset.",
deserializer=CastDeserializerList(AcceleratorORM),
example=[
{
"vendor": "NVIDIA",
"type": "GPU",
"model_name": "A100",
"architecture": "Ampere",
"cores": 6912,
"memory": 80.0,
},
],
)
storage: list[Storage] | None = OneToMany(
default_factory_pydantic=list, # no deletion trigger: cascading delete is used
description="The Storage associated with the Computational Asset.",
deserializer=CastDeserializerList(StorageORM),
example=[
{
"model": "Samsung 990 PRO",
"vendor": "Samsung",
"amount": 2000,
"type": "NVMe SSD",
"read_bandwidth": 7450,
"write_bandwidth": 6900,
},
],
)
location: list[Location] | None = OneToMany(
default_factory_pydantic=list, # no deletion trigger: cascading delete is used
description="A geographical specification of where the resource resides.",
deserializer=CastDeserializerList(LocationORM),
example=[
{
"address": {
"street": "Gustav-Kicks-Allee 1",
"postal_code": "85748",
"locality": "Garching",
"region": "Bavaria",
"country": "DEU",
"address": "Gustav-Kicks-Allee 1, 85748 Garching bei München",
},
"geo": {
"latitude": 48.2676,
"longitude": 11.6718,
"elevation_millimeters": 482000,
},
},
],
)
48 changes: 48 additions & 0 deletions src/database/model/computational_asset/cpu.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
from typing import TYPE_CHECKING
from database.model.field_length import NORMAL
from sqlmodel import Field, SQLModel, Relationship

if TYPE_CHECKING: # avoid circular imports; only import while type checking
from database.model.computational_asset.computational_asset import ComputationalAsset


class CpuBase(SQLModel):
num_cpu_cores: int | None = Field(
description="The number of cores used by the CPU.",
schema_extra={"example": 8},
)
architecture: str | None = Field(
description="The CPU architecture.",
max_length=NORMAL,
schema_extra={"example": "ARM"},
)
vendor: str | None = Field(
description="The manufacturer of the CPU.",
max_length=NORMAL,
schema_extra={"example": "CPU_AMD"},
)
model_name: str | None = Field(
description="The name of the CPU model.",
max_length=NORMAL,
schema_extra={"example": "Athlon"},
)
cpu_family: str | None = Field(
description="The family in which the CPU model belongs.",
max_length=NORMAL,
schema_extra={"example": "Athlon"},
)
clock_speed: float | None = Field(
description="The CPU clock speed (GHz).",
schema_extra={"example": 3.2},
)


class CpuORM(CpuBase, table=True): # type: ignore [call-arg]
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As I read the metadata model, CPU should be a subclass of AIoD concept?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same for accelerator, and other added items?

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We thought about these entities to be similarly as "location". That is, not a subclass of AIoD concept, at least at the implementation level.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Then maybe I misunderstand how to read
this documentation. I am having a meeting with Antonis about this next week.

__tablename__ = "cpu"
identifier: int = Field(default=None, primary_key=True)
computational_asset_identifier: int | None = Field(foreign_key="computational_asset.identifier")
computational_asset: "ComputationalAsset" = Relationship(back_populates="cpu")


class Cpu(CpuBase):
pass
42 changes: 42 additions & 0 deletions src/database/model/computational_asset/memory.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
from typing import TYPE_CHECKING
from database.model.field_length import NORMAL
from sqlmodel import Field, SQLModel, Relationship

if TYPE_CHECKING: # avoid circular imports; only import while type checking
from database.model.computational_asset.computational_asset import ComputationalAsset


class MemoryBase(SQLModel):
type: str | None = Field(
description="Memory type",
max_length=NORMAL,
schema_extra={"example": "DDR5"},
)
amount_gb: float | None = Field(
description="The total memory capacity measured in Gigabytes.",
schema_extra={"example": 32},
)
read_bandwidth: int | None = Field(
PGijsbers marked this conversation as resolved.
Show resolved Hide resolved
description="The rate at which data can be retrieved in Megabytes per second.",
schema_extra={"example": 38400},
)
write_bandwidth: int | None = Field(
PGijsbers marked this conversation as resolved.
Show resolved Hide resolved
description="The rate at which data can be stored in Megabytes per second",
schema_extra={"example": 38400},
)
rdma: str | None = Field(
description="Tech. that enables 2 networked computers to exchange data in mainmemory.",
max_length=NORMAL,
schema_extra={"example": "InfiniBand"},
)


class MemoryORM(MemoryBase, table=True): # type: ignore [call-arg]
__tablename__ = "memory"
identifier: int = Field(default=None, primary_key=True)
computational_asset_identifier: int | None = Field(foreign_key="computational_asset.identifier")
computational_asset: "ComputationalAsset" = Relationship(back_populates="memory")


class Memory(MemoryBase):
pass
52 changes: 52 additions & 0 deletions src/database/model/computational_asset/storage.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
from typing import TYPE_CHECKING
from database.model.field_length import NORMAL
from sqlmodel import Field, SQLModel, Relationship

if TYPE_CHECKING: # avoid circular imports; only import while type checking
from database.model.computational_asset.computational_asset import ComputationalAsset


class StorageBase(SQLModel):
model: str | None = Field(
description="The full name of the model as provided by the manufacturer.",
max_length=NORMAL,
schema_extra={"example": "Model"},
)
vendor: str | None = Field(
description="The manufacturer of the Storage.",
max_length=NORMAL,
schema_extra={"example": "AMD"},
)
amount: int | None = Field(
PGijsbers marked this conversation as resolved.
Show resolved Hide resolved
description="The total storage capacity (GB).",
schema_extra={"example": 1024},
)
type: str | None = Field(
description="Storage type",
max_length=NORMAL,
schema_extra={"example": "SSD"},
)
read_bandwidth: int | None = Field(
PGijsbers marked this conversation as resolved.
Show resolved Hide resolved
description=(
"The rate at which data can be retrieved from the storage in Megabytes per second."
),
schema_extra={"example": 38400},
)
write_bandwidth: int | None = Field(
PGijsbers marked this conversation as resolved.
Show resolved Hide resolved
description=(
"Rate at which data can be transferred form computer and stored "
"onto storage in Megabytes per second."
),
schema_extra={"example": 38400},
)


class StorageORM(StorageBase, table=True): # type: ignore [call-arg]
__tablename__ = "storage"
identifier: int = Field(default=None, primary_key=True)
computational_asset_identifier: int | None = Field(foreign_key="computational_asset.identifier")
computational_asset: "ComputationalAsset" = Relationship(back_populates="storage")


class Storage(StorageBase):
pass
Loading