-
Notifications
You must be signed in to change notification settings - Fork 7
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Feature/update computational asset definition #363
base: develop
Are you sure you want to change the base?
Changes from 13 commits
3c6b137
210e04d
567ed47
ad9b793
907e3d8
b581f3d
b542f7e
50e104a
a604b5f
57ce219
3321c71
7b8ecb6
c2259bd
503166d
44946b7
e62de9c
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,48 @@ | ||
from typing import TYPE_CHECKING | ||
from database.model.field_length import NORMAL | ||
from sqlmodel import Field, SQLModel, Relationship | ||
|
||
if TYPE_CHECKING: # avoid circular imports; only import while type checking | ||
from database.model.computational_asset.computational_asset import ComputationalAsset | ||
|
||
|
||
class AcceleratorBase(SQLModel): | ||
vendor: str | None = Field( | ||
description="The manufacturer of the Accelerator.", | ||
max_length=NORMAL, | ||
schema_extra={"example": "NVIDIA"}, | ||
) | ||
type: str | None = Field( | ||
description="Accelerator type.", | ||
max_length=NORMAL, | ||
schema_extra={"example": "GPU"}, | ||
) | ||
model_name: str | None = Field( | ||
description="The name of the Accelerator model.", | ||
max_length=NORMAL, | ||
schema_extra={"example": "A100"}, | ||
) | ||
architecture: str | None = Field( | ||
description="The accelerator architecture.", | ||
max_length=NORMAL, | ||
schema_extra={"example": "Ampere"}, | ||
) | ||
cores: int | None = Field( | ||
description="The number of cores used by the Accelerator.", | ||
schema_extra={"example": 8}, | ||
) | ||
memory: float | None = Field( | ||
description="The Accelerator memory (GB).", | ||
schema_extra={"example": 64}, | ||
) | ||
|
||
|
||
class AcceleratorORM(AcceleratorBase, table=True): # type: ignore [call-arg] | ||
__tablename__ = "accelerator" | ||
identifier: int = Field(default=None, primary_key=True) | ||
computational_asset_identifier: int | None = Field(foreign_key="computational_asset.identifier") | ||
computational_asset: "ComputationalAsset" = Relationship(back_populates="accelerator") | ||
|
||
|
||
class Accelerator(AcceleratorBase): | ||
pass |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,12 +1,21 @@ | ||
from typing import Optional | ||
|
||
from sqlmodel import Field, Relationship | ||
from typing import Optional | ||
from database.model.field_length import NORMAL | ||
from database.model.relationships import ManyToOne, OneToMany | ||
from database.model.serializers import ( | ||
AttributeSerializer, | ||
CastDeserializerList, | ||
FindByNameDeserializer, | ||
) | ||
|
||
from database.model.ai_asset.ai_asset import AIAssetBase, AIAsset | ||
from database.model.computational_asset.computational_asset_type import ComputationalAssetType | ||
from database.model.field_length import NORMAL | ||
from database.model.relationships import ManyToOne | ||
from database.model.serializers import AttributeSerializer, FindByNameDeserializer | ||
|
||
from database.model.computational_asset.cpu import Cpu, CpuORM | ||
from database.model.computational_asset.memory import Memory, MemoryORM | ||
from database.model.computational_asset.accelerator import Accelerator, AcceleratorORM | ||
from database.model.computational_asset.storage import Storage, StorageORM | ||
from database.model.agent.location import LocationORM, Location | ||
|
||
|
||
class ComputationalAssetBase(AIAssetBase): | ||
|
@@ -15,6 +24,25 @@ class ComputationalAssetBase(AIAssetBase): | |
max_length=NORMAL, | ||
schema_extra={"example": "https://www.example.com/cluster-status"}, | ||
) | ||
os: str | None = Field( | ||
description="The operating system installed in the Computational Asset.", | ||
max_length=NORMAL, | ||
schema_extra={"example": "Redhat"}, | ||
) | ||
kernel: str | None = Field( | ||
description="TBC", | ||
max_length=NORMAL, | ||
schema_extra={"example": "Linux"}, | ||
) | ||
pricing_scheme: str | None = Field( | ||
description="A text describing the pricing scheme, or a URL to such text.", | ||
schema_extra={ | ||
"example": "Academic use: Free for researchers and students with valid" | ||
"institutional credentials. Commercial use: Based on resource consumption" | ||
" - contact [email protected] or visit https://example.com/pricing for " | ||
"current rates." | ||
}, | ||
) | ||
|
||
|
||
class ComputationalAsset(ComputationalAssetBase, AIAsset, table=True): # type: ignore [call-arg] | ||
|
@@ -28,11 +56,18 @@ class ComputationalAsset(ComputationalAssetBase, AIAsset, table=True): # type: | |
""" | ||
|
||
__tablename__ = "computational_asset" | ||
|
||
identifier: int = Field(default=None, primary_key=True) | ||
type_identifier: int | None = Field( | ||
foreign_key=ComputationalAssetType.__tablename__ + ".identifier" | ||
) | ||
type: Optional[ComputationalAssetType] = Relationship() | ||
cpu: list[CpuORM] = Relationship(sa_relationship_kwargs={"cascade": "all, delete"}) | ||
memory: list[MemoryORM] = Relationship(sa_relationship_kwargs={"cascade": "all, delete"}) | ||
accelerator: list[AcceleratorORM] = Relationship( | ||
sa_relationship_kwargs={"cascade": "all, delete"} | ||
) | ||
storage: list[StorageORM] = Relationship(sa_relationship_kwargs={"cascade": "all, delete"}) | ||
location: list[LocationORM] = Relationship(sa_relationship_kwargs={"cascade": "all, delete"}) | ||
|
||
class RelationshipConfig(AIAsset.RelationshipConfig): | ||
type: Optional[str] = ManyToOne( | ||
|
@@ -42,3 +77,86 @@ class RelationshipConfig(AIAsset.RelationshipConfig): | |
deserializer=FindByNameDeserializer(ComputationalAssetType), | ||
example="storage", | ||
) | ||
|
||
cpu: list[Cpu] | None = OneToMany( | ||
default_factory_pydantic=list, # no deletion trigger: cascading delete is used | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Would it make sense for the components (accelerator, cpu, etc.) to be many-to-many? E.g., one CPU model may be used in different computational assets. Perhaps this is not common enough to avoid duplication in the database? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @hbonell-iti @soniasantiago It's marked ready to (re)review but I would like to still know if what's the thought process here. It doesn't look like There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Apologies for not answering to your comment before. However, if you think this is not the best approach, let us know and we will do in the other way. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I am fine going with your judgement, I just wanted to ensure it wasn't an oversight. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Perfect. I just wanted to make sure that this is done in the right way. Thanks for checking this! |
||
description="The CPU of the Computational Asset.", | ||
deserializer=CastDeserializerList(CpuORM), | ||
example=[ | ||
{ | ||
"num_cpu_cores": 8, | ||
"architecture": "x86_64", | ||
"vendor": "CPU_AMD", | ||
"model_name": "Ryzen 7 5800X", | ||
"cpu_family": "Zen 3", | ||
"clock_speed": 3.8, | ||
} | ||
], | ||
) | ||
|
||
memory: list[Memory] | None = OneToMany( | ||
default_factory_pydantic=list, # no deletion trigger: cascading delete is used | ||
description="The Memory of Computational Asset.", | ||
deserializer=CastDeserializerList(MemoryORM), | ||
example=[ | ||
{ | ||
"type": "DDR5", | ||
"amount_gb": 32.0, | ||
"read_bandwidth": 38400, | ||
"write_bandwidth": 38400, | ||
"rdma": "InfiniBand", | ||
}, | ||
], | ||
) | ||
accelerator: list[Accelerator] | None = OneToMany( | ||
default_factory_pydantic=list, # no deletion trigger: cascading delete is used | ||
description="The Accelerator integrated into the Computational Asset.", | ||
deserializer=CastDeserializerList(AcceleratorORM), | ||
example=[ | ||
{ | ||
"vendor": "NVIDIA", | ||
"type": "GPU", | ||
"model_name": "A100", | ||
"architecture": "Ampere", | ||
"cores": 6912, | ||
"memory": 80.0, | ||
}, | ||
], | ||
) | ||
storage: list[Storage] | None = OneToMany( | ||
default_factory_pydantic=list, # no deletion trigger: cascading delete is used | ||
description="The Storage associated with the Computational Asset.", | ||
deserializer=CastDeserializerList(StorageORM), | ||
example=[ | ||
{ | ||
"model": "Samsung 990 PRO", | ||
"vendor": "Samsung", | ||
"amount": 2000, | ||
"type": "NVMe SSD", | ||
"read_bandwidth": 7450, | ||
"write_bandwidth": 6900, | ||
}, | ||
], | ||
) | ||
location: list[Location] | None = OneToMany( | ||
default_factory_pydantic=list, # no deletion trigger: cascading delete is used | ||
description="A geographical specification of where the resource resides.", | ||
deserializer=CastDeserializerList(LocationORM), | ||
example=[ | ||
{ | ||
"address": { | ||
"street": "Gustav-Kicks-Allee 1", | ||
"postal_code": "85748", | ||
"locality": "Garching", | ||
"region": "Bavaria", | ||
"country": "DEU", | ||
"address": "Gustav-Kicks-Allee 1, 85748 Garching bei München", | ||
}, | ||
"geo": { | ||
"latitude": 48.2676, | ||
"longitude": 11.6718, | ||
"elevation_millimeters": 482000, | ||
}, | ||
}, | ||
], | ||
) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,48 @@ | ||
from typing import TYPE_CHECKING | ||
from database.model.field_length import NORMAL | ||
from sqlmodel import Field, SQLModel, Relationship | ||
|
||
if TYPE_CHECKING: # avoid circular imports; only import while type checking | ||
from database.model.computational_asset.computational_asset import ComputationalAsset | ||
|
||
|
||
class CpuBase(SQLModel): | ||
num_cpu_cores: int | None = Field( | ||
description="The number of cores used by the CPU.", | ||
schema_extra={"example": 8}, | ||
) | ||
architecture: str | None = Field( | ||
description="The CPU architecture.", | ||
max_length=NORMAL, | ||
schema_extra={"example": "ARM"}, | ||
) | ||
vendor: str | None = Field( | ||
description="The manufacturer of the CPU.", | ||
max_length=NORMAL, | ||
schema_extra={"example": "CPU_AMD"}, | ||
) | ||
model_name: str | None = Field( | ||
description="The name of the CPU model.", | ||
max_length=NORMAL, | ||
schema_extra={"example": "Athlon"}, | ||
) | ||
cpu_family: str | None = Field( | ||
description="The family in which the CPU model belongs.", | ||
max_length=NORMAL, | ||
schema_extra={"example": "Athlon"}, | ||
) | ||
clock_speed: float | None = Field( | ||
description="The CPU clock speed (GHz).", | ||
schema_extra={"example": 3.2}, | ||
) | ||
|
||
|
||
class CpuORM(CpuBase, table=True): # type: ignore [call-arg] | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. As I read the metadata model, CPU should be a subclass of AIoD concept? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Same for accelerator, and other added items? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We thought about these entities to be similarly as "location". That is, not a subclass of AIoD concept, at least at the implementation level. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Then maybe I misunderstand how to read |
||
__tablename__ = "cpu" | ||
identifier: int = Field(default=None, primary_key=True) | ||
computational_asset_identifier: int | None = Field(foreign_key="computational_asset.identifier") | ||
computational_asset: "ComputationalAsset" = Relationship(back_populates="cpu") | ||
|
||
|
||
class Cpu(CpuBase): | ||
pass |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,42 @@ | ||
from typing import TYPE_CHECKING | ||
from database.model.field_length import NORMAL | ||
from sqlmodel import Field, SQLModel, Relationship | ||
|
||
if TYPE_CHECKING: # avoid circular imports; only import while type checking | ||
from database.model.computational_asset.computational_asset import ComputationalAsset | ||
|
||
|
||
class MemoryBase(SQLModel): | ||
type: str | None = Field( | ||
description="Memory type", | ||
max_length=NORMAL, | ||
schema_extra={"example": "DDR5"}, | ||
) | ||
amount_gb: float | None = Field( | ||
description="The total memory capacity measured in Gigabytes.", | ||
schema_extra={"example": 32}, | ||
) | ||
read_bandwidth: int | None = Field( | ||
PGijsbers marked this conversation as resolved.
Show resolved
Hide resolved
|
||
description="The rate at which data can be retrieved in Megabytes per second.", | ||
schema_extra={"example": 38400}, | ||
) | ||
write_bandwidth: int | None = Field( | ||
PGijsbers marked this conversation as resolved.
Show resolved
Hide resolved
|
||
description="The rate at which data can be stored in Megabytes per second", | ||
schema_extra={"example": 38400}, | ||
) | ||
rdma: str | None = Field( | ||
description="Tech. that enables 2 networked computers to exchange data in mainmemory.", | ||
max_length=NORMAL, | ||
schema_extra={"example": "InfiniBand"}, | ||
) | ||
|
||
|
||
class MemoryORM(MemoryBase, table=True): # type: ignore [call-arg] | ||
__tablename__ = "memory" | ||
identifier: int = Field(default=None, primary_key=True) | ||
computational_asset_identifier: int | None = Field(foreign_key="computational_asset.identifier") | ||
computational_asset: "ComputationalAsset" = Relationship(back_populates="memory") | ||
|
||
|
||
class Memory(MemoryBase): | ||
pass |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,52 @@ | ||
from typing import TYPE_CHECKING | ||
from database.model.field_length import NORMAL | ||
from sqlmodel import Field, SQLModel, Relationship | ||
|
||
if TYPE_CHECKING: # avoid circular imports; only import while type checking | ||
from database.model.computational_asset.computational_asset import ComputationalAsset | ||
|
||
|
||
class StorageBase(SQLModel): | ||
model: str | None = Field( | ||
description="The full name of the model as provided by the manufacturer.", | ||
max_length=NORMAL, | ||
schema_extra={"example": "Model"}, | ||
) | ||
vendor: str | None = Field( | ||
description="The manufacturer of the Storage.", | ||
max_length=NORMAL, | ||
schema_extra={"example": "AMD"}, | ||
) | ||
amount: int | None = Field( | ||
PGijsbers marked this conversation as resolved.
Show resolved
Hide resolved
|
||
description="The total storage capacity (GB).", | ||
schema_extra={"example": 1024}, | ||
) | ||
type: str | None = Field( | ||
description="Storage type", | ||
max_length=NORMAL, | ||
schema_extra={"example": "SSD"}, | ||
) | ||
read_bandwidth: int | None = Field( | ||
PGijsbers marked this conversation as resolved.
Show resolved
Hide resolved
|
||
description=( | ||
"The rate at which data can be retrieved from the storage in Megabytes per second." | ||
), | ||
schema_extra={"example": 38400}, | ||
) | ||
write_bandwidth: int | None = Field( | ||
PGijsbers marked this conversation as resolved.
Show resolved
Hide resolved
|
||
description=( | ||
"Rate at which data can be transferred form computer and stored " | ||
"onto storage in Megabytes per second." | ||
), | ||
schema_extra={"example": 38400}, | ||
) | ||
|
||
|
||
class StorageORM(StorageBase, table=True): # type: ignore [call-arg] | ||
__tablename__ = "storage" | ||
identifier: int = Field(default=None, primary_key=True) | ||
computational_asset_identifier: int | None = Field(foreign_key="computational_asset.identifier") | ||
computational_asset: "ComputationalAsset" = Relationship(back_populates="storage") | ||
|
||
|
||
class Storage(StorageBase): | ||
pass |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
For changes to tables which already exist in the database, we need to add a database schema migration script. Please see this documentation for more information.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Perfect, thanks for the reference!