From 968d351d72e7bd632e3cca65d6bb59f68c1712df Mon Sep 17 00:00:00 2001 From: Tuan Tran Date: Mon, 11 Dec 2023 09:51:32 -0800 Subject: [PATCH] revert NO-FAIR-CARD --- doc/reference/asset.rst | 41 ++++++++++++++++++++++++++++++------ src/fairseq2/assets/store.py | 4 ---- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/doc/reference/asset.rst b/doc/reference/asset.rst index e13a83d25..a3df93d61 100644 --- a/doc/reference/asset.rst +++ b/doc/reference/asset.rst @@ -6,9 +6,18 @@ fairseq2.asset ``fairseq2.asset`` provides API to load the different model using the "model cards" from different "stores". -A model card is a .YAML file that contains information about a model and instructs a -:py:class:`fairseq2.models.utils.generic_loaders.ModelLoader` on how to load the model into the memory. -A store is a place where all the model cards are stored. By default, fairseq2 will look up the following stores: + +.. autosummary:: + :toctree: generated/data + + AssetStore + AssetCard + +Model store +~~~~~~~~~~~ + +A store is a place where all the model cards are stored. In fairseq2, a store is accessed via +`fairseq2.assets.AssetStore`. Multiple stores are allowed. By default, fairseq2 will look up the following stores: * System asset store: Cards that are shared by all users. By default, the system store is `/etc/fairseq2/assets`, but this can be changed via the environment variable `FAIRSEQ2_ASSET_DIR` @@ -16,7 +25,25 @@ A store is a place where all the model cards are stored. By default, fairseq2 wi * User asset store: Cards that are only available to the user. By default, the user store is `~/.config/fairseq2/assets`, but this can be changed via the environment variable `FAIRSEQ2_USER_ASSET_DIR` -* (Internal only) Meta asset store: For Meta employees' convenience, we set up a central store that contains - model cards with e.g intermediate checkpoints, extra internal information etc. This store is registered automatically - when one logs into the Fair cluster. If you wish not to use this central store, set the environment variable - `NO_FAIR_CARD=ON` \ No newline at end of file +To register a new store, implement a :py:class:`fairseq2.assets.AssetMetadataProvider` and add them to +:py:class:`fairseq2.assets.asset_store`. Here is an example to register a new directory as a model store: + + from pathlib import Path + from fairseq2.assets import FileAssetMetadataProvider, asset_store + + my_dir = Path("/path/to/model_store") + asset_store.metadata_providers.append(FileAssetMetadataProvider(my_dir)) + + +Model store +~~~~~~~~~~~ + +A model card is a .YAML file that contains information about a model and instructs a +:py:class:`fairseq2.models.utils.generic_loaders.ModelLoader` on how to load the model into the memory. Each model card +must have 2 mandatory attributes: `name` and `checkpoint`. `name` will be used to identify the model card, and it must +be unique _across_ all +fairseq2 provides example cards for differen LLMs in +`fairseq2.assets.cards`. + +In fairseq2, a model card is accessed via :py:class:`fairseq2.assets.AssetCard`. Alternatively, one can call +:py:method:`fairseq2.assets.AssetMetadataProvider.get_metadata(name: str)` to get the meta data of a given model card name. diff --git a/src/fairseq2/assets/store.py b/src/fairseq2/assets/store.py index fd9f22daa..a4f142382 100644 --- a/src/fairseq2/assets/store.py +++ b/src/fairseq2/assets/store.py @@ -216,10 +216,6 @@ def _load_faircluster() -> None: if "FAIR_ENV_CLUSTER" not in os.environ: return - # Disable the central Fair model store i.e. for debugging - if os.getenv("NO_FAIR_CARD", None): - return - asset_store.env_resolvers.append(lambda: "faircluster") # This directory is meant to store cluster-wide asset cards.