diff --git a/docs/en/api/visualization.rst b/docs/en/api/visualization.rst index bf17a4148e..39b2de476f 100644 --- a/docs/en/api/visualization.rst +++ b/docs/en/api/visualization.rst @@ -36,3 +36,5 @@ visualization Backend WandbVisBackend ClearMLVisBackend NeptuneVisBackend + DVCLiveVisBackend + AimVisBackend diff --git a/docs/en/common_usage/better_optimizers.md b/docs/en/common_usage/better_optimizers.md index 63a1e56866..9f6bdd6c39 100644 --- a/docs/en/common_usage/better_optimizers.md +++ b/docs/en/common_usage/better_optimizers.md @@ -90,3 +90,66 @@ runner = Runner( ) runner.train() ``` + +## bitsandbytes + +[bitsandbytes](https://github.com/TimDettmers/bitsandbytes) provides `AdamW8bit`, `Adam8bit`, `Adagrad8bit`, `PagedAdam8bit`, `PagedAdamW8bit`, `LAMB8bit`, `LARS8bit`, `RMSprop8bit`, `Lion8bit`, `PagedLion8bit` and `SGD8bit` optimziers。 + +```{note} +If you use the optimizer provided by bitsandbytes, you need to upgrade mmengine to `0.8.5`. +``` + +- Installation + +```bash +pip install bitsandbytes +``` + +- Usage + +Take the `AdamW8bit` as an example. + +```python +runner = Runner( + model=ResNet18(), + work_dir='./work_dir', + train_dataloader=train_dataloader_cfg, + # To view the input parameters for AdamW8bit, you can refer to + # https://github.com/TimDettmers/bitsandbytes/blob/main/bitsandbytes/optim/adamw.py + optim_wrapper=dict(optimizer=dict(type='AdamW8bit', lr=1e-4, weight_decay=1e-2)), + train_cfg=dict(by_epoch=True, max_epochs=3), +) +runner.train() +``` + +## transformers + +[transformers](https://github.com/huggingface/transformers) provides `Adafactor` optimzier。 + +```{note} +If you use the optimizer provided by transformers, you need to upgrade mmengine to `0.8.5`. +``` + +- Installation + +```bash +pip install transformers +``` + +- Usage + +Take the `Adafactor` as an example. + +```python +runner = Runner( + model=ResNet18(), + work_dir='./work_dir', + train_dataloader=train_dataloader_cfg, + # To view the input parameters for Adafactor, you can refer to + # https://github.com/huggingface/transformers/blob/v4.33.2/src/transformers/optimization.py#L492 + optim_wrapper=dict(optimizer=dict(type='Adafactor', lr=1e-5, + weight_decay=1e-2, scale_parameter=False, relative_step=False)), + train_cfg=dict(by_epoch=True, max_epochs=3), +) +runner.train() +``` diff --git a/docs/en/common_usage/visualize_training_log.md b/docs/en/common_usage/visualize_training_log.md index d85ba075a9..239220f3f7 100644 --- a/docs/en/common_usage/visualize_training_log.md +++ b/docs/en/common_usage/visualize_training_log.md @@ -1,6 +1,6 @@ # Visualize Training Logs -MMEngine integrates experiment management tools such as [TensorBoard](https://www.tensorflow.org/tensorboard), [Weights & Biases (WandB)](https://docs.wandb.ai/), [MLflow](https://mlflow.org/docs/latest/index.html), [ClearML](https://clear.ml/docs/latest/docs) and [Neptune](https://docs.neptune.ai/), making it easy to track and visualize metrics like loss and accuracy. +MMEngine integrates experiment management tools such as [TensorBoard](https://www.tensorflow.org/tensorboard), [Weights & Biases (WandB)](https://docs.wandb.ai/), [MLflow](https://mlflow.org/docs/latest/index.html), [ClearML](https://clear.ml/docs/latest/docs), [Neptune](https://docs.neptune.ai/), [DVCLive](https://dvc.org/doc/dvclive) and [Aim](https://aimstack.readthedocs.io/en/latest/overview.html), making it easy to track and visualize metrics like loss and accuracy. Below, we'll show you how to configure an experiment management tool in just one line, based on the example from [15 minutes to get started with MMEngine](../get_started/15_minutes.md). @@ -149,3 +149,88 @@ runner.train() ``` More initialization configuration parameters are available at [neptune.init_run API](https://docs.neptune.ai/api/neptune/#init_run). + +## DVCLive + +Before using DVCLive, you need to install `dvclive` dependency library and refer to [iterative.ai](https://dvc.org/doc/start) for configuration. Common configurations are as follows: + +```bash +pip install dvclive +cd ${WORK_DIR} +git init +dvc init +git commit -m "DVC init" +``` + +Configure the `Runner` in the initialization parameters of the Runner, and set `vis_backends` to [DVCLiveVisBackend](mmengine.visualization.DVCLiveVisBackend). + +```python +runner = Runner( + model=MMResNet50(), + work_dir='./work_dir_dvc', + train_dataloader=train_dataloader, + optim_wrapper=dict(optimizer=dict(type=SGD, lr=0.001, momentum=0.9)), + train_cfg=dict(by_epoch=True, max_epochs=5, val_interval=1), + val_dataloader=val_dataloader, + val_cfg=dict(), + val_evaluator=dict(type=Accuracy), + visualizer=dict(type='Visualizer', vis_backends=[dict(type='DVCLiveVisBackend')]), +) +runner.train() +``` + +```{note} +Recommend not to set `work_dir` as `work_dirs`. Or DVC will give a warning `WARNING:dvclive:Error in cache: bad DVC file name 'work_dirs\xxx.dvc' is git-ignored` if you run experiments in a OpenMMLab's repo. +``` + +Open the `report.html` file under `work_dir_dvc`, and you will see the visualization as shown in the following image. + +![image](https://github.com/open-mmlab/mmengine/assets/58739961/47d85520-9a4a-4143-a449-12ed7347cc63) + +You can also configure a VSCode extension of [DVC](https://marketplace.visualstudio.com/items?itemName=Iterative.dvc) to visualize the training process. + +More initialization configuration parameters are available at [DVCLive API Reference](https://dvc.org/doc/dvclive/live). + +## Aim + +Before using Aim, you need to install `aim` dependency library. + +```bash +pip install aim +``` + +Configure the `Runner` in the initialization parameters of the Runner, and set `vis_backends` to [AimVisBackend](mmengine.visualization.AimVisBackend). + +```python +runner = Runner( + model=MMResNet50(), + work_dir='./work_dir', + train_dataloader=train_dataloader, + optim_wrapper=dict(optimizer=dict(type=SGD, lr=0.001, momentum=0.9)), + train_cfg=dict(by_epoch=True, max_epochs=5, val_interval=1), + val_dataloader=val_dataloader, + val_cfg=dict(), + val_evaluator=dict(type=Accuracy), + visualizer=dict(type='Visualizer', vis_backends=[dict(type='AimVisBackend')]), +) +runner.train() +``` + +In the terminal, use the following command, + +```bash +aim up +``` + +or in the Jupyter Notebook, use the following command, + +```bash +%load_ext aim +%aim up +``` + +to launch the Aim UI as shown below. + +![image](https://github.com/open-mmlab/mmengine/assets/58739961/2fc6cdd8-1de7-4125-a20a-c95c1a8bdb1b) + +Initialization configuration parameters are available at [Aim SDK Reference](https://aimstack.readthedocs.io/en/latest/refs/sdk.html#module-aim.sdk.run). diff --git a/docs/zh_cn/api/visualization.rst b/docs/zh_cn/api/visualization.rst index bf17a4148e..39b2de476f 100644 --- a/docs/zh_cn/api/visualization.rst +++ b/docs/zh_cn/api/visualization.rst @@ -36,3 +36,5 @@ visualization Backend WandbVisBackend ClearMLVisBackend NeptuneVisBackend + DVCLiveVisBackend + AimVisBackend diff --git a/docs/zh_cn/common_usage/better_optimizers.md b/docs/zh_cn/common_usage/better_optimizers.md index a70c84b7f0..467d7f87e9 100644 --- a/docs/zh_cn/common_usage/better_optimizers.md +++ b/docs/zh_cn/common_usage/better_optimizers.md @@ -90,3 +90,34 @@ runner = Runner( ) runner.train() ``` + +## bitsandbytes + +[bitsandbytes](https://github.com/TimDettmers/bitsandbytes) 提供了 `AdamW8bit`、`Adam8bit`、`Adagrad8bit`、`PagedAdam8bit`、`PagedAdamW8bit`、`LAMB8bit`、 `LARS8bit`、`RMSprop8bit`、`Lion8bit`、`PagedLion8bit` 和 `SGD8bit` 优化器。 + +```{note} +如使用 D-Adaptation 提供的优化器，需将 mmengine 升级至 `0.8.5`。 +``` + +- 安装 + +```bash +pip install bitsandbytes +``` + +- 使用 + +以 `AdamW8bit` 为例。 + +```python +runner = Runner( + model=ResNet18(), + work_dir='./work_dir', + train_dataloader=train_dataloader_cfg, + # 如需查看 AdamW8bit 的输入参数，可查看 + # https://github.com/TimDettmers/bitsandbytes/blob/main/bitsandbytes/optim/adamw.py + optim_wrapper=dict(optimizer=dict(type='AdamW8bit', lr=1e-4, weight_decay=1e-2)), + train_cfg=dict(by_epoch=True, max_epochs=3), +) +runner.train() +``` diff --git a/docs/zh_cn/common_usage/visualize_training_log.md b/docs/zh_cn/common_usage/visualize_training_log.md index 36830df96d..ede2152385 100644 --- a/docs/zh_cn/common_usage/visualize_training_log.md +++ b/docs/zh_cn/common_usage/visualize_training_log.md @@ -1,8 +1,8 @@ # 可视化训练日志 -MMEngine 集成了 [TensorBoard](https://www.tensorflow.org/tensorboard?hl=zh-cn)、[Weights & Biases (WandB)](https://docs.wandb.ai/)、[MLflow](https://mlflow.org/docs/latest/index.html) 、[ClearML](https://clear.ml/docs/latest/docs) 和 [Neptune](https://docs.neptune.ai/) 实验管理工具，你可以很方便地跟踪和可视化损失及准确率等指标。 +MMEngine 集成了 [TensorBoard](https://www.tensorflow.org/tensorboard?hl=zh-cn)、[Weights & Biases (WandB)](https://docs.wandb.ai/)、[MLflow](https://mlflow.org/docs/latest/index.html) 、[ClearML](https://clear.ml/docs/latest/docs)、[Neptune](https://docs.neptune.ai/)、[DVCLive](https://dvc.org/doc/dvclive) 和 [Aim](https://aimstack.readthedocs.io/en/latest/overview.html) 实验管理工具，你可以很方便地跟踪和可视化损失及准确率等指标。 -下面基于[15 分钟上手 MMENGINE](../get_started/15_minutes.md)中的例子介绍如何一行配置实验管理工具。 +下面基于 [15 分钟上手 MMENGINE](../get_started/15_minutes.md) 中的例子介绍如何一行配置实验管理工具。 ## TensorBoard @@ -149,3 +149,88 @@ runner.train() ``` 更多初始化配置参数可点击 [neptune.init_run API](https://docs.neptune.ai/api/neptune/#init_run) 查询。 + +## DVCLive + +使用 DVCLive 前需先安装依赖库 `dvclive` 并参考 [iterative.ai](https://dvc.org/doc/start) 进行配置。常见的配置方式如下： + +```bash +pip install dvclive +cd ${WORK_DIR} +git init +dvc init +git commit -m "DVC init" +``` + +设置 `Runner` 初始化参数中的 `visualizer`，并将 `vis_backends` 设置为 [DVCLiveVisBackend](mmengine.visualization.DVCLiveVisBackend)。 + +```python +runner = Runner( + model=MMResNet50(), + work_dir='./work_dir_dvc', + train_dataloader=train_dataloader, + optim_wrapper=dict(optimizer=dict(type=SGD, lr=0.001, momentum=0.9)), + train_cfg=dict(by_epoch=True, max_epochs=5, val_interval=1), + val_dataloader=val_dataloader, + val_cfg=dict(), + val_evaluator=dict(type=Accuracy), + visualizer=dict(type='Visualizer', vis_backends=[dict(type='DVCLiveVisBackend')]), +) +runner.train() +``` + +```{note} +推荐将 `work_dir` 设置为 `work_dirs`。否则，你在 OpenMMLab 仓库中运行试验时，DVC 会给出警告 `WARNING:dvclive:Error in cache: bad DVC file name 'work_dirs\xxx.dvc' is git-ignored`。 +``` + +打开 `work_dir_dvc` 下面的 `report.html` 文件，即可看到如下图的可视化效果。 + +![image](https://github.com/open-mmlab/mmengine/assets/58739961/47d85520-9a4a-4143-a449-12ed7347cc63) + +你还可以安装 VSCode 扩展 [DVC](https://marketplace.visualstudio.com/items?itemName=Iterative.dvc) 进行可视化。 + +更多初始化配置参数可点击 [DVCLive API Reference](https://dvc.org/doc/dvclive/live) 查询。 + +## Aim + +使用 Aim 前需先安装依赖库 `aim`。 + +```bash +pip install aim +``` + +设置 `Runner` 初始化参数中的 `visualizer`，并将 `vis_backends` 设置为 [AimVisBackend](mmengine.visualization.AimVisBackend)。 + +```python +runner = Runner( + model=MMResNet50(), + work_dir='./work_dir', + train_dataloader=train_dataloader, + optim_wrapper=dict(optimizer=dict(type=SGD, lr=0.001, momentum=0.9)), + train_cfg=dict(by_epoch=True, max_epochs=5, val_interval=1), + val_dataloader=val_dataloader, + val_cfg=dict(), + val_evaluator=dict(type=Accuracy), + visualizer=dict(type='Visualizer', vis_backends=[dict(type='AimVisBackend')]), +) +runner.train() +``` + +在终端中输入 + +```bash +aim up +``` + +或者在 Jupyter Notebook 中输入 + +```bash +%load_ext aim +%aim up +``` + +即可启动 Aim UI，界面如下图所示。 + +![image](https://github.com/open-mmlab/mmengine/assets/58739961/2fc6cdd8-1de7-4125-a20a-c95c1a8bdb1b) + +初始化配置参数可点击 [Aim SDK Reference](https://aimstack.readthedocs.io/en/latest/refs/sdk.html#module-aim.sdk.run) 查询。 diff --git a/mmengine/config/config.py b/mmengine/config/config.py index f636cc81d5..316ac65d4d 100644 --- a/mmengine/config/config.py +++ b/mmengine/config/config.py @@ -17,6 +17,7 @@ from pathlib import Path from typing import Any, Optional, Sequence, Tuple, Union +import yapf from addict import Dict from rich.console import Console from rich.text import Text @@ -1472,8 +1473,11 @@ def _format_dict(input_dict, outest_level=False): blank_line_before_nested_class_or_def=True, split_before_expression_after_opening_paren=True) try: - text, _ = FormatCode( - text, style_config=yapf_style, verify=True) + if digit_version(yapf.__version__) >= digit_version('0.40.2'): + text, _ = FormatCode(text, style_config=yapf_style) + else: + text, _ = FormatCode( + text, style_config=yapf_style, verify=True) except: # noqa: E722 raise SyntaxError('Failed to format the config file, please ' f'check the syntax of: \n{text}') diff --git a/mmengine/hooks/logger_hook.py b/mmengine/hooks/logger_hook.py index ebea51ad66..fa0b79dcf9 100644 --- a/mmengine/hooks/logger_hook.py +++ b/mmengine/hooks/logger_hook.py @@ -46,7 +46,7 @@ class LoggerHook(Hook): checkpoints. If not specified, ``runner.work_dir`` will be used by default. If specified, the ``out_dir`` will be the concatenation of ``out_dir`` and the last level directory of ``runner.work_dir``. - For example, if the input ``our_dir`` is ``./tmp`` and + For example, if the input ``out_dir`` is ``./tmp`` and ``runner.work_dir`` is ``./work_dir/cur_exp``, then the log will be saved in ``./tmp/cur_exp``. Defaults to None. out_suffix (Tuple[str] or str): Those files in ``runner._log_dir`` diff --git a/mmengine/optim/optimizer/builder.py b/mmengine/optim/optimizer/builder.py index 9dcfde0e36..6543dacdd2 100644 --- a/mmengine/optim/optimizer/builder.py +++ b/mmengine/optim/optimizer/builder.py @@ -129,6 +129,49 @@ def register_sophia_optimizers() -> List[str]: SOPHIA_OPTIMIZERS = register_sophia_optimizers() +def register_bitsandbytes_optimizers() -> List[str]: + """Register optimizers in ``bitsandbytes`` to the ``OPTIMIZERS`` registry. + + Returns: + List[str]: A list of registered optimizers' name. + """ + dadaptation_optimizers = [] + try: + import bitsandbytes as bnb + except ImportError: + pass + else: + for module_name in [ + 'AdamW8bit', 'Adam8bit', 'Adagrad8bit', 'PagedAdam8bit', + 'PagedAdamW8bit', 'LAMB8bit', 'LARS8bit', 'RMSprop8bit', + 'Lion8bit', 'PagedLion8bit', 'SGD8bit' + ]: + _optim = getattr(bnb.optim, module_name) + if inspect.isclass(_optim) and issubclass(_optim, + torch.optim.Optimizer): + OPTIMIZERS.register_module(module=_optim) + dadaptation_optimizers.append(module_name) + return dadaptation_optimizers + + +BITSANDBYTES_OPTIMIZERS = register_bitsandbytes_optimizers() + + +def register_transformers_optimizers(): + transformer_optimizers = [] + try: + from transformers import Adafactor + except ImportError: + pass + else: + OPTIMIZERS.register_module(name='Adafactor', module=Adafactor) + transformer_optimizers.append('Adafactor') + return transformer_optimizers + + +TRANSFORMERS_OPTIMIZERS = register_transformers_optimizers() + + def build_optim_wrapper(model: nn.Module, cfg: Union[dict, Config, ConfigDict]) -> OptimWrapper: """Build function of OptimWrapper. diff --git a/mmengine/runner/log_processor.py b/mmengine/runner/log_processor.py index d3f9d95714..0453377d0f 100644 --- a/mmengine/runner/log_processor.py +++ b/mmengine/runner/log_processor.py @@ -13,7 +13,7 @@ from mmengine.registry import LOG_PROCESSORS -@LOG_PROCESSORS.register_module() # type: ignore +@LOG_PROCESSORS.register_module() class LogProcessor: """A log processor used to format log information collected from ``runner.message_hub.log_scalars``. @@ -24,7 +24,7 @@ class LogProcessor: ``custom_cfg`` of constructor can control the statistics method of logs. Args: - window_size (int): default smooth interval Defaults to 10. + window_size (int): default smooth interval. Defaults to 10. by_epoch (bool): Whether to format logs with epoch stype. Defaults to True. custom_cfg (list[dict], optional): Contains multiple log config dict, @@ -35,7 +35,7 @@ class LogProcessor: - If custom_cfg is None, all logs will be formatted via default methods, such as smoothing loss by default window_size. If custom_cfg is defined as a list of config dict, for example: - [dict(data_src=loss, method='mean', log_name='global_loss', + [dict(data_src='loss', method='mean', log_name='global_loss', window_size='global')]. It means the log item ``loss`` will be counted as global mean and additionally logged as ``global_loss`` (defined by ``log_name``). If ``log_name`` is not defined in @@ -43,8 +43,8 @@ class LogProcessor: - The original log item cannot be overwritten twice. Here is an error example: - [dict(data_src=loss, method='mean', window_size='global'), - dict(data_src=loss, method='mean', window_size='epoch')]. + [dict(data_src='loss', method='mean', window_size='global'), + dict(data_src='loss', method='mean', window_size='epoch')]. Both log config dict in custom_cfg do not have ``log_name`` key, which means the loss item will be overwritten twice. @@ -52,7 +52,7 @@ class LogProcessor: if ``by_epoch`` is set to False, ``windows_size`` should not be `epoch` to statistics log value by epoch. num_digits (int): The number of significant digit shown in the - logging message. + logging message. Defaults to 4. log_with_hierarchy (bool): Whether to log with hierarchy. If it is True, the information is written to visualizer backend such as :obj:`LocalVisBackend` and :obj:`TensorboardBackend` @@ -122,7 +122,7 @@ def __init__(self, def get_log_after_iter(self, runner, batch_idx: int, mode: str) -> Tuple[dict, str]: - """Format log string after training, validation or testing epoch. + """Format log string after training, validation or testing iteration. Args: runner (Runner): The runner of training phase. @@ -131,7 +131,7 @@ def get_log_after_iter(self, runner, batch_idx: int, mode (str): Current mode of runner, train, test or val. Return: - Tuple(dict, str): Formatted log dict/string which will be + Tuple[dict, str]: Formatted log dict/string which will be recorded by :obj:`runner.message_hub` and :obj:`runner.visualizer`. """ assert mode in ['train', 'test', 'val'] @@ -139,11 +139,11 @@ def get_log_after_iter(self, runner, batch_idx: int, parsed_cfg = self._parse_windows_size(runner, batch_idx, self.custom_cfg) # log_tag is used to write log information to terminal + log_tag = self._collect_scalars(parsed_cfg, runner, mode) + # If `self.log_with_hierarchy` is False, the tag is the same as # log_tag. Otherwise, each key in tag starts with prefix `train`, # `test` or `val` - log_tag = self._collect_scalars(parsed_cfg, runner, mode) - if not self.log_with_hierarchy: tag = copy.deepcopy(log_tag) else: @@ -259,7 +259,7 @@ def get_log_after_epoch(self, returned tag. Defaults to False. Return: - Tuple(dict, str): Formatted log dict/string which will be + Tuple[dict, str]: Formatted log dict/string which will be recorded by :obj:`runner.message_hub` and :obj:`runner.visualizer`. """ assert mode in [ @@ -377,12 +377,11 @@ def _collect_scalars(self, tag[key] = mode_history_scalars[key].current() # Update custom keys. for log_cfg in custom_cfg: - if not reserve_prefix: - data_src = log_cfg.pop('data_src') - log_name = f"{log_cfg.pop('log_name', data_src)}" - else: - data_src = f"{mode}/{log_cfg.pop('data_src')}" - log_name = f"{mode}/{log_cfg.pop('log_name', data_src)}" + data_src = log_cfg.pop('data_src') + log_name = log_cfg.pop('log_name', data_src) + if reserve_prefix: + data_src = f'{mode}/{data_src}' + log_name = f'{mode}/{log_name}' # log item in custom_cfg could only exist in train or val # mode. if data_src in mode_history_scalars: diff --git a/mmengine/runner/runner.py b/mmengine/runner/runner.py index bd6757a844..d66262c559 100644 --- a/mmengine/runner/runner.py +++ b/mmengine/runner/runner.py @@ -463,7 +463,7 @@ def from_cfg(cls, cfg: ConfigType) -> 'Runner': load_from=cfg.get('load_from'), resume=cfg.get('resume', False), launcher=cfg.get('launcher', 'none'), - env_cfg=cfg.get('env_cfg'), # type: ignore + env_cfg=cfg.get('env_cfg', dict(dist_cfg=dict(backend='nccl'))), log_processor=cfg.get('log_processor'), log_level=cfg.get('log_level', 'INFO'), visualizer=cfg.get('visualizer'), diff --git a/mmengine/utils/misc.py b/mmengine/utils/misc.py index 948329f603..15c1f89fae 100644 --- a/mmengine/utils/misc.py +++ b/mmengine/utils/misc.py @@ -519,7 +519,7 @@ def get_object_from_string(obj_name: str): try: module = import_module(module_name) part = next(parts) - # mmcv.ops has nms.py has nms function at the same time. So the + # mmcv.ops has nms.py and nms function at the same time. So the # function will have a higher priority obj = getattr(module, part, None) if obj is not None and not ismodule(obj): @@ -532,11 +532,12 @@ def get_object_from_string(obj_name: str): return None # get class or attribute from module + obj = module while True: try: - obj_cls = getattr(module, part) + obj = getattr(obj, part) part = next(parts) except StopIteration: - return obj_cls + return obj except AttributeError: return None diff --git a/mmengine/visualization/__init__.py b/mmengine/visualization/__init__.py index 9dcd772db4..8f59452c54 100644 --- a/mmengine/visualization/__init__.py +++ b/mmengine/visualization/__init__.py @@ -1,11 +1,12 @@ # Copyright (c) OpenMMLab. All rights reserved. -from .vis_backend import (BaseVisBackend, ClearMLVisBackend, LocalVisBackend, - MLflowVisBackend, NeptuneVisBackend, - TensorboardVisBackend, WandbVisBackend) +from .vis_backend import (AimVisBackend, BaseVisBackend, ClearMLVisBackend, + DVCLiveVisBackend, LocalVisBackend, MLflowVisBackend, + NeptuneVisBackend, TensorboardVisBackend, + WandbVisBackend) from .visualizer import Visualizer __all__ = [ 'Visualizer', 'BaseVisBackend', 'LocalVisBackend', 'WandbVisBackend', 'TensorboardVisBackend', 'MLflowVisBackend', 'ClearMLVisBackend', - 'NeptuneVisBackend' + 'NeptuneVisBackend', 'DVCLiveVisBackend', 'AimVisBackend' ] diff --git a/mmengine/visualization/vis_backend.py b/mmengine/visualization/vis_backend.py index 48c6a90761..37b54ba553 100644 --- a/mmengine/visualization/vis_backend.py +++ b/mmengine/visualization/vis_backend.py @@ -4,6 +4,7 @@ import logging import os import os.path as osp +import platform import warnings from abc import ABCMeta, abstractmethod from collections.abc import MutableMapping @@ -13,12 +14,12 @@ import numpy as np import torch -from mmengine.config import Config +from mmengine.config import Config, ConfigDict from mmengine.fileio import dump from mmengine.hooks.logger_hook import SUFFIX_TYPE from mmengine.logging import MMLogger, print_log from mmengine.registry import VISBACKENDS -from mmengine.utils import scandir +from mmengine.utils import digit_version, scandir from mmengine.utils.dl_utils import TORCH_VERSION @@ -1130,3 +1131,311 @@ def close(self) -> None: """close an opened object.""" if hasattr(self, '_neptune'): self._neptune.stop() + + +@VISBACKENDS.register_module() +class DVCLiveVisBackend(BaseVisBackend): + """DVCLive visualization backend class. + + Examples: + >>> from mmengine.visualization import DVCLiveVisBackend + >>> import numpy as np + >>> dvclive_vis_backend = DVCLiveVisBackend(save_dir='temp_dir') + >>> img=np.random.randint(0, 256, size=(10, 10, 3)) + >>> dvclive_vis_backend.add_image('img', img) + >>> dvclive_vis_backend.add_scalar('mAP', 0.6) + >>> dvclive_vis_backend.add_scalars({'loss': 0.1, 'acc': 0.8}) + >>> cfg = Config(dict(a=1, b=dict(b1=[0, 1]))) + >>> dvclive_vis_backend.add_config(cfg) + + Note: + `New in version 0.8.5.` + + Args: + save_dir (str, optional): The root directory to save the files + produced by the visualizer. + artifact_suffix (Tuple[str] or str, optional): The artifact suffix. + Defaults to ('.json', '.py', 'yaml'). + init_kwargs (dict, optional): DVCLive initialization parameters. + See `DVCLive `_ for details. + Defaults to None. + """ + + def __init__(self, + save_dir: str, + artifact_suffix: SUFFIX_TYPE = ('.json', '.py', 'yaml'), + init_kwargs: Optional[dict] = None): + super().__init__(save_dir) + self._artifact_suffix = artifact_suffix + self._init_kwargs = init_kwargs + + def _init_env(self): + """Setup env for dvclive.""" + if digit_version(platform.python_version()) < digit_version('3.8'): + raise RuntimeError('Please use Python 3.8 or higher version ' + 'to use DVCLiveVisBackend.') + + try: + import pygit2 + from dvclive import Live + except ImportError: + raise ImportError( + 'Please run "pip install dvclive" to install dvclive') + # if no git info, init dvc without git to avoid SCMError + try: + path = pygit2.discover_repository(os.fspath(os.curdir), True, '') + pygit2.Repository(path).default_signature + except KeyError: + os.system('dvc init -f --no-scm') + + if self._init_kwargs is None: + self._init_kwargs = {} + self._init_kwargs.setdefault('dir', self._save_dir) + self._init_kwargs.setdefault('save_dvc_exp', True) + self._init_kwargs.setdefault('cache_images', True) + + self._dvclive = Live(**self._init_kwargs) + + @property # type: ignore + @force_init_env + def experiment(self): + """Return dvclive object. + + The experiment attribute can get the dvclive backend, If you want to + write other data, such as writing a table, you can directly get the + dvclive backend through experiment. + """ + return self._dvclive + + @force_init_env + def add_config(self, config: Config, **kwargs) -> None: + """Record the config to dvclive. + + Args: + config (Config): The Config object + """ + assert isinstance(config, Config) + self.cfg = config + self._dvclive.log_params(self._to_dvc_paramlike(self.cfg)) + + @force_init_env + def add_image(self, + name: str, + image: np.ndarray, + step: int = 0, + **kwargs) -> None: + """Record the image to dvclive. + + Args: + name (str): The image identifier. + image (np.ndarray): The image to be saved. The format + should be RGB. + step (int): Useless parameter. Dvclive does not + need this parameter. Defaults to 0. + """ + assert image.dtype == np.uint8 + save_file_name = f'{name}.png' + + self._dvclive.log_image(save_file_name, image) + + @force_init_env + def add_scalar(self, + name: str, + value: Union[int, float, torch.Tensor, np.ndarray], + step: int = 0, + **kwargs) -> None: + """Record the scalar data to dvclive. + + Args: + name (str): The scalar identifier. + value (int, float, torch.Tensor, np.ndarray): Value to save. + step (int): Global step value to record. Defaults to 0. + """ + if isinstance(value, torch.Tensor): + value = value.numpy() + self._dvclive.step = step + self._dvclive.log_metric(name, value) + + @force_init_env + def add_scalars(self, + scalar_dict: dict, + step: int = 0, + file_path: Optional[str] = None, + **kwargs) -> None: + """Record the scalar's data to dvclive. + + Args: + scalar_dict (dict): Key-value pair storing the tag and + corresponding values. + step (int): Global step value to record. Defaults to 0. + file_path (str, optional): Useless parameter. Just for + interface unification. Defaults to None. + """ + for key, value in scalar_dict.items(): + self.add_scalar(key, value, step, **kwargs) + + def close(self) -> None: + """close an opened dvclive object.""" + if not hasattr(self, '_dvclive'): + return + + file_paths = dict() + for filename in scandir(self._save_dir, self._artifact_suffix, True): + file_path = osp.join(self._save_dir, filename) + relative_path = os.path.relpath(file_path, self._save_dir) + dir_path = os.path.dirname(relative_path) + file_paths[file_path] = dir_path + + for file_path, dir_path in file_paths.items(): + self._dvclive.log_artifact(file_path, dir_path) + + self._dvclive.end() + + def _to_dvc_paramlike(self, + value: Union[int, float, dict, list, tuple, Config, + ConfigDict, torch.Tensor, np.ndarray]): + """Convert the input value to a DVC `ParamLike` recursively. + + Or the `log_params` method of dvclive will raise an error. + """ + + if isinstance(value, (dict, Config, ConfigDict)): + return {k: self._to_dvc_paramlike(v) for k, v in value.items()} + elif isinstance(value, (tuple, list)): + return [self._to_dvc_paramlike(item) for item in value] + elif isinstance(value, (torch.Tensor, np.ndarray)): + return value.tolist() + elif isinstance(value, np.generic): + return value.item() + else: + return value + + +@VISBACKENDS.register_module() +class AimVisBackend(BaseVisBackend): + """Aim visualization backend class. + + Examples: + >>> from mmengine.visualization import AimVisBackend + >>> import numpy as np + >>> aim_vis_backend = AimVisBackend() + >>> img=np.random.randint(0, 256, size=(10, 10, 3)) + >>> aim_vis_backend.add_image('img', img) + >>> aim_vis_backend.add_scalar('mAP', 0.6) + >>> aim_vis_backend.add_scalars({'loss': 0.1, 'acc': 0.8}) + >>> cfg = Config(dict(a=1, b=dict(b1=[0, 1]))) + >>> aim_vis_backend.add_config(cfg) + + Note: + 1. `New in version 0.8.5.` + 2. Refer to + `Github issue `_ , + Aim is not unable to be install on Windows for now. + + Args: + save_dir (str, optional): The root directory to save the files + produced by the visualizer. + init_kwargs (dict, optional): Aim initialization parameters. See + `Aim `_ + for details. Defaults to None. + """ + + def __init__(self, + save_dir: Optional[str] = None, + init_kwargs: Optional[dict] = None): + super().__init__(save_dir) # type:ignore + self._init_kwargs = init_kwargs + + def _init_env(self): + """Setup env for Aim.""" + try: + from aim import Run + except ImportError: + raise ImportError('Please run "pip install aim" to install aim') + + from datetime import datetime + + if self._save_dir is not None: + path_list = os.path.normpath(self._save_dir).split(os.sep) + exp_name = f'{path_list[-2]}_{path_list[-1]}' + else: + exp_name = datetime.now().strftime('%Y%m%d_%H%M%S') + + if self._init_kwargs is None: + self._init_kwargs = {} + self._init_kwargs.setdefault('experiment', exp_name) + self._aim_run = Run(**self._init_kwargs) + + @property # type: ignore + @force_init_env + def experiment(self): + """Return Aim object.""" + return self._aim_run + + @force_init_env + def add_config(self, config, **kwargs) -> None: + """Record the config to Aim. + + Args: + config (Config): The Config object + """ + if isinstance(config, Config): + config = config.to_dict() + self._aim_run['hparams'] = config + + @force_init_env + def add_image(self, + name: str, + image: np.ndarray, + step: int = 0, + **kwargs) -> None: + """Record the image. + + Args: + name (str): The image identifier. + image (np.ndarray): The image to be saved. The format + should be RGB. Defaults to None. + step (int): Global step value to record. Defaults to 0. + """ + from aim import Image + self._aim_run.track(name=name, value=Image(image), step=step) + + @force_init_env + def add_scalar(self, + name: str, + value: Union[int, float, torch.Tensor, np.ndarray], + step: int = 0, + **kwargs) -> None: + """Record the scalar data to Aim. + + Args: + name (str): The scalar identifier. + value (int, float, torch.Tensor, np.ndarray): Value to save. + step (int): Global step value to record. Default to 0. + """ + self._aim_run.track(name=name, value=value, step=step) + + @force_init_env + def add_scalars(self, + scalar_dict: dict, + step: int = 0, + file_path: Optional[str] = None, + **kwargs) -> None: + """Record the scalar's data to wandb. + + Args: + scalar_dict (dict): Key-value pair storing the tag and + corresponding values. + step (int): Global step value to record. Default to 0. + file_path (str, optional): Useless parameter. Just for + interface unification. Defaults to None. + """ + for key, value in scalar_dict.items(): + self._aim_run.track(name=key, value=value, step=step) + + def close(self) -> None: + """Close the Aim.""" + if not hasattr(self, '_aim_run'): + return + + self._aim_run.close() diff --git a/requirements/tests.txt b/requirements/tests.txt index 4084323ed8..599163fd1a 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -1,9 +1,14 @@ +aim;sys_platform!='win32' +bitsandbytes clearml coverage dadaptation +dvclive lion-pytorch lmdb mlflow neptune parameterized +pydantic==1.10.9 pytest +transformers diff --git a/tests/test_optim/test_optimizer/test_optimizer.py b/tests/test_optim/test_optimizer/test_optimizer.py index 9f851bd327..0cf60fcb83 100644 --- a/tests/test_optim/test_optimizer/test_optimizer.py +++ b/tests/test_optim/test_optimizer/test_optimizer.py @@ -14,9 +14,11 @@ from mmengine.optim import (OPTIM_WRAPPER_CONSTRUCTORS, OPTIMIZERS, DefaultOptimWrapperConstructor, OptimWrapper, build_optim_wrapper) -from mmengine.optim.optimizer.builder import (DADAPTATION_OPTIMIZERS, +from mmengine.optim.optimizer.builder import (BITSANDBYTES_OPTIMIZERS, + DADAPTATION_OPTIMIZERS, LION_OPTIMIZERS, - TORCH_OPTIMIZERS) + TORCH_OPTIMIZERS, + TRANSFORMERS_OPTIMIZERS) from mmengine.registry import DefaultScope, Registry, build_from_cfg from mmengine.testing._internal import MultiProcessTestCase from mmengine.utils.dl_utils import TORCH_VERSION, mmcv_full_available @@ -44,6 +46,22 @@ def has_lion() -> bool: return False +def has_bitsandbytes() -> bool: + try: + import bitsandbytes # noqa: F401 + return True + except ImportError: + return False + + +def has_transformers() -> bool: + try: + import transformers # noqa: F401 + return True + except ImportError: + return False + + class ExampleModel(nn.Module): def __init__(self): @@ -235,6 +253,22 @@ def test_dadaptation_optimizers(self): def test_lion_optimizers(self): assert 'Lion' in LION_OPTIMIZERS + @unittest.skipIf(not has_bitsandbytes(), 'bitsandbytes is not installed') + def test_bitsandbytes_optimizers(self): + bitsandbytes_optimizers = [ + 'AdamW8bit', 'Adam8bit', 'Adagrad8bit', 'PagedAdam8bit', + 'PagedAdamW8bit', 'LAMB8bit', 'LARS8bit', 'RMSprop8bit', + 'Lion8bit', 'PagedLion8bit', 'SGD8bit' + ] + assert set(bitsandbytes_optimizers).issubset( + set(BITSANDBYTES_OPTIMIZERS)) + + @unittest.skipIf(not has_transformers(), 'transformers is not installed') + def test_transformers_optimizers(self): + transformers_optimizers = ['Adafactor'] + assert set(transformers_optimizers).issubset( + set(TRANSFORMERS_OPTIMIZERS)) + def test_build_optimizer(self): # test build function without ``constructor`` and ``paramwise_cfg`` optim_wrapper_cfg = dict( diff --git a/tests/test_runner/test_amp.py b/tests/test_runner/test_amp.py index 89794f3414..a80c7f35cb 100644 --- a/tests/test_runner/test_amp.py +++ b/tests/test_runner/test_amp.py @@ -5,7 +5,7 @@ import torch.nn as nn import mmengine -from mmengine.device import get_device, is_mlu_available +from mmengine.device import get_device, is_mlu_available, is_npu_available from mmengine.runner import autocast from mmengine.utils import digit_version from mmengine.utils.dl_utils import TORCH_VERSION @@ -14,7 +14,22 @@ class TestAmp(unittest.TestCase): def test_autocast(self): - if is_mlu_available(): + if is_npu_available(): + device = 'npu' + with autocast(device_type=device): + # torch.autocast support npu mode. + layer = nn.Conv2d(1, 1, 1).to(device) + res = layer(torch.randn(1, 1, 1, 1).to(device)) + self.assertIn(res.dtype, (torch.bfloat16, torch.float16)) + with autocast(enabled=False, device_type=device): + res = layer(torch.randn(1, 1, 1, 1).to(device)) + self.assertEqual(res.dtype, torch.float32) + # Test with fp32_enabled + with autocast(enabled=False, device_type=device): + layer = nn.Conv2d(1, 1, 1).to(device) + res = layer(torch.randn(1, 1, 1, 1).to(device)) + self.assertEqual(res.dtype, torch.float32) + elif is_mlu_available(): device = 'mlu' with autocast(device_type=device): # torch.autocast support mlu mode. diff --git a/tests/test_utils/test_misc.py b/tests/test_utils/test_misc.py index 700cde8759..580a3f2d73 100644 --- a/tests/test_utils/test_misc.py +++ b/tests/test_utils/test_misc.py @@ -336,3 +336,5 @@ def test_locate(): assert get_object_from_string('mmengine.model') is model_module assert get_object_from_string( 'mmengine.model.BaseModel') is model_module.BaseModel + assert get_object_from_string('mmengine.model.BaseModel.forward') is \ + model_module.BaseModel.forward diff --git a/tests/test_visualizer/test_vis_backend.py b/tests/test_visualizer/test_vis_backend.py index b53a67dc53..2f9f665a1e 100644 --- a/tests/test_visualizer/test_vis_backend.py +++ b/tests/test_visualizer/test_vis_backend.py @@ -1,5 +1,6 @@ # Copyright (c) OpenMMLab. All rights reserved. import os +import platform import shutil import sys import warnings @@ -12,7 +13,9 @@ from mmengine import Config from mmengine.fileio import load from mmengine.registry import VISBACKENDS -from mmengine.visualization import (ClearMLVisBackend, LocalVisBackend, +from mmengine.utils import digit_version +from mmengine.visualization import (AimVisBackend, ClearMLVisBackend, + DVCLiveVisBackend, LocalVisBackend, MLflowVisBackend, NeptuneVisBackend, TensorboardVisBackend, WandbVisBackend) @@ -391,3 +394,95 @@ def test_close(self): neptune_vis_backend = NeptuneVisBackend() neptune_vis_backend._init_env() neptune_vis_backend.close() + + +@pytest.mark.skipif( + digit_version(platform.python_version()) < digit_version('3.8'), + reason='DVCLiveVisBackend does not support python version < 3.8') +class TestDVCLiveVisBackend: + + def test_init(self): + DVCLiveVisBackend('temp_dir') + VISBACKENDS.build(dict(type='DVCLiveVisBackend', save_dir='temp_dir')) + + def test_experiment(self): + dvclive_vis_backend = DVCLiveVisBackend('temp_dir') + assert dvclive_vis_backend.experiment == dvclive_vis_backend._dvclive + shutil.rmtree('temp_dir') + + def test_add_config(self): + cfg = Config(dict(a=1, b=dict(b1=[0, 1]))) + dvclive_vis_backend = DVCLiveVisBackend('temp_dir') + dvclive_vis_backend.add_config(cfg) + shutil.rmtree('temp_dir') + + def test_add_image(self): + img = np.random.randint(0, 256, size=(10, 10, 3)).astype(np.uint8) + dvclive_vis_backend = DVCLiveVisBackend('temp_dir') + dvclive_vis_backend.add_image('img', img) + shutil.rmtree('temp_dir') + + def test_add_scalar(self): + dvclive_vis_backend = DVCLiveVisBackend('temp_dir') + dvclive_vis_backend.add_scalar('mAP', 0.9) + # test append mode + dvclive_vis_backend.add_scalar('mAP', 0.9) + dvclive_vis_backend.add_scalar('mAP', 0.95) + shutil.rmtree('temp_dir') + + def test_add_scalars(self): + dvclive_vis_backend = DVCLiveVisBackend('temp_dir') + input_dict = {'map': 0.7, 'acc': 0.9} + dvclive_vis_backend.add_scalars(input_dict) + # test append mode + dvclive_vis_backend.add_scalars({'map': 0.8, 'acc': 0.8}) + shutil.rmtree('temp_dir') + + def test_close(self): + cfg = Config(dict(work_dir='temp_dir')) + dvclive_vis_backend = DVCLiveVisBackend('temp_dir') + dvclive_vis_backend._init_env() + dvclive_vis_backend.add_config(cfg) + dvclive_vis_backend.close() + shutil.rmtree('temp_dir') + + +@pytest.mark.skipif( + platform.system() == 'Windows', + reason='Aim does not support Windows for now.') +class TestAimVisBackend: + + def test_init(self): + AimVisBackend() + VISBACKENDS.build(dict(type='AimVisBackend')) + + def test_experiment(self): + aim_vis_backend = AimVisBackend() + assert aim_vis_backend.experiment == aim_vis_backend._aim_run + + def test_add_config(self): + cfg = Config(dict(a=1, b=dict(b1=[0, 1]))) + aim_vis_backend = AimVisBackend() + aim_vis_backend.add_config(cfg) + + def test_add_image(self): + image = np.random.randint(0, 256, size=(10, 10, 3)).astype(np.uint8) + aim_vis_backend = AimVisBackend() + aim_vis_backend.add_image('img', image) + aim_vis_backend.add_image('img', image, step=1) + + def test_add_scalar(self): + aim_vis_backend = AimVisBackend() + aim_vis_backend.add_scalar('map', 0.9) + aim_vis_backend.add_scalar('map', 0.9, step=1) + aim_vis_backend.add_scalar('map', 0.95, step=2) + + def test_add_scalars(self): + aim_vis_backend = AimVisBackend() + input_dict = {'map': 0.7, 'acc': 0.9} + aim_vis_backend.add_scalars(input_dict) + + def test_close(self): + aim_vis_backend = AimVisBackend() + aim_vis_backend._init_env() + aim_vis_backend.close()