From 01abe275ef2d6552b6883dde20e3e9f68e7019e4 Mon Sep 17 00:00:00 2001 From: erenzhou <592267829@qq.com> Date: Sat, 22 Oct 2022 17:22:40 +0800 Subject: [PATCH 1/3] update --- configs/convnext/README.md | 2 +- ...nvnext-tiny_fpn_kld-stable_adamw-1x_dota.py | 2 +- .../kld/r3det-oc_r50_fpn_kld-stable_1x_dota.py | 18 ++++++++++-------- configs/kld/r3det-oc_r50_fpn_kld_1x_dota.py | 18 ++++++++++-------- .../kld/r3det-tiny-oc_r50_fpn_kld_1x_dota.py | 18 ++++++++++-------- ...nanet-hbox-oc_r50_fpn_kld-stable_1x_dota.py | 2 +- ...ox-le90_r50_fpn_kld-stable_adamw-1x_dota.py | 2 +- .../oriented-rcnn-le90_r50_fpn_amp-1x_dota.py | 2 +- ...oriented-rcnn-le90_swin-tiny_fpn_1x_dota.py | 2 +- configs/redet/README.md | 1 + .../redet/redet-le90_re50_refpn_amp-1x_dota.py | 2 +- .../roi-trans-le90_r50_fpn_amp-1x_dota.py | 2 +- .../roi-trans-le90_swin-tiny_fpn_1x_dota.py | 2 +- configs/rotated_reppoints/metafile.yml | 4 ++-- ...-retinanet-rbox-le90_r50_fpn_amp-1x_dota.py | 2 +- mmrotate/models/backbones/re_resnet.py | 2 +- .../rotate_single_level_roi_extractor.py | 1 + mmrotate/structures/bbox/rotated_boxes.py | 4 ++-- 18 files changed, 47 insertions(+), 39 deletions(-) diff --git a/configs/convnext/README.md b/configs/convnext/README.md index 931e002da..b407f8351 100644 --- a/configs/convnext/README.md +++ b/configs/convnext/README.md @@ -25,7 +25,7 @@ DOTA1.0 - ConvNeXt backbone needs to install [MMClassification](https://github.com/open-mmlab/mmclassification) first, which has abundant backbones for downstream tasks. ```shell -pip install mmcls>=0.22.0 +pip install "mmcls>=1.0.0rc0" ``` - The performance may be unstable according to mmdetection's experience. diff --git a/configs/convnext/rotated-retinanet-rbox-le90_convnext-tiny_fpn_kld-stable_adamw-1x_dota.py b/configs/convnext/rotated-retinanet-rbox-le90_convnext-tiny_fpn_kld-stable_adamw-1x_dota.py index b24a070ed..b92af8aaa 100644 --- a/configs/convnext/rotated-retinanet-rbox-le90_convnext-tiny_fpn_kld-stable_adamw-1x_dota.py +++ b/configs/convnext/rotated-retinanet-rbox-le90_convnext-tiny_fpn_kld-stable_adamw-1x_dota.py @@ -2,7 +2,7 @@ '../kld/rotated-retinanet-rbox-le90_r50_fpn_kld-stable_adamw-1x_dota.py' ] -# please install mmcls>=0.22.0 +# please install mmcls>=1.0.0rc0 # import mmcls.models to trigger register_module in mmcls custom_imports = dict(imports=['mmcls.models'], allow_failed_imports=False) checkpoint_file = 'https://download.openmmlab.com/mmclassification/v0/convnext/downstream/convnext-tiny_3rdparty_32xb128-noema_in1k_20220301-795e9634.pth' # noqa diff --git a/configs/kld/r3det-oc_r50_fpn_kld-stable_1x_dota.py b/configs/kld/r3det-oc_r50_fpn_kld-stable_1x_dota.py index 339ca69d8..bb2b503eb 100644 --- a/configs/kld/r3det-oc_r50_fpn_kld-stable_1x_dota.py +++ b/configs/kld/r3det-oc_r50_fpn_kld-stable_1x_dota.py @@ -2,7 +2,7 @@ angle_version = 'oc' model = dict( - bbox_head=dict( + bbox_head_init=dict( reg_decoded_bbox=True, loss_bbox=dict( _delete_=True, @@ -12,26 +12,28 @@ tau=1.0, sqrt=False, loss_weight=5.0)), - refine_heads=[ + bbox_head_refine=[ dict( - type='RotatedRetinaRefineHead', + type='R3RefineHead', num_classes=15, in_channels=256, stacked_convs=4, feat_channels=256, - assign_by_circumhbbox=None, + frm_cfg=dict( + type='FRM', feat_channels=256, strides=[8, 16, 32, 64, 128]), anchor_generator=dict( - type='PseudoAnchorGenerator', strides=[8, 16, 32, 64, 128]), + type='PseudoRotatedAnchorGenerator', + strides=[8, 16, 32, 64, 128]), bbox_coder=dict( - type='DeltaXYWHAOBBoxCoder', - angle_range=angle_version, + type='DeltaXYWHTRBBoxCoder', + angle_version=angle_version, norm_factor=None, edge_swap=False, proj_xy=False, target_means=(0.0, 0.0, 0.0, 0.0, 0.0), target_stds=(1.0, 1.0, 1.0, 1.0, 1.0)), loss_cls=dict( - type='FocalLoss', + type='mmdet.FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, diff --git a/configs/kld/r3det-oc_r50_fpn_kld_1x_dota.py b/configs/kld/r3det-oc_r50_fpn_kld_1x_dota.py index a677cedee..91e17fa6a 100644 --- a/configs/kld/r3det-oc_r50_fpn_kld_1x_dota.py +++ b/configs/kld/r3det-oc_r50_fpn_kld_1x_dota.py @@ -2,7 +2,7 @@ angle_version = 'oc' model = dict( - bbox_head=dict( + bbox_head_init=dict( reg_decoded_bbox=True, loss_bbox=dict( _delete_=True, @@ -11,26 +11,28 @@ fun='log1p', tau=1.0, loss_weight=1.0)), - refine_heads=[ + bbox_head_refine=[ dict( - type='RotatedRetinaRefineHead', + type='R3RefineHead', num_classes=15, in_channels=256, stacked_convs=4, feat_channels=256, - assign_by_circumhbbox=None, + frm_cfg=dict( + type='FRM', feat_channels=256, strides=[8, 16, 32, 64, 128]), anchor_generator=dict( - type='PseudoAnchorGenerator', strides=[8, 16, 32, 64, 128]), + type='PseudoRotatedAnchorGenerator', + strides=[8, 16, 32, 64, 128]), bbox_coder=dict( - type='DeltaXYWHAOBBoxCoder', - angle_range=angle_version, + type='DeltaXYWHTRBBoxCoder', + angle_version=angle_version, norm_factor=None, edge_swap=False, proj_xy=False, target_means=(0.0, 0.0, 0.0, 0.0, 0.0), target_stds=(1.0, 1.0, 1.0, 1.0, 1.0)), loss_cls=dict( - type='FocalLoss', + type='mmdet.FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, diff --git a/configs/kld/r3det-tiny-oc_r50_fpn_kld_1x_dota.py b/configs/kld/r3det-tiny-oc_r50_fpn_kld_1x_dota.py index 68cb75e0a..f456300e8 100644 --- a/configs/kld/r3det-tiny-oc_r50_fpn_kld_1x_dota.py +++ b/configs/kld/r3det-tiny-oc_r50_fpn_kld_1x_dota.py @@ -2,7 +2,7 @@ angle_version = 'oc' model = dict( - bbox_head=dict( + bbox_head_init=dict( reg_decoded_bbox=True, loss_bbox=dict( _delete_=True, @@ -11,26 +11,28 @@ fun='log1p', tau=1.0, loss_weight=1.0)), - refine_heads=[ + bbox_head_refine=[ dict( - type='RotatedRetinaRefineHead', + type='R3RefineHead', num_classes=15, in_channels=256, stacked_convs=2, feat_channels=256, - assign_by_circumhbbox=None, + frm_cfg=dict( + type='FRM', feat_channels=256, strides=[8, 16, 32, 64, 128]), anchor_generator=dict( - type='PseudoAnchorGenerator', strides=[8, 16, 32, 64, 128]), + type='PseudoRotatedAnchorGenerator', + strides=[8, 16, 32, 64, 128]), bbox_coder=dict( - type='DeltaXYWHAOBBoxCoder', - angle_range=angle_version, + type='DeltaXYWHTRBBoxCoder', + angle_version=angle_version, norm_factor=None, edge_swap=False, proj_xy=False, target_means=(0.0, 0.0, 0.0, 0.0, 0.0), target_stds=(1.0, 1.0, 1.0, 1.0, 1.0)), loss_cls=dict( - type='FocalLoss', + type='mmdet.FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, diff --git a/configs/kld/rotated-retinanet-hbox-oc_r50_fpn_kld-stable_1x_dota.py b/configs/kld/rotated-retinanet-hbox-oc_r50_fpn_kld-stable_1x_dota.py index 55229632c..a0b55e93f 100644 --- a/configs/kld/rotated-retinanet-hbox-oc_r50_fpn_kld-stable_1x_dota.py +++ b/configs/kld/rotated-retinanet-hbox-oc_r50_fpn_kld-stable_1x_dota.py @@ -1,4 +1,4 @@ -_base_ = '../rotated_retinanet/rotated_retinanet-hbox-oc_r50_fpn_1x_dota.py' +_base_ = '../rotated_retinanet/rotated-retinanet-hbox-oc_r50_fpn_1x_dota.py' model = dict( bbox_head=dict( diff --git a/configs/kld/rotated-retinanet-rbox-le90_r50_fpn_kld-stable_adamw-1x_dota.py b/configs/kld/rotated-retinanet-rbox-le90_r50_fpn_kld-stable_adamw-1x_dota.py index 9409b33f1..d5a6c4a09 100644 --- a/configs/kld/rotated-retinanet-rbox-le90_r50_fpn_kld-stable_adamw-1x_dota.py +++ b/configs/kld/rotated-retinanet-rbox-le90_r50_fpn_kld-stable_adamw-1x_dota.py @@ -1,4 +1,4 @@ -_base_ = './rotated_retinanet-rbox-le90_r50_fpn_kld-stable_1x_dota.py' +_base_ = './rotated-retinanet-rbox-le90_r50_fpn_kld-stable_1x_dota.py' optim_wrapper = dict( optimizer=dict( diff --git a/configs/oriented_rcnn/oriented-rcnn-le90_r50_fpn_amp-1x_dota.py b/configs/oriented_rcnn/oriented-rcnn-le90_r50_fpn_amp-1x_dota.py index 0c06452d3..4888d77ea 100644 --- a/configs/oriented_rcnn/oriented-rcnn-le90_r50_fpn_amp-1x_dota.py +++ b/configs/oriented_rcnn/oriented-rcnn-le90_r50_fpn_amp-1x_dota.py @@ -1,3 +1,3 @@ _base_ = './oriented-rcnn-le90_r50_fpn_1x_dota.py' -optim_wrapper = dict(type='AmpOptimWrapper', loss_scale='dynamic') +optim_wrapper = dict(type='AmpOptimWrapper') diff --git a/configs/oriented_rcnn/oriented-rcnn-le90_swin-tiny_fpn_1x_dota.py b/configs/oriented_rcnn/oriented-rcnn-le90_swin-tiny_fpn_1x_dota.py index 4411ba1e6..daa46490d 100644 --- a/configs/oriented_rcnn/oriented-rcnn-le90_swin-tiny_fpn_1x_dota.py +++ b/configs/oriented_rcnn/oriented-rcnn-le90_swin-tiny_fpn_1x_dota.py @@ -5,7 +5,7 @@ model = dict( backbone=dict( _delete_=True, - type='SwinTransformer', + type='mmdet.SwinTransformer', embed_dims=96, depths=depths, num_heads=[3, 6, 12, 24], diff --git a/configs/redet/README.md b/configs/redet/README.md index 49413edac..3fe8300df 100644 --- a/configs/redet/README.md +++ b/configs/redet/README.md @@ -34,6 +34,7 @@ Notes: - `MS` means multiple scale image split. - `RR` means random rotation. - Please download pretrained weight of ReResNet from [ReDet](https://github.com/csuhan/ReDet), and put it on `work_dirs/pretrain`. BTW, it is normal for `missing keys in source state_dict: xxx.filter ` to appear in the log. Don't worry! +- Please use distributed training, there are some bug when use `train.py`. ## Citation diff --git a/configs/redet/redet-le90_re50_refpn_amp-1x_dota.py b/configs/redet/redet-le90_re50_refpn_amp-1x_dota.py index 943427c1d..1e2f483f4 100644 --- a/configs/redet/redet-le90_re50_refpn_amp-1x_dota.py +++ b/configs/redet/redet-le90_re50_refpn_amp-1x_dota.py @@ -1,3 +1,3 @@ _base_ = './redet-le90_re50_refpn_1x_dota.py' -optim_wrapper = dict(type='AmpOptimWrapper', loss_scale='dynamic') +optim_wrapper = dict(type='AmpOptimWrapper') diff --git a/configs/roi_trans/roi-trans-le90_r50_fpn_amp-1x_dota.py b/configs/roi_trans/roi-trans-le90_r50_fpn_amp-1x_dota.py index 8b5cb38d6..d1fc6f5e5 100644 --- a/configs/roi_trans/roi-trans-le90_r50_fpn_amp-1x_dota.py +++ b/configs/roi_trans/roi-trans-le90_r50_fpn_amp-1x_dota.py @@ -1,3 +1,3 @@ _base_ = './roi-trans-le90_r50_fpn_1x_dota.py' -optim_wrapper = dict(type='AmpOptimWrapper', loss_scale='dynamic') +optim_wrapper = dict(type='AmpOptimWrapper') diff --git a/configs/roi_trans/roi-trans-le90_swin-tiny_fpn_1x_dota.py b/configs/roi_trans/roi-trans-le90_swin-tiny_fpn_1x_dota.py index d3f39c74e..7ea31993e 100644 --- a/configs/roi_trans/roi-trans-le90_swin-tiny_fpn_1x_dota.py +++ b/configs/roi_trans/roi-trans-le90_swin-tiny_fpn_1x_dota.py @@ -6,7 +6,7 @@ model = dict( backbone=dict( _delete_=True, - type='SwinTransformer', + type='mmet.SwinTransformer', embed_dims=96, depths=depths, num_heads=[3, 6, 12, 24], diff --git a/configs/rotated_reppoints/metafile.yml b/configs/rotated_reppoints/metafile.yml index d1433b75e..1b2a520aa 100644 --- a/configs/rotated_reppoints/metafile.yml +++ b/configs/rotated_reppoints/metafile.yml @@ -14,9 +14,9 @@ Collections: README: configs/cfa/README.md Models: - - Name: rotated_reppoints_r50_fpn_1x_dota_oc + - Name: rotated-reppoints-qbox_r50_fpn_1x_dota In Collection: rotated_reppoints - Config: configs/rotated_reppoints/rotated_reppoints_r50_fpn_1x_dota_oc.py + Config: configs/rotated_reppoints/rotated-reppoints-qbox_r50_fpn_1x_dota.py Metadata: Training Data: DOTAv1.0 Results: diff --git a/configs/rotated_retinanet/rotated-retinanet-rbox-le90_r50_fpn_amp-1x_dota.py b/configs/rotated_retinanet/rotated-retinanet-rbox-le90_r50_fpn_amp-1x_dota.py index 039fc9279..a24d3a03f 100644 --- a/configs/rotated_retinanet/rotated-retinanet-rbox-le90_r50_fpn_amp-1x_dota.py +++ b/configs/rotated_retinanet/rotated-retinanet-rbox-le90_r50_fpn_amp-1x_dota.py @@ -1,3 +1,3 @@ _base_ = ['./rotated-retinanet-rbox-le90_r50_fpn_1x_dota.py'] -optim_wrapper = dict(type='AmpOptimWrapper', loss_scale='dynamic') +optim_wrapper = dict(type='AmpOptimWrapper') diff --git a/mmrotate/models/backbones/re_resnet.py b/mmrotate/models/backbones/re_resnet.py index 105446bf0..083253fae 100644 --- a/mmrotate/models/backbones/re_resnet.py +++ b/mmrotate/models/backbones/re_resnet.py @@ -490,7 +490,7 @@ def __init__(self, zero_init_residual: bool = True, init_cfg: OptMultiConfig = None) -> None: super().__init__(init_cfg=init_cfg) - self.in_type = build_enn_trivial_feature(3) + self.in_type = build_enn_trivial_feature(in_channels) if depth not in self.arch_settings: raise KeyError(f'invalid depth {depth} for resnet') diff --git a/mmrotate/models/roi_heads/roi_extractors/rotate_single_level_roi_extractor.py b/mmrotate/models/roi_heads/roi_extractors/rotate_single_level_roi_extractor.py index d07ca8151..cc06dc98b 100644 --- a/mmrotate/models/roi_heads/roi_extractors/rotate_single_level_roi_extractor.py +++ b/mmrotate/models/roi_heads/roi_extractors/rotate_single_level_roi_extractor.py @@ -97,6 +97,7 @@ def forward(self, feats, rois, roi_scale_factor=None): Returns: torch.Tensor: Scaled RoI features. """ + rois = rois.type_as(feats[0]) from mmrotate import digit_version, mmcv_version if isinstance(self.roi_layers[0], ops.RiRoIAlignRotated ) or mmcv_version == digit_version('1.4.5'): diff --git a/mmrotate/structures/bbox/rotated_boxes.py b/mmrotate/structures/bbox/rotated_boxes.py index 13e2915f2..8280684f1 100644 --- a/mmrotate/structures/bbox/rotated_boxes.py +++ b/mmrotate/structures/bbox/rotated_boxes.py @@ -413,10 +413,10 @@ def overlaps(boxes1: BaseBoxes, Returns: Tensor: shape (m, n) if ``is_aligned`` is False else shape (m,) """ - from ..iou_calculators import rbox_overlaps + from mmrotate.structures.bbox import rbbox_overlaps boxes1 = boxes1.convert_to('rbox') boxes2 = boxes2.convert_to('rbox') - return rbox_overlaps( + return rbbox_overlaps( boxes1.tensor, boxes2.tensor, mode=mode, From cde6992c0dd427a13c84cf547db6f807931dac76 Mon Sep 17 00:00:00 2001 From: erenzhou <592267829@qq.com> Date: Sun, 23 Oct 2022 08:47:09 +0800 Subject: [PATCH 2/3] init --- .dev_scripts/batch_train_list.txt | 41 +++++ .dev_scripts/check_links.py | 157 ++++++++++++++++++ .../convert_train_benchmark_script.py | 68 ++++++++ .dev_scripts/train_benchmark.sh | 82 +++++++++ 4 files changed, 348 insertions(+) create mode 100644 .dev_scripts/batch_train_list.txt create mode 100644 .dev_scripts/check_links.py create mode 100644 .dev_scripts/convert_train_benchmark_script.py create mode 100644 .dev_scripts/train_benchmark.sh diff --git a/.dev_scripts/batch_train_list.txt b/.dev_scripts/batch_train_list.txt new file mode 100644 index 000000000..7cad6d644 --- /dev/null +++ b/.dev_scripts/batch_train_list.txt @@ -0,0 +1,41 @@ +configs/cfa/cfa-qbox_r50_fpn_1x_dota.py +configs/cfa/cfa-qbox_r50_fpn_40e_dota.py +configs/convnext/rotated-retinanet-rbox-le90_convnext-tiny_fpn_kld-stable_adamw-1x_dota.py +configs/csl/rotated-retinanet-rbox-le90_r50_fpn_csl-gaussian_amp-1x_dota.py +configs/gliding_vertex/gliding-vertex-rbox_r50_fpn_1x_dota.py +configs/gwd/rotated-retinanet-hbox-oc_r50_fpn_gwd_1x_dota.py +configs/kfiou/rotated-retinanet-hbox-le90_r50_fpn_kfiou_1x_dota.py +configs/kfiou/rotated-retinanet-hbox-oc_r50_fpn_kfiou_1x_dota.py +configs/kfiou/rotated-retinanet-hbox-le135_r50_fpn_kfiou_1x_dota.py +configs/kfiou/r3det-oc_r50_fpn_kfiou-ln_1x_dota.py +configs/kld/rotated-retinanet-hbox-oc_r50_fpn_kld_1x_dota.py +configs/kld/rotated-retinanet-hbox-oc_r50_fpn_kld-stable_1x_dota.py +configs/kld/rotated-retinanet-rbox-le90_r50_fpn_kld-stable_1x_dota.py +configs/kld/r3det-oc_r50_fpn_kld_1x_dota.py +configs/kld/r3det-oc_r50_fpn_kld-stable_1x_dota.py +configs/kld/r3det-tiny-oc_r50_fpn_kld_1x_dota.py +configs/kld/rotated-retinanet-rbox-le90_r50_fpn_kld-stable_adamw-1x_dota.py +configs/oriented_rcnn/oriented-rcnn-le90_r50_fpn_amp-1x_dota.py +configs/oriented_rcnn/oriented-rcnn-le90_r50_fpn_1x_dota.py +configs/oriented_reppoints/oriented-reppoints-qbox_r50_fpn_1x_dota.py +configs/r3det/r3det-oc_r50_fpn_1x_dota.py +configs/r3det/r3det-tiny-oc_r50_fpn_1x_dota.py +configs/redet/redet-le90_re50_refpn_amp-1x_dota.py +configs/redet/redet-le90_re50_refpn_1x_dota.py +configs/roi_trans/roi-trans-le90_r50_fpn_amp-1x_dota.py +configs/roi_trans/roi-trans-le90_r50_fpn_1x_dota.py +configs/roi_trans/roi-trans-le90_swin-tiny_fpn_1x_dota.py +configs/rotated_atss/rotated-atss-le90_r50_fpn_1x_dota.py +configs/rotated_faster_rcnn/rotated-faster-rcnn-le90_r50_fpn_1x_dota.py +configs/rotated_fcos/rotated-fcos-hbox-le90_r50_fpn_1x_dota.py +configs/rotated_fcos/rotated-fcos-le90_r50_fpn_1x_dota.py +configs/rotated_fcos/rotated-fcos-hbox-le90_r50_fpn_csl-gaussian_1x_dota.py +configs/rotated_fcos/rotated-fcos-le90_r50_fpn_kld_1x_dota.py +configs/rotated_reppoints/rotated_reppoints_r50_fpn_1x_dota_oc.py +configs/rotated_retinanet/rotated-retinanet-hbox-oc_r50_fpn_1x_dota.py +configs/rotated_retinanet/rotated-retinanet-rbox-le90_r50_fpn_1x_dota.py +configs/rotated_retinanet/rotated-retinanet-rbox-le90_r50_fpn_amp-1x_dota.py +configs/rotated_retinanet/rotated-retinanet-rbox-le135_r50_fpn_1x_dota.py +configs/s2anet/s2anet-le135_r50_fpn_1x_dota.py +configs/s2anet/s2anet-le135_r50_fpn_amp-1x_dota.py +configs/sasm_reppoints/sasm-reppoints-qbox_r50_fpn_1x_dota.py \ No newline at end of file diff --git a/.dev_scripts/check_links.py b/.dev_scripts/check_links.py new file mode 100644 index 000000000..ccf4fad5e --- /dev/null +++ b/.dev_scripts/check_links.py @@ -0,0 +1,157 @@ +# Modified from: +# https://github.com/allenai/allennlp/blob/main/scripts/check_links.py + +import argparse +import logging +import os +import pathlib +import re +import sys +from multiprocessing.dummy import Pool +from typing import NamedTuple, Optional, Tuple + +import requests +from mmengine.logging import MMLogger + + +def parse_args(): + parser = argparse.ArgumentParser( + description='Goes through all the inline-links ' + 'in markdown files and reports the breakages') + parser.add_argument( + '--num-threads', + type=int, + default=100, + help='Number of processes to confirm the link') + parser.add_argument('--https-proxy', type=str, help='https proxy') + parser.add_argument( + '--out', + type=str, + default='link_reports.txt', + help='output path of reports') + args = parser.parse_args() + return args + + +OK_STATUS_CODES = ( + 200, + 401, # the resource exists but may require some sort of login. + 403, # ^ same + 405, # HEAD method not allowed. + # the resource exists, but our default 'Accept-' header may not + # match what the server can provide. + 406, +) + + +class MatchTuple(NamedTuple): + source: str + name: str + link: str + + +def check_link( + match_tuple: MatchTuple, + http_session: requests.Session, + logger: logging = None) -> Tuple[MatchTuple, bool, Optional[str]]: + reason: Optional[str] = None + if match_tuple.link.startswith('http'): + result_ok, reason = check_url(match_tuple, http_session) + else: + result_ok = check_path(match_tuple) + if logger is None: + print(f" {'✓' if result_ok else '✗'} {match_tuple.link}") + else: + logger.info(f" {'✓' if result_ok else '✗'} {match_tuple.link}") + return match_tuple, result_ok, reason + + +def check_url(match_tuple: MatchTuple, + http_session: requests.Session) -> Tuple[bool, str]: + """Check if a URL is reachable.""" + try: + result = http_session.head( + match_tuple.link, timeout=5, allow_redirects=True) + return ( + result.ok or result.status_code in OK_STATUS_CODES, + f'status code = {result.status_code}', + ) + except (requests.ConnectionError, requests.Timeout): + return False, 'connection error' + + +def check_path(match_tuple: MatchTuple) -> bool: + """Check if a file in this repository exists.""" + relative_path = match_tuple.link.split('#')[0] + full_path = os.path.join( + os.path.dirname(str(match_tuple.source)), relative_path) + return os.path.exists(full_path) + + +def main(): + args = parse_args() + + # setup logger + logger = MMLogger.get_instance(name='mmdet', log_file=args.out) + + # setup https_proxy + if args.https_proxy: + os.environ['https_proxy'] = args.https_proxy + + # setup http_session + http_session = requests.Session() + for resource_prefix in ('http://', 'https://'): + http_session.mount( + resource_prefix, + requests.adapters.HTTPAdapter( + max_retries=5, + pool_connections=20, + pool_maxsize=args.num_threads), + ) + + logger.info('Finding all markdown files in the current directory...') + + project_root = (pathlib.Path(__file__).parent / '..').resolve() + markdown_files = project_root.glob('**/*.md') + + all_matches = set() + url_regex = re.compile(r'\[([^!][^\]]+)\]\(([^)(]+)\)') + for markdown_file in markdown_files: + with open(markdown_file) as handle: + for line in handle.readlines(): + matches = url_regex.findall(line) + for name, link in matches: + if 'localhost' not in link: + all_matches.add( + MatchTuple( + source=str(markdown_file), + name=name, + link=link)) + + logger.info(f' {len(all_matches)} markdown files found') + logger.info('Checking to make sure we can retrieve each link...') + + with Pool(processes=args.num_threads) as pool: + results = pool.starmap(check_link, [(match, http_session, logger) + for match in list(all_matches)]) + + # collect unreachable results + unreachable_results = [(match_tuple, reason) + for match_tuple, success, reason in results + if not success] + + if unreachable_results: + logger.info('================================================') + logger.info(f'Unreachable links ({len(unreachable_results)}):') + for match_tuple, reason in unreachable_results: + logger.info(' > Source: ' + match_tuple.source) + logger.info(' Name: ' + match_tuple.name) + logger.info(' Link: ' + match_tuple.link) + if reason is not None: + logger.info(' Reason: ' + reason) + sys.exit(1) + logger.info('No Unreachable link found.') + + +if __name__ == '__main__': + main() diff --git a/.dev_scripts/convert_train_benchmark_script.py b/.dev_scripts/convert_train_benchmark_script.py new file mode 100644 index 000000000..57514844e --- /dev/null +++ b/.dev_scripts/convert_train_benchmark_script.py @@ -0,0 +1,68 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import argparse +import os +import os.path as osp + + +def parse_args(): + parser = argparse.ArgumentParser( + description='Convert benchmark model json to script') + parser.add_argument( + 'txt_path', type=str, help='txt path output by benchmark_filter') + parser.add_argument( + '--run', action='store_true', help='run script directly') + parser.add_argument( + '--out', type=str, help='path to save model benchmark script') + + args = parser.parse_args() + return args + + +def main(): + args = parse_args() + if args.out: + out_suffix = args.out.split('.')[-1] + assert args.out.endswith('.sh'), \ + f'Expected out file path suffix is .sh, but get .{out_suffix}' + assert args.out or args.run, \ + ('Please specify at least one operation (save/run/ the ' + 'script) with the argument "--out" or "--run"') + + root_name = './tools' + train_script_name = osp.join(root_name, 'train.py') + + commands = [] + + with open(args.txt_path, 'r') as f: + model_cfgs = f.readlines() + for i, cfg in enumerate(model_cfgs): + cfg = cfg.strip() + if len(cfg) == 0: + continue + # print cfg name + echo_info = f'echo \'{cfg}\' &' + commands.append(echo_info) + commands.append('\n') + + command_info = f'python {train_script_name} ' + command_info += f'{cfg} ' + + command_info += '--cfg-options default_hooks.checkpoint.' \ + 'max_keep_ckpts=1 ' + command_info += '&' + + commands.append(command_info) + + if i < len(model_cfgs): + commands.append('\n') + + command_str = ''.join(commands) + if args.out: + with open(args.out, 'w') as f: + f.write(command_str) + if args.run: + os.system(command_str) + + +if __name__ == '__main__': + main() diff --git a/.dev_scripts/train_benchmark.sh b/.dev_scripts/train_benchmark.sh new file mode 100644 index 000000000..9bafb12aa --- /dev/null +++ b/.dev_scripts/train_benchmark.sh @@ -0,0 +1,82 @@ +echo 'configs/cfa/cfa-qbox_r50_fpn_1x_dota.py' & +python ./tools/train.py configs/cfa/cfa-qbox_r50_fpn_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/cfa/cfa-qbox_r50_fpn_40e_dota.py' & +python ./tools/train.py configs/cfa/cfa-qbox_r50_fpn_40e_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/convnext/rotated-retinanet-rbox-le90_convnext-tiny_fpn_kld-stable_adamw-1x_dota.py' & +python ./tools/train.py configs/convnext/rotated-retinanet-rbox-le90_convnext-tiny_fpn_kld-stable_adamw-1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/csl/rotated-retinanet-rbox-le90_r50_fpn_csl-gaussian_amp-1x_dota.py' & +python ./tools/train.py configs/csl/rotated-retinanet-rbox-le90_r50_fpn_csl-gaussian_amp-1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/gliding_vertex/gliding-vertex-rbox_r50_fpn_1x_dota.py' & +python ./tools/train.py configs/gliding_vertex/gliding-vertex-rbox_r50_fpn_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/gwd/rotated-retinanet-hbox-oc_r50_fpn_gwd_1x_dota.py' & +python ./tools/train.py configs/gwd/rotated-retinanet-hbox-oc_r50_fpn_gwd_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/kfiou/rotated-retinanet-hbox-le90_r50_fpn_kfiou_1x_dota.py' & +python ./tools/train.py configs/kfiou/rotated-retinanet-hbox-le90_r50_fpn_kfiou_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/kfiou/rotated-retinanet-hbox-oc_r50_fpn_kfiou_1x_dota.py' & +python ./tools/train.py configs/kfiou/rotated-retinanet-hbox-oc_r50_fpn_kfiou_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/kfiou/rotated-retinanet-hbox-le135_r50_fpn_kfiou_1x_dota.py' & +python ./tools/train.py configs/kfiou/rotated-retinanet-hbox-le135_r50_fpn_kfiou_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/kfiou/r3det-oc_r50_fpn_kfiou-ln_1x_dota.py' & +python ./tools/train.py configs/kfiou/r3det-oc_r50_fpn_kfiou-ln_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/kld/rotated-retinanet-hbox-oc_r50_fpn_kld_1x_dota.py' & +python ./tools/train.py configs/kld/rotated-retinanet-hbox-oc_r50_fpn_kld_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/kld/rotated-retinanet-hbox-oc_r50_fpn_kld-stable_1x_dota.py' & +python ./tools/train.py configs/kld/rotated-retinanet-hbox-oc_r50_fpn_kld-stable_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/kld/rotated-retinanet-rbox-le90_r50_fpn_kld-stable_1x_dota.py' & +python ./tools/train.py configs/kld/rotated-retinanet-rbox-le90_r50_fpn_kld-stable_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/kld/r3det-oc_r50_fpn_kld_1x_dota.py' & +python ./tools/train.py configs/kld/r3det-oc_r50_fpn_kld_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/kld/r3det-oc_r50_fpn_kld-stable_1x_dota.py' & +python ./tools/train.py configs/kld/r3det-oc_r50_fpn_kld-stable_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/kld/r3det-tiny-oc_r50_fpn_kld_1x_dota.py' & +python ./tools/train.py configs/kld/r3det-tiny-oc_r50_fpn_kld_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/kld/rotated-retinanet-rbox-le90_r50_fpn_kld-stable_adamw-1x_dota.py' & +python ./tools/train.py configs/kld/rotated-retinanet-rbox-le90_r50_fpn_kld-stable_adamw-1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/oriented_rcnn/oriented-rcnn-le90_r50_fpn_amp-1x_dota.py' & +python ./tools/train.py configs/oriented_rcnn/oriented-rcnn-le90_r50_fpn_amp-1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/oriented_rcnn/oriented-rcnn-le90_r50_fpn_1x_dota.py' & +python ./tools/train.py configs/oriented_rcnn/oriented-rcnn-le90_r50_fpn_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/oriented_reppoints/oriented-reppoints-qbox_r50_fpn_1x_dota.py' & +python ./tools/train.py configs/oriented_reppoints/oriented-reppoints-qbox_r50_fpn_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/r3det/r3det-oc_r50_fpn_1x_dota.py' & +python ./tools/train.py configs/r3det/r3det-oc_r50_fpn_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/r3det/r3det-tiny-oc_r50_fpn_1x_dota.py' & +python ./tools/train.py configs/r3det/r3det-tiny-oc_r50_fpn_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/redet/redet-le90_re50_refpn_amp-1x_dota.py' & +python ./tools/train.py configs/redet/redet-le90_re50_refpn_amp-1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/redet/redet-le90_re50_refpn_1x_dota.py' & +python ./tools/train.py configs/redet/redet-le90_re50_refpn_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/roi_trans/roi-trans-le90_r50_fpn_amp-1x_dota.py' & +python ./tools/train.py configs/roi_trans/roi-trans-le90_r50_fpn_amp-1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/roi_trans/roi-trans-le90_r50_fpn_1x_dota.py' & +python ./tools/train.py configs/roi_trans/roi-trans-le90_r50_fpn_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/roi_trans/roi-trans-le90_swin-tiny_fpn_1x_dota.py' & +python ./tools/train.py configs/roi_trans/roi-trans-le90_swin-tiny_fpn_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/rotated_atss/rotated-atss-le90_r50_fpn_1x_dota.py' & +python ./tools/train.py configs/rotated_atss/rotated-atss-le90_r50_fpn_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/rotated_faster_rcnn/rotated-faster-rcnn-le90_r50_fpn_1x_dota.py' & +python ./tools/train.py configs/rotated_faster_rcnn/rotated-faster-rcnn-le90_r50_fpn_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/rotated_fcos/rotated-fcos-hbox-le90_r50_fpn_1x_dota.py' & +python ./tools/train.py configs/rotated_fcos/rotated-fcos-hbox-le90_r50_fpn_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/rotated_fcos/rotated-fcos-le90_r50_fpn_1x_dota.py' & +python ./tools/train.py configs/rotated_fcos/rotated-fcos-le90_r50_fpn_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/rotated_fcos/rotated-fcos-hbox-le90_r50_fpn_csl-gaussian_1x_dota.py' & +python ./tools/train.py configs/rotated_fcos/rotated-fcos-hbox-le90_r50_fpn_csl-gaussian_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/rotated_fcos/rotated-fcos-le90_r50_fpn_kld_1x_dota.py' & +python ./tools/train.py configs/rotated_fcos/rotated-fcos-le90_r50_fpn_kld_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/rotated_reppoints/rotated_reppoints_r50_fpn_1x_dota_oc.py' & +python ./tools/train.py configs/rotated_reppoints/rotated_reppoints_r50_fpn_1x_dota_oc.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/rotated_retinanet/rotated-retinanet-hbox-oc_r50_fpn_1x_dota.py' & +python ./tools/train.py configs/rotated_retinanet/rotated-retinanet-hbox-oc_r50_fpn_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/rotated_retinanet/rotated-retinanet-rbox-le90_r50_fpn_1x_dota.py' & +python ./tools/train.py configs/rotated_retinanet/rotated-retinanet-rbox-le90_r50_fpn_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/rotated_retinanet/rotated-retinanet-rbox-le90_r50_fpn_amp-1x_dota.py' & +python ./tools/train.py configs/rotated_retinanet/rotated-retinanet-rbox-le90_r50_fpn_amp-1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/rotated_retinanet/rotated-retinanet-rbox-le135_r50_fpn_1x_dota.py' & +python ./tools/train.py configs/rotated_retinanet/rotated-retinanet-rbox-le135_r50_fpn_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/s2anet/s2anet-le135_r50_fpn_1x_dota.py' & +python ./tools/train.py configs/s2anet/s2anet-le135_r50_fpn_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/s2anet/s2anet-le135_r50_fpn_amp-1x_dota.py' & +python ./tools/train.py configs/s2anet/s2anet-le135_r50_fpn_amp-1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/sasm_reppoints/sasm-reppoints-qbox_r50_fpn_1x_dota.py' & +python ./tools/train.py configs/sasm_reppoints/sasm-reppoints-qbox_r50_fpn_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & From 9576feba6a206003394c49158c550b1a0a75b03a Mon Sep 17 00:00:00 2001 From: erenzhou <592267829@qq.com> Date: Sun, 23 Oct 2022 08:47:09 +0800 Subject: [PATCH 3/3] update --- .dev_scripts/batch_train_list.txt | 2 +- .dev_scripts/train_benchmark.sh | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.dev_scripts/batch_train_list.txt b/.dev_scripts/batch_train_list.txt index 7cad6d644..54a3f300d 100644 --- a/.dev_scripts/batch_train_list.txt +++ b/.dev_scripts/batch_train_list.txt @@ -31,7 +31,7 @@ configs/rotated_fcos/rotated-fcos-hbox-le90_r50_fpn_1x_dota.py configs/rotated_fcos/rotated-fcos-le90_r50_fpn_1x_dota.py configs/rotated_fcos/rotated-fcos-hbox-le90_r50_fpn_csl-gaussian_1x_dota.py configs/rotated_fcos/rotated-fcos-le90_r50_fpn_kld_1x_dota.py -configs/rotated_reppoints/rotated_reppoints_r50_fpn_1x_dota_oc.py +configs/rotated_reppoints/rotated-reppoints-qbox_r50_fpn_1x_dota.py configs/rotated_retinanet/rotated-retinanet-hbox-oc_r50_fpn_1x_dota.py configs/rotated_retinanet/rotated-retinanet-rbox-le90_r50_fpn_1x_dota.py configs/rotated_retinanet/rotated-retinanet-rbox-le90_r50_fpn_amp-1x_dota.py diff --git a/.dev_scripts/train_benchmark.sh b/.dev_scripts/train_benchmark.sh index 9bafb12aa..9a528328b 100644 --- a/.dev_scripts/train_benchmark.sh +++ b/.dev_scripts/train_benchmark.sh @@ -6,8 +6,8 @@ echo 'configs/convnext/rotated-retinanet-rbox-le90_convnext-tiny_fpn_kld-stable_ python ./tools/train.py configs/convnext/rotated-retinanet-rbox-le90_convnext-tiny_fpn_kld-stable_adamw-1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/csl/rotated-retinanet-rbox-le90_r50_fpn_csl-gaussian_amp-1x_dota.py' & python ./tools/train.py configs/csl/rotated-retinanet-rbox-le90_r50_fpn_csl-gaussian_amp-1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & -echo 'configs/gliding_vertex/gliding-vertex-rbox_r50_fpn_1x_dota.py' & -python ./tools/train.py configs/gliding_vertex/gliding-vertex-rbox_r50_fpn_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +# echo 'configs/gliding_vertex/gliding-vertex-rbox_r50_fpn_1x_dota.py' & +# python ./tools/train.py configs/gliding_vertex/gliding-vertex-rbox_r50_fpn_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/gwd/rotated-retinanet-hbox-oc_r50_fpn_gwd_1x_dota.py' & python ./tools/train.py configs/gwd/rotated-retinanet-hbox-oc_r50_fpn_gwd_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/kfiou/rotated-retinanet-hbox-le90_r50_fpn_kfiou_1x_dota.py' & @@ -43,9 +43,9 @@ python ./tools/train.py configs/r3det/r3det-oc_r50_fpn_1x_dota.py --cfg-options echo 'configs/r3det/r3det-tiny-oc_r50_fpn_1x_dota.py' & python ./tools/train.py configs/r3det/r3det-tiny-oc_r50_fpn_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/redet/redet-le90_re50_refpn_amp-1x_dota.py' & -python ./tools/train.py configs/redet/redet-le90_re50_refpn_amp-1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +./tools/dist_train.sh configs/redet/redet-le90_re50_refpn_amp-1x_dota.py 1 --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/redet/redet-le90_re50_refpn_1x_dota.py' & -python ./tools/train.py configs/redet/redet-le90_re50_refpn_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +./tools/dist_train.sh configs/redet/redet-le90_re50_refpn_1x_dota.py 1 --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/roi_trans/roi-trans-le90_r50_fpn_amp-1x_dota.py' & python ./tools/train.py configs/roi_trans/roi-trans-le90_r50_fpn_amp-1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/roi_trans/roi-trans-le90_r50_fpn_1x_dota.py' & @@ -64,8 +64,8 @@ echo 'configs/rotated_fcos/rotated-fcos-hbox-le90_r50_fpn_csl-gaussian_1x_dota.p python ./tools/train.py configs/rotated_fcos/rotated-fcos-hbox-le90_r50_fpn_csl-gaussian_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/rotated_fcos/rotated-fcos-le90_r50_fpn_kld_1x_dota.py' & python ./tools/train.py configs/rotated_fcos/rotated-fcos-le90_r50_fpn_kld_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & -echo 'configs/rotated_reppoints/rotated_reppoints_r50_fpn_1x_dota_oc.py' & -python ./tools/train.py configs/rotated_reppoints/rotated_reppoints_r50_fpn_1x_dota_oc.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & +echo 'configs/rotated_reppoints/rotated-reppoints-qbox_r50_fpn_1x_dota.py' & +python ./tools/train.py configs/rotated_reppoints/rotated-reppoints-qbox_r50_fpn_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/rotated_retinanet/rotated-retinanet-hbox-oc_r50_fpn_1x_dota.py' & python ./tools/train.py configs/rotated_retinanet/rotated-retinanet-hbox-oc_r50_fpn_1x_dota.py --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/rotated_retinanet/rotated-retinanet-rbox-le90_r50_fpn_1x_dota.py' &