From b35452473ac28165a11bdf51f356a2e9ab0f8e95 Mon Sep 17 00:00:00 2001 From: HubHop Date: Thu, 16 Jun 2022 20:54:55 +1000 Subject: [PATCH] Upload code --- .gitignore | 141 ++ README.md | 170 +- classification/README.md | 83 + classification/config.py | 235 +++ .../configs/litv2-base-finetune-384.yaml | 27 + classification/configs/litv2-base.yaml | 16 + classification/configs/litv2-medium.yaml | 16 + classification/configs/litv2-small.yaml | 16 + classification/data/__init__.py | 1 + classification/data/build.py | 132 ++ classification/data/cached_image_folder.py | 244 +++ classification/data/samplers.py | 22 + classification/data/zipreader.py | 96 + classification/logger.py | 34 + classification/lr_scheduler.py | 95 + classification/main.py | 362 ++++ .../mm_modules/DCN/deform_conv2d_naive.py | 93 + .../mm_modules/DCN/functions/__init__.py | 2 + .../DCN/functions/deform_conv2d_func.py | 62 + .../functions/modulated_deform_conv2d_func.py | 62 + .../mm_modules/DCN/modules/__init__.py | 2 + .../mm_modules/DCN/modules/deform_conv2d.py | 143 ++ .../DCN/modules/modulated_deform_conv2d.py | 111 + classification/mm_modules/DCN/setup.py | 63 + .../DCN/src/cpu/deform_conv2d_cpu.cpp | 47 + .../DCN/src/cpu/deform_conv2d_cpu.h | 39 + .../src/cpu/modulated_deform_conv2d_cpu.cpp | 49 + .../DCN/src/cpu/modulated_deform_conv2d_cpu.h | 41 + .../DCN/src/cuda/deform_2d_im2col_cuda.cuh | 391 ++++ .../DCN/src/cuda/deform_conv2d_cuda.cu | 273 +++ .../DCN/src/cuda/deform_conv2d_cuda.h | 38 + .../cuda/modulated_deform_2d_im2col_cuda.cuh | 420 ++++ .../src/cuda/modulated_deform_conv2d_cuda.cu | 280 +++ .../src/cuda/modulated_deform_conv2d_cuda.h | 40 + .../mm_modules/DCN/src/deform_conv2d.h | 84 + .../DCN/src/modulated_deform_conv2d.h | 87 + classification/mm_modules/DCN/src/vision.cpp | 10 + classification/mm_modules/__init__.py | 3 + .../mm_modules/cocoapi_evaluator.py | 209 ++ classification/mm_modules/distributed_util.py | 162 ++ .../mm_modules/fp16_utils/README.md | 16 + .../mm_modules/fp16_utils/__init__.py | 16 + .../mm_modules/fp16_utils/fp16_optimizer.py | 561 +++++ .../mm_modules/fp16_utils/fp16util.py | 185 ++ .../mm_modules/fp16_utils/loss_scaler.py | 186 ++ classification/mm_modules/utils.py | 138 ++ classification/mm_modules/vis_utils.py | 113 + classification/mm_modules/voc_evaluator.py | 204 ++ classification/models/__init__.py | 1 + classification/models/attentions.py | 284 +++ classification/models/blocks.py | 141 ++ classification/models/build.py | 28 + classification/models/layers.py | 70 + classification/models/litv2.py | 111 + classification/models/patch_embed.py | 132 ++ classification/optimizer.py | 60 + classification/utils.py | 136 ++ detection/.github/CODE_OF_CONDUCT.md | 76 + detection/.github/CONTRIBUTING.md | 1 + detection/.github/ISSUE_TEMPLATE/config.yml | 9 + .../.github/ISSUE_TEMPLATE/error-report.md | 47 + .../.github/ISSUE_TEMPLATE/feature_request.md | 22 + .../ISSUE_TEMPLATE/general_questions.md | 8 + .../reimplementation_questions.md | 68 + detection/.gitignore | 121 ++ detection/LICENSE | 220 ++ detection/README.md | 81 + .../configs/_base_/datasets/coco_detection.py | 48 + .../configs/_base_/datasets/coco_instance.py | 48 + .../_base_/datasets/coco_instance_semantic.py | 53 + detection/configs/_base_/default_runtime.py | 16 + .../configs/_base_/models/mask_rcnn_fpn.py | 127 ++ .../configs/_base_/models/retinanet_fpn.py | 67 + .../configs/_base_/schedules/schedule_1x.py | 11 + .../configs/_base_/schedules/schedule_20e.py | 11 + .../configs/_base_/schedules/schedule_2x.py | 11 + .../litv2/mask_rcnn_litv2_b_fpn_1x_coco.py | 34 + .../mask_rcnn_litv2_b_fpn_1x_coco_ws_4.py | 34 + .../litv2/mask_rcnn_litv2_m_fpn_1x_coco.py | 34 + .../mask_rcnn_litv2_m_fpn_1x_coco_ws_4.py | 34 + .../litv2/mask_rcnn_litv2_s_fpn_1x_coco.py | 34 + .../mask_rcnn_litv2_s_fpn_1x_coco_ws_4.py | 34 + .../litv2/retinanet_litv2_b_fpn_1x_coco.py | 42 + .../retinanet_litv2_b_fpn_1x_coco_ws_4.py | 42 + .../litv2/retinanet_litv2_m_fpn_1x_coco.py | 41 + .../retinanet_litv2_m_fpn_1x_coco_ws_4.py | 41 + .../litv2/retinanet_litv2_s_fpn_1x_coco.py | 43 + .../retinanet_litv2_s_fpn_1x_coco_ws_4.py | 43 + .../mm_modules/DCN/deform_conv2d_naive.py | 93 + .../mm_modules/DCN/functions/__init__.py | 2 + .../DCN/functions/deform_conv2d_func.py | 64 + .../functions/modulated_deform_conv2d_func.py | 65 + detection/mm_modules/DCN/make.sh | 33 + detection/mm_modules/DCN/make_dsai.sh | 22 + detection/mm_modules/DCN/modules/__init__.py | 2 + .../mm_modules/DCN/modules/deform_conv2d.py | 152 ++ .../DCN/modules/modulated_deform_conv2d.py | 111 + detection/mm_modules/DCN/setup.py | 63 + .../DCN/src/cpu/deform_conv2d_cpu.cpp | 47 + .../DCN/src/cpu/deform_conv2d_cpu.h | 39 + .../src/cpu/modulated_deform_conv2d_cpu.cpp | 49 + .../DCN/src/cpu/modulated_deform_conv2d_cpu.h | 41 + .../DCN/src/cuda/deform_2d_im2col_cuda.cuh | 391 ++++ .../DCN/src/cuda/deform_conv2d_cuda.cu | 273 +++ .../DCN/src/cuda/deform_conv2d_cuda.h | 38 + .../cuda/modulated_deform_2d_im2col_cuda.cuh | 420 ++++ .../src/cuda/modulated_deform_conv2d_cuda.cu | 280 +++ .../src/cuda/modulated_deform_conv2d_cuda.h | 40 + detection/mm_modules/DCN/src/deform_conv2d.h | 84 + .../DCN/src/modulated_deform_conv2d.h | 87 + detection/mm_modules/DCN/src/vision.cpp | 10 + detection/mm_modules/__init__.py | 3 + detection/mm_modules/cocoapi_evaluator.py | 209 ++ detection/mm_modules/distributed_util.py | 162 ++ detection/mm_modules/fp16_utils/README.md | 16 + detection/mm_modules/fp16_utils/__init__.py | 16 + .../mm_modules/fp16_utils/fp16_optimizer.py | 561 +++++ detection/mm_modules/fp16_utils/fp16util.py | 185 ++ .../mm_modules/fp16_utils/loss_scaler.py | 186 ++ detection/mm_modules/utils.py | 138 ++ detection/mm_modules/vis_utils.py | 113 + detection/mm_modules/voc_evaluator.py | 204 ++ detection/mmcv_custom/__init__.py | 5 + detection/mmcv_custom/checkpoint.py | 500 +++++ detection/mmcv_custom/runner/__init__.py | 8 + detection/mmcv_custom/runner/checkpoint.py | 85 + .../mmcv_custom/runner/epoch_based_runner.py | 104 + detection/mmdet/__init__.py | 28 + detection/mmdet/apis/__init__.py | 10 + detection/mmdet/apis/inference.py | 217 ++ detection/mmdet/apis/test.py | 190 ++ detection/mmdet/apis/train.py | 185 ++ detection/mmdet/core/__init__.py | 7 + detection/mmdet/core/anchor/__init__.py | 11 + .../mmdet/core/anchor/anchor_generator.py | 727 +++++++ detection/mmdet/core/anchor/builder.py | 7 + .../mmdet/core/anchor/point_generator.py | 37 + detection/mmdet/core/anchor/utils.py | 71 + detection/mmdet/core/bbox/__init__.py | 27 + .../mmdet/core/bbox/assigners/__init__.py | 16 + .../bbox/assigners/approx_max_iou_assigner.py | 145 ++ .../core/bbox/assigners/assign_result.py | 204 ++ .../core/bbox/assigners/atss_assigner.py | 178 ++ .../core/bbox/assigners/base_assigner.py | 9 + .../bbox/assigners/center_region_assigner.py | 335 +++ .../core/bbox/assigners/grid_assigner.py | 155 ++ .../core/bbox/assigners/hungarian_assigner.py | 145 ++ .../core/bbox/assigners/max_iou_assigner.py | 212 ++ .../core/bbox/assigners/point_assigner.py | 133 ++ .../core/bbox/assigners/region_assigner.py | 221 ++ detection/mmdet/core/bbox/builder.py | 20 + detection/mmdet/core/bbox/coder/__init__.py | 13 + .../mmdet/core/bbox/coder/base_bbox_coder.py | 17 + .../core/bbox/coder/bucketing_bbox_coder.py | 350 ++++ .../core/bbox/coder/delta_xywh_bbox_coder.py | 237 +++ .../coder/legacy_delta_xywh_bbox_coder.py | 215 ++ .../core/bbox/coder/pseudo_bbox_coder.py | 18 + .../mmdet/core/bbox/coder/tblr_bbox_coder.py | 198 ++ .../mmdet/core/bbox/coder/yolo_bbox_coder.py | 89 + detection/mmdet/core/bbox/demodata.py | 41 + .../core/bbox/iou_calculators/__init__.py | 4 + .../core/bbox/iou_calculators/builder.py | 8 + .../bbox/iou_calculators/iou2d_calculator.py | 159 ++ .../mmdet/core/bbox/match_costs/__init__.py | 7 + .../mmdet/core/bbox/match_costs/builder.py | 8 + .../mmdet/core/bbox/match_costs/match_cost.py | 184 ++ .../mmdet/core/bbox/samplers/__init__.py | 15 + .../mmdet/core/bbox/samplers/base_sampler.py | 101 + .../core/bbox/samplers/combined_sampler.py | 20 + .../samplers/instance_balanced_pos_sampler.py | 55 + .../bbox/samplers/iou_balanced_neg_sampler.py | 157 ++ .../mmdet/core/bbox/samplers/ohem_sampler.py | 107 + .../core/bbox/samplers/pseudo_sampler.py | 41 + .../core/bbox/samplers/random_sampler.py | 78 + .../core/bbox/samplers/sampling_result.py | 152 ++ .../core/bbox/samplers/score_hlr_sampler.py | 264 +++ detection/mmdet/core/bbox/transforms.py | 240 +++ detection/mmdet/core/evaluation/__init__.py | 15 + .../mmdet/core/evaluation/bbox_overlaps.py | 48 + .../mmdet/core/evaluation/class_names.py | 116 ++ detection/mmdet/core/evaluation/eval_hooks.py | 303 +++ detection/mmdet/core/evaluation/mean_ap.py | 469 +++++ detection/mmdet/core/evaluation/recall.py | 189 ++ detection/mmdet/core/export/__init__.py | 8 + detection/mmdet/core/export/pytorch2onnx.py | 154 ++ detection/mmdet/core/mask/__init__.py | 8 + detection/mmdet/core/mask/mask_target.py | 122 ++ detection/mmdet/core/mask/structures.py | 1024 ++++++++++ detection/mmdet/core/mask/utils.py | 63 + .../mmdet/core/post_processing/__init__.py | 8 + .../mmdet/core/post_processing/bbox_nms.py | 168 ++ .../mmdet/core/post_processing/merge_augs.py | 150 ++ detection/mmdet/core/utils/__init__.py | 7 + detection/mmdet/core/utils/dist_utils.py | 69 + detection/mmdet/core/utils/misc.py | 61 + .../mmdet/core/visualization/__init__.py | 4 + detection/mmdet/core/visualization/image.py | 303 +++ detection/mmdet/datasets/__init__.py | 24 + detection/mmdet/datasets/builder.py | 143 ++ detection/mmdet/datasets/cityscapes.py | 334 +++ detection/mmdet/datasets/coco.py | 546 +++++ detection/mmdet/datasets/custom.py | 323 +++ detection/mmdet/datasets/dataset_wrappers.py | 282 +++ detection/mmdet/datasets/deepfashion.py | 10 + detection/mmdet/datasets/lvis.py | 742 +++++++ .../mmdet/datasets/pipelines/__init__.py | 25 + .../mmdet/datasets/pipelines/auto_augment.py | 890 ++++++++ detection/mmdet/datasets/pipelines/compose.py | 51 + .../mmdet/datasets/pipelines/formating.py | 364 ++++ .../mmdet/datasets/pipelines/instaboost.py | 98 + detection/mmdet/datasets/pipelines/loading.py | 458 +++++ .../mmdet/datasets/pipelines/test_time_aug.py | 119 ++ .../mmdet/datasets/pipelines/transforms.py | 1811 +++++++++++++++++ detection/mmdet/datasets/samplers/__init__.py | 4 + .../datasets/samplers/distributed_sampler.py | 39 + .../mmdet/datasets/samplers/group_sampler.py | 148 ++ detection/mmdet/datasets/utils.py | 158 ++ detection/mmdet/datasets/voc.py | 93 + detection/mmdet/datasets/wider_face.py | 51 + detection/mmdet/datasets/xml_style.py | 170 ++ detection/mmdet/models/__init__.py | 16 + detection/mmdet/models/backbones/__init__.py | 19 + detection/mmdet/models/backbones/darknet.py | 199 ++ .../models/backbones/detectors_resnet.py | 305 +++ .../models/backbones/detectors_resnext.py | 122 ++ detection/mmdet/models/backbones/hourglass.py | 198 ++ detection/mmdet/models/backbones/hrnet.py | 537 +++++ detection/mmdet/models/backbones/litv2.py | 646 ++++++ detection/mmdet/models/backbones/regnet.py | 325 +++ detection/mmdet/models/backbones/res2net.py | 351 ++++ detection/mmdet/models/backbones/resnest.py | 317 +++ detection/mmdet/models/backbones/resnet.py | 663 ++++++ detection/mmdet/models/backbones/resnext.py | 153 ++ detection/mmdet/models/backbones/ssd_vgg.py | 169 ++ .../mmdet/models/backbones/trident_resnet.py | 292 +++ detection/mmdet/models/builder.py | 77 + .../mmdet/models/dense_heads/__init__.py | 41 + .../models/dense_heads/anchor_free_head.py | 340 ++++ .../mmdet/models/dense_heads/anchor_head.py | 751 +++++++ .../mmdet/models/dense_heads/atss_head.py | 689 +++++++ .../models/dense_heads/base_dense_head.py | 59 + .../models/dense_heads/cascade_rpn_head.py | 784 +++++++ .../models/dense_heads/centripetal_head.py | 421 ++++ .../mmdet/models/dense_heads/corner_head.py | 1074 ++++++++++ .../models/dense_heads/dense_test_mixins.py | 100 + .../models/dense_heads/embedding_rpn_head.py | 100 + .../mmdet/models/dense_heads/fcos_head.py | 629 ++++++ .../mmdet/models/dense_heads/fovea_head.py | 341 ++++ .../dense_heads/free_anchor_retina_head.py | 270 +++ .../mmdet/models/dense_heads/fsaf_head.py | 422 ++++ .../models/dense_heads/ga_retina_head.py | 109 + .../mmdet/models/dense_heads/ga_rpn_head.py | 171 ++ .../mmdet/models/dense_heads/gfl_head.py | 647 ++++++ .../models/dense_heads/guided_anchor_head.py | 860 ++++++++ detection/mmdet/models/dense_heads/ld_head.py | 261 +++ .../mmdet/models/dense_heads/nasfcos_head.py | 75 + .../mmdet/models/dense_heads/paa_head.py | 671 ++++++ .../models/dense_heads/pisa_retinanet_head.py | 154 ++ .../mmdet/models/dense_heads/pisa_ssd_head.py | 139 ++ .../models/dense_heads/reppoints_head.py | 763 +++++++ .../mmdet/models/dense_heads/retina_head.py | 114 ++ .../models/dense_heads/retina_sepbn_head.py | 113 + .../mmdet/models/dense_heads/rpn_head.py | 236 +++ .../models/dense_heads/rpn_test_mixin.py | 59 + .../models/dense_heads/sabl_retina_head.py | 621 ++++++ .../mmdet/models/dense_heads/ssd_head.py | 265 +++ .../models/dense_heads/transformer_head.py | 654 ++++++ .../mmdet/models/dense_heads/vfnet_head.py | 794 ++++++++ .../mmdet/models/dense_heads/yolact_head.py | 943 +++++++++ .../mmdet/models/dense_heads/yolo_head.py | 577 ++++++ detection/mmdet/models/detectors/__init__.py | 40 + detection/mmdet/models/detectors/atss.py | 17 + detection/mmdet/models/detectors/base.py | 355 ++++ .../mmdet/models/detectors/cascade_rcnn.py | 46 + detection/mmdet/models/detectors/cornernet.py | 95 + detection/mmdet/models/detectors/detr.py | 46 + detection/mmdet/models/detectors/fast_rcnn.py | 52 + .../mmdet/models/detectors/faster_rcnn.py | 24 + detection/mmdet/models/detectors/fcos.py | 17 + detection/mmdet/models/detectors/fovea.py | 17 + detection/mmdet/models/detectors/fsaf.py | 17 + detection/mmdet/models/detectors/gfl.py | 16 + detection/mmdet/models/detectors/grid_rcnn.py | 29 + detection/mmdet/models/detectors/htc.py | 15 + .../mmdet/models/detectors/kd_one_stage.py | 100 + detection/mmdet/models/detectors/mask_rcnn.py | 24 + .../models/detectors/mask_scoring_rcnn.py | 27 + detection/mmdet/models/detectors/nasfcos.py | 20 + detection/mmdet/models/detectors/paa.py | 17 + .../mmdet/models/detectors/point_rend.py | 29 + .../models/detectors/reppoints_detector.py | 22 + detection/mmdet/models/detectors/retinanet.py | 17 + detection/mmdet/models/detectors/rpn.py | 154 ++ detection/mmdet/models/detectors/scnet.py | 10 + .../mmdet/models/detectors/single_stage.py | 154 ++ .../mmdet/models/detectors/sparse_rcnn.py | 110 + .../models/detectors/trident_faster_rcnn.py | 66 + detection/mmdet/models/detectors/two_stage.py | 215 ++ detection/mmdet/models/detectors/vfnet.py | 18 + detection/mmdet/models/detectors/yolact.py | 146 ++ detection/mmdet/models/detectors/yolo.py | 18 + detection/mmdet/models/losses/__init__.py | 29 + detection/mmdet/models/losses/accuracy.py | 78 + detection/mmdet/models/losses/ae_loss.py | 102 + .../mmdet/models/losses/balanced_l1_loss.py | 120 ++ .../mmdet/models/losses/cross_entropy_loss.py | 214 ++ detection/mmdet/models/losses/focal_loss.py | 181 ++ .../models/losses/gaussian_focal_loss.py | 91 + detection/mmdet/models/losses/gfocal_loss.py | 188 ++ detection/mmdet/models/losses/ghm_loss.py | 172 ++ detection/mmdet/models/losses/iou_loss.py | 436 ++++ detection/mmdet/models/losses/kd_loss.py | 87 + detection/mmdet/models/losses/mse_loss.py | 49 + detection/mmdet/models/losses/pisa_loss.py | 183 ++ .../mmdet/models/losses/smooth_l1_loss.py | 139 ++ detection/mmdet/models/losses/utils.py | 100 + .../mmdet/models/losses/varifocal_loss.py | 133 ++ detection/mmdet/models/necks/__init__.py | 16 + detection/mmdet/models/necks/bfp.py | 104 + .../mmdet/models/necks/channel_mapper.py | 74 + detection/mmdet/models/necks/fpg.py | 398 ++++ detection/mmdet/models/necks/fpn.py | 221 ++ detection/mmdet/models/necks/fpn_carafe.py | 267 +++ detection/mmdet/models/necks/hrfpn.py | 102 + detection/mmdet/models/necks/nas_fpn.py | 160 ++ detection/mmdet/models/necks/nasfcos_fpn.py | 161 ++ detection/mmdet/models/necks/pafpn.py | 142 ++ detection/mmdet/models/necks/rfp.py | 128 ++ detection/mmdet/models/necks/yolo_neck.py | 136 ++ detection/mmdet/models/roi_heads/__init__.py | 34 + .../mmdet/models/roi_heads/base_roi_head.py | 103 + .../models/roi_heads/bbox_heads/__init__.py | 13 + .../models/roi_heads/bbox_heads/bbox_head.py | 483 +++++ .../roi_heads/bbox_heads/convfc_bbox_head.py | 205 ++ .../models/roi_heads/bbox_heads/dii_head.py | 415 ++++ .../roi_heads/bbox_heads/double_bbox_head.py | 172 ++ .../models/roi_heads/bbox_heads/sabl_head.py | 572 ++++++ .../roi_heads/bbox_heads/scnet_bbox_head.py | 76 + .../models/roi_heads/cascade_roi_head.py | 507 +++++ .../mmdet/models/roi_heads/double_roi_head.py | 33 + .../models/roi_heads/dynamic_roi_head.py | 154 ++ .../mmdet/models/roi_heads/grid_roi_head.py | 176 ++ .../mmdet/models/roi_heads/htc_roi_head.py | 589 ++++++ .../models/roi_heads/mask_heads/__init__.py | 17 + .../roi_heads/mask_heads/coarse_mask_head.py | 91 + .../roi_heads/mask_heads/fcn_mask_head.py | 377 ++++ .../mask_heads/feature_relay_head.py | 55 + .../mask_heads/fused_semantic_head.py | 107 + .../mask_heads/global_context_head.py | 102 + .../models/roi_heads/mask_heads/grid_head.py | 359 ++++ .../roi_heads/mask_heads/htc_mask_head.py | 43 + .../roi_heads/mask_heads/mask_point_head.py | 300 +++ .../roi_heads/mask_heads/maskiou_head.py | 186 ++ .../roi_heads/mask_heads/scnet_mask_head.py | 27 + .../mask_heads/scnet_semantic_head.py | 27 + .../models/roi_heads/mask_scoring_roi_head.py | 122 ++ .../mmdet/models/roi_heads/pisa_roi_head.py | 159 ++ .../models/roi_heads/point_rend_roi_head.py | 218 ++ .../roi_heads/roi_extractors/__init__.py | 7 + .../roi_extractors/base_roi_extractor.py | 83 + .../roi_extractors/generic_roi_extractor.py | 83 + .../single_level_roi_extractor.py | 108 + .../mmdet/models/roi_heads/scnet_roi_head.py | 582 ++++++ .../models/roi_heads/shared_heads/__init__.py | 3 + .../roi_heads/shared_heads/res_layer.py | 77 + .../mmdet/models/roi_heads/sparse_roi_head.py | 311 +++ .../models/roi_heads/standard_roi_head.py | 295 +++ .../mmdet/models/roi_heads/test_mixins.py | 348 ++++ .../models/roi_heads/trident_roi_head.py | 119 ++ detection/mmdet/models/utils/__init__.py | 16 + detection/mmdet/models/utils/builder.py | 14 + .../mmdet/models/utils/gaussian_target.py | 185 ++ .../mmdet/models/utils/positional_encoding.py | 150 ++ detection/mmdet/models/utils/res_layer.py | 187 ++ detection/mmdet/models/utils/transformer.py | 860 ++++++++ detection/mmdet/utils/__init__.py | 5 + detection/mmdet/utils/collect_env.py | 16 + detection/mmdet/utils/contextmanagers.py | 121 ++ detection/mmdet/utils/logger.py | 19 + detection/mmdet/utils/optimizer.py | 33 + detection/mmdet/utils/profiling.py | 39 + detection/mmdet/utils/util_mixins.py | 104 + detection/mmdet/utils/util_random.py | 33 + detection/mmdet/version.py | 19 + detection/requirements.txt | 4 + detection/setup.cfg | 13 + detection/setup.py | 161 ++ detection/tools/benchmark.py | 189 ++ detection/tools/dist_test.sh | 10 + detection/tools/dist_train.sh | 9 + detection/tools/get_flops.py | 125 ++ detection/tools/test.py | 220 ++ detection/tools/train.py | 190 ++ segmentation/.github/CODE_OF_CONDUCT.md | 76 + segmentation/.github/CONTRIBUTING.md | 57 + .../.github/ISSUE_TEMPLATE/config.yml | 6 + .../.github/ISSUE_TEMPLATE/error-report.md | 48 + .../.github/ISSUE_TEMPLATE/feature_request.md | 22 + .../ISSUE_TEMPLATE/general_questions.md | 8 + segmentation/.gitignore | 117 ++ segmentation/LICENSE | 203 ++ segmentation/README.md | 91 + .../configs/_base_/datasets/ade20k.py | 54 + .../configs/_base_/default_runtime.py | 14 + .../configs/_base_/models/fpn_r50_lit.py | 44 + .../configs/_base_/schedules/schedule_160k.py | 9 + .../configs/_base_/schedules/schedule_20k.py | 9 + .../configs/_base_/schedules/schedule_40k.py | 9 + .../configs/_base_/schedules/schedule_80k.py | 9 + .../litv2_b_fpn_r50_512x512_80k_ade20k.py | 28 + .../litv2_m_fpn_r50_512x512_80k_ade20k.py | 28 + .../litv2_s_fpn_r50_512x512_80k_ade20k.py | 28 + .../mm_modules/DCN/deform_conv2d_naive.py | 93 + .../mm_modules/DCN/functions/__init__.py | 2 + .../DCN/functions/deform_conv2d_func.py | 64 + .../functions/modulated_deform_conv2d_func.py | 65 + segmentation/mm_modules/DCN/make.sh | 33 + segmentation/mm_modules/DCN/make_dsai.sh | 22 + .../mm_modules/DCN/modules/__init__.py | 2 + .../mm_modules/DCN/modules/deform_conv2d.py | 152 ++ .../DCN/modules/modulated_deform_conv2d.py | 111 + segmentation/mm_modules/DCN/setup.py | 63 + .../DCN/src/cpu/deform_conv2d_cpu.cpp | 47 + .../DCN/src/cpu/deform_conv2d_cpu.h | 39 + .../src/cpu/modulated_deform_conv2d_cpu.cpp | 49 + .../DCN/src/cpu/modulated_deform_conv2d_cpu.h | 41 + .../DCN/src/cuda/deform_2d_im2col_cuda.cuh | 391 ++++ .../DCN/src/cuda/deform_conv2d_cuda.cu | 273 +++ .../DCN/src/cuda/deform_conv2d_cuda.h | 38 + .../cuda/modulated_deform_2d_im2col_cuda.cuh | 420 ++++ .../src/cuda/modulated_deform_conv2d_cuda.cu | 280 +++ .../src/cuda/modulated_deform_conv2d_cuda.h | 40 + .../mm_modules/DCN/src/deform_conv2d.h | 84 + .../DCN/src/modulated_deform_conv2d.h | 87 + segmentation/mm_modules/DCN/src/vision.cpp | 10 + segmentation/mm_modules/__init__.py | 3 + segmentation/mm_modules/cocoapi_evaluator.py | 209 ++ segmentation/mm_modules/distributed_util.py | 162 ++ segmentation/mm_modules/fp16_utils/README.md | 16 + .../mm_modules/fp16_utils/__init__.py | 16 + .../mm_modules/fp16_utils/fp16_optimizer.py | 561 +++++ .../mm_modules/fp16_utils/fp16util.py | 185 ++ .../mm_modules/fp16_utils/loss_scaler.py | 186 ++ segmentation/mm_modules/utils.py | 138 ++ segmentation/mm_modules/vis_utils.py | 113 + segmentation/mm_modules/voc_evaluator.py | 204 ++ segmentation/mmcv_custom/__init__.py | 5 + segmentation/mmcv_custom/checkpoint.py | 500 +++++ segmentation/mmcv_custom/runner/__init__.py | 8 + segmentation/mmcv_custom/runner/checkpoint.py | 85 + .../mmcv_custom/runner/epoch_based_runner.py | 104 + segmentation/mmseg/__init__.py | 30 + segmentation/mmseg/apis/__init__.py | 9 + segmentation/mmseg/apis/inference.py | 135 ++ segmentation/mmseg/apis/test.py | 238 +++ segmentation/mmseg/apis/train.py | 116 ++ segmentation/mmseg/core/__init__.py | 3 + .../mmseg/core/evaluation/__init__.py | 8 + .../mmseg/core/evaluation/class_names.py | 152 ++ .../mmseg/core/evaluation/eval_hooks.py | 109 + segmentation/mmseg/core/evaluation/metrics.py | 326 +++ segmentation/mmseg/core/seg/__init__.py | 4 + segmentation/mmseg/core/seg/builder.py | 8 + .../mmseg/core/seg/sampler/__init__.py | 4 + .../core/seg/sampler/base_pixel_sampler.py | 12 + .../core/seg/sampler/ohem_pixel_sampler.py | 76 + segmentation/mmseg/core/utils/__init__.py | 3 + segmentation/mmseg/core/utils/misc.py | 17 + segmentation/mmseg/datasets/__init__.py | 19 + segmentation/mmseg/datasets/ade.py | 163 ++ segmentation/mmseg/datasets/builder.py | 169 ++ segmentation/mmseg/datasets/chase_db1.py | 27 + segmentation/mmseg/datasets/cityscapes.py | 217 ++ segmentation/mmseg/datasets/custom.py | 400 ++++ .../mmseg/datasets/dataset_wrappers.py | 50 + segmentation/mmseg/datasets/drive.py | 27 + segmentation/mmseg/datasets/hrf.py | 27 + segmentation/mmseg/datasets/pascal_context.py | 103 + .../mmseg/datasets/pipelines/__init__.py | 16 + .../mmseg/datasets/pipelines/compose.py | 51 + .../mmseg/datasets/pipelines/formating.py | 288 +++ .../mmseg/datasets/pipelines/loading.py | 153 ++ .../mmseg/datasets/pipelines/test_time_aug.py | 133 ++ .../mmseg/datasets/pipelines/transforms.py | 889 ++++++++ segmentation/mmseg/datasets/stare.py | 27 + segmentation/mmseg/datasets/voc.py | 29 + segmentation/mmseg/models/__init__.py | 12 + .../mmseg/models/backbones/__init__.py | 18 + segmentation/mmseg/models/backbones/cgnet.py | 367 ++++ .../mmseg/models/backbones/fast_scnn.py | 375 ++++ segmentation/mmseg/models/backbones/hrnet.py | 555 +++++ segmentation/mmseg/models/backbones/litv2.py | 641 ++++++ .../mmseg/models/backbones/mobilenet_v2.py | 180 ++ .../mmseg/models/backbones/mobilenet_v3.py | 255 +++ .../mmseg/models/backbones/resnest.py | 314 +++ segmentation/mmseg/models/backbones/resnet.py | 688 +++++++ .../mmseg/models/backbones/resnext.py | 145 ++ segmentation/mmseg/models/backbones/unet.py | 429 ++++ segmentation/mmseg/models/backbones/vit.py | 470 +++++ segmentation/mmseg/models/builder.py | 46 + .../mmseg/models/decode_heads/__init__.py | 28 + .../mmseg/models/decode_heads/ann_head.py | 245 +++ .../mmseg/models/decode_heads/apc_head.py | 158 ++ .../mmseg/models/decode_heads/aspp_head.py | 107 + .../decode_heads/cascade_decode_head.py | 57 + .../mmseg/models/decode_heads/cc_head.py | 42 + .../mmseg/models/decode_heads/da_head.py | 178 ++ .../mmseg/models/decode_heads/decode_head.py | 234 +++ .../mmseg/models/decode_heads/dm_head.py | 140 ++ .../mmseg/models/decode_heads/dnl_head.py | 131 ++ .../mmseg/models/decode_heads/ema_head.py | 168 ++ .../mmseg/models/decode_heads/enc_head.py | 187 ++ .../mmseg/models/decode_heads/fcn_head.py | 81 + .../mmseg/models/decode_heads/fpn_head.py | 68 + .../mmseg/models/decode_heads/gc_head.py | 47 + .../mmseg/models/decode_heads/lraspp_head.py | 90 + .../mmseg/models/decode_heads/nl_head.py | 49 + .../mmseg/models/decode_heads/ocr_head.py | 127 ++ .../mmseg/models/decode_heads/point_head.py | 349 ++++ .../mmseg/models/decode_heads/psa_head.py | 196 ++ .../mmseg/models/decode_heads/psp_head.py | 101 + .../models/decode_heads/sep_aspp_head.py | 101 + .../mmseg/models/decode_heads/sep_fcn_head.py | 51 + .../mmseg/models/decode_heads/uper_head.py | 126 ++ segmentation/mmseg/models/losses/__init__.py | 12 + segmentation/mmseg/models/losses/accuracy.py | 78 + .../mmseg/models/losses/cross_entropy_loss.py | 198 ++ segmentation/mmseg/models/losses/dice_loss.py | 119 ++ .../mmseg/models/losses/lovasz_loss.py | 303 +++ segmentation/mmseg/models/losses/utils.py | 121 ++ segmentation/mmseg/models/necks/__init__.py | 4 + segmentation/mmseg/models/necks/fpn.py | 212 ++ .../mmseg/models/necks/multilevel_neck.py | 69 + .../mmseg/models/segmentors/__init__.py | 5 + segmentation/mmseg/models/segmentors/base.py | 273 +++ .../segmentors/cascade_encoder_decoder.py | 98 + .../models/segmentors/encoder_decoder.py | 298 +++ segmentation/mmseg/models/utils/__init__.py | 13 + segmentation/mmseg/models/utils/drop.py | 31 + .../mmseg/models/utils/inverted_residual.py | 208 ++ .../mmseg/models/utils/make_divisible.py | 27 + segmentation/mmseg/models/utils/res_layer.py | 94 + segmentation/mmseg/models/utils/se_layer.py | 57 + .../models/utils/self_attention_block.py | 159 ++ .../mmseg/models/utils/up_conv_block.py | 101 + .../mmseg/models/utils/weight_init.py | 62 + segmentation/mmseg/ops/__init__.py | 4 + segmentation/mmseg/ops/encoding.py | 74 + segmentation/mmseg/ops/wrappers.py | 50 + segmentation/mmseg/utils/__init__.py | 4 + segmentation/mmseg/utils/collect_env.py | 17 + segmentation/mmseg/utils/logger.py | 27 + segmentation/mmseg/version.py | 18 + segmentation/requirements.txt | 3 + segmentation/requirements/docs.txt | 4 + segmentation/requirements/optional.txt | 1 + segmentation/requirements/readthedocs.txt | 3 + segmentation/requirements/runtime.txt | 3 + segmentation/requirements/tests.txt | 7 + segmentation/resources/mmseg-logo.png | Bin 0 -> 44728 bytes segmentation/resources/seg_demo.gif | Bin 0 -> 951973 bytes segmentation/setup.cfg | 13 + segmentation/setup.py | 126 ++ segmentation/tools/benchmark.py | 121 ++ segmentation/tools/dist_test.sh | 9 + segmentation/tools/dist_train.sh | 9 + segmentation/tools/get_flops.py | 97 + segmentation/tools/test.py | 157 ++ segmentation/tools/train.py | 163 ++ 569 files changed, 85430 insertions(+), 42 deletions(-) create mode 100644 .gitignore create mode 100644 classification/README.md create mode 100644 classification/config.py create mode 100644 classification/configs/litv2-base-finetune-384.yaml create mode 100644 classification/configs/litv2-base.yaml create mode 100644 classification/configs/litv2-medium.yaml create mode 100644 classification/configs/litv2-small.yaml create mode 100644 classification/data/__init__.py create mode 100644 classification/data/build.py create mode 100644 classification/data/cached_image_folder.py create mode 100644 classification/data/samplers.py create mode 100644 classification/data/zipreader.py create mode 100644 classification/logger.py create mode 100644 classification/lr_scheduler.py create mode 100644 classification/main.py create mode 100644 classification/mm_modules/DCN/deform_conv2d_naive.py create mode 100644 classification/mm_modules/DCN/functions/__init__.py create mode 100644 classification/mm_modules/DCN/functions/deform_conv2d_func.py create mode 100644 classification/mm_modules/DCN/functions/modulated_deform_conv2d_func.py create mode 100644 classification/mm_modules/DCN/modules/__init__.py create mode 100644 classification/mm_modules/DCN/modules/deform_conv2d.py create mode 100644 classification/mm_modules/DCN/modules/modulated_deform_conv2d.py create mode 100644 classification/mm_modules/DCN/setup.py create mode 100644 classification/mm_modules/DCN/src/cpu/deform_conv2d_cpu.cpp create mode 100644 classification/mm_modules/DCN/src/cpu/deform_conv2d_cpu.h create mode 100644 classification/mm_modules/DCN/src/cpu/modulated_deform_conv2d_cpu.cpp create mode 100644 classification/mm_modules/DCN/src/cpu/modulated_deform_conv2d_cpu.h create mode 100644 classification/mm_modules/DCN/src/cuda/deform_2d_im2col_cuda.cuh create mode 100644 classification/mm_modules/DCN/src/cuda/deform_conv2d_cuda.cu create mode 100644 classification/mm_modules/DCN/src/cuda/deform_conv2d_cuda.h create mode 100644 classification/mm_modules/DCN/src/cuda/modulated_deform_2d_im2col_cuda.cuh create mode 100644 classification/mm_modules/DCN/src/cuda/modulated_deform_conv2d_cuda.cu create mode 100644 classification/mm_modules/DCN/src/cuda/modulated_deform_conv2d_cuda.h create mode 100644 classification/mm_modules/DCN/src/deform_conv2d.h create mode 100644 classification/mm_modules/DCN/src/modulated_deform_conv2d.h create mode 100644 classification/mm_modules/DCN/src/vision.cpp create mode 100644 classification/mm_modules/__init__.py create mode 100644 classification/mm_modules/cocoapi_evaluator.py create mode 100644 classification/mm_modules/distributed_util.py create mode 100644 classification/mm_modules/fp16_utils/README.md create mode 100644 classification/mm_modules/fp16_utils/__init__.py create mode 100644 classification/mm_modules/fp16_utils/fp16_optimizer.py create mode 100644 classification/mm_modules/fp16_utils/fp16util.py create mode 100644 classification/mm_modules/fp16_utils/loss_scaler.py create mode 100644 classification/mm_modules/utils.py create mode 100644 classification/mm_modules/vis_utils.py create mode 100644 classification/mm_modules/voc_evaluator.py create mode 100644 classification/models/__init__.py create mode 100644 classification/models/attentions.py create mode 100644 classification/models/blocks.py create mode 100644 classification/models/build.py create mode 100644 classification/models/layers.py create mode 100644 classification/models/litv2.py create mode 100644 classification/models/patch_embed.py create mode 100644 classification/optimizer.py create mode 100644 classification/utils.py create mode 100644 detection/.github/CODE_OF_CONDUCT.md create mode 100644 detection/.github/CONTRIBUTING.md create mode 100644 detection/.github/ISSUE_TEMPLATE/config.yml create mode 100644 detection/.github/ISSUE_TEMPLATE/error-report.md create mode 100644 detection/.github/ISSUE_TEMPLATE/feature_request.md create mode 100644 detection/.github/ISSUE_TEMPLATE/general_questions.md create mode 100644 detection/.github/ISSUE_TEMPLATE/reimplementation_questions.md create mode 100644 detection/.gitignore create mode 100644 detection/LICENSE create mode 100644 detection/README.md create mode 100644 detection/configs/_base_/datasets/coco_detection.py create mode 100644 detection/configs/_base_/datasets/coco_instance.py create mode 100644 detection/configs/_base_/datasets/coco_instance_semantic.py create mode 100644 detection/configs/_base_/default_runtime.py create mode 100644 detection/configs/_base_/models/mask_rcnn_fpn.py create mode 100644 detection/configs/_base_/models/retinanet_fpn.py create mode 100644 detection/configs/_base_/schedules/schedule_1x.py create mode 100644 detection/configs/_base_/schedules/schedule_20e.py create mode 100644 detection/configs/_base_/schedules/schedule_2x.py create mode 100644 detection/configs/litv2/mask_rcnn_litv2_b_fpn_1x_coco.py create mode 100644 detection/configs/litv2/mask_rcnn_litv2_b_fpn_1x_coco_ws_4.py create mode 100644 detection/configs/litv2/mask_rcnn_litv2_m_fpn_1x_coco.py create mode 100644 detection/configs/litv2/mask_rcnn_litv2_m_fpn_1x_coco_ws_4.py create mode 100644 detection/configs/litv2/mask_rcnn_litv2_s_fpn_1x_coco.py create mode 100644 detection/configs/litv2/mask_rcnn_litv2_s_fpn_1x_coco_ws_4.py create mode 100644 detection/configs/litv2/retinanet_litv2_b_fpn_1x_coco.py create mode 100644 detection/configs/litv2/retinanet_litv2_b_fpn_1x_coco_ws_4.py create mode 100644 detection/configs/litv2/retinanet_litv2_m_fpn_1x_coco.py create mode 100644 detection/configs/litv2/retinanet_litv2_m_fpn_1x_coco_ws_4.py create mode 100644 detection/configs/litv2/retinanet_litv2_s_fpn_1x_coco.py create mode 100644 detection/configs/litv2/retinanet_litv2_s_fpn_1x_coco_ws_4.py create mode 100644 detection/mm_modules/DCN/deform_conv2d_naive.py create mode 100644 detection/mm_modules/DCN/functions/__init__.py create mode 100644 detection/mm_modules/DCN/functions/deform_conv2d_func.py create mode 100644 detection/mm_modules/DCN/functions/modulated_deform_conv2d_func.py create mode 100644 detection/mm_modules/DCN/make.sh create mode 100644 detection/mm_modules/DCN/make_dsai.sh create mode 100644 detection/mm_modules/DCN/modules/__init__.py create mode 100644 detection/mm_modules/DCN/modules/deform_conv2d.py create mode 100644 detection/mm_modules/DCN/modules/modulated_deform_conv2d.py create mode 100644 detection/mm_modules/DCN/setup.py create mode 100644 detection/mm_modules/DCN/src/cpu/deform_conv2d_cpu.cpp create mode 100644 detection/mm_modules/DCN/src/cpu/deform_conv2d_cpu.h create mode 100644 detection/mm_modules/DCN/src/cpu/modulated_deform_conv2d_cpu.cpp create mode 100644 detection/mm_modules/DCN/src/cpu/modulated_deform_conv2d_cpu.h create mode 100644 detection/mm_modules/DCN/src/cuda/deform_2d_im2col_cuda.cuh create mode 100644 detection/mm_modules/DCN/src/cuda/deform_conv2d_cuda.cu create mode 100644 detection/mm_modules/DCN/src/cuda/deform_conv2d_cuda.h create mode 100644 detection/mm_modules/DCN/src/cuda/modulated_deform_2d_im2col_cuda.cuh create mode 100644 detection/mm_modules/DCN/src/cuda/modulated_deform_conv2d_cuda.cu create mode 100644 detection/mm_modules/DCN/src/cuda/modulated_deform_conv2d_cuda.h create mode 100644 detection/mm_modules/DCN/src/deform_conv2d.h create mode 100644 detection/mm_modules/DCN/src/modulated_deform_conv2d.h create mode 100644 detection/mm_modules/DCN/src/vision.cpp create mode 100644 detection/mm_modules/__init__.py create mode 100644 detection/mm_modules/cocoapi_evaluator.py create mode 100644 detection/mm_modules/distributed_util.py create mode 100644 detection/mm_modules/fp16_utils/README.md create mode 100644 detection/mm_modules/fp16_utils/__init__.py create mode 100644 detection/mm_modules/fp16_utils/fp16_optimizer.py create mode 100644 detection/mm_modules/fp16_utils/fp16util.py create mode 100644 detection/mm_modules/fp16_utils/loss_scaler.py create mode 100644 detection/mm_modules/utils.py create mode 100644 detection/mm_modules/vis_utils.py create mode 100644 detection/mm_modules/voc_evaluator.py create mode 100644 detection/mmcv_custom/__init__.py create mode 100644 detection/mmcv_custom/checkpoint.py create mode 100644 detection/mmcv_custom/runner/__init__.py create mode 100644 detection/mmcv_custom/runner/checkpoint.py create mode 100644 detection/mmcv_custom/runner/epoch_based_runner.py create mode 100644 detection/mmdet/__init__.py create mode 100644 detection/mmdet/apis/__init__.py create mode 100644 detection/mmdet/apis/inference.py create mode 100644 detection/mmdet/apis/test.py create mode 100644 detection/mmdet/apis/train.py create mode 100644 detection/mmdet/core/__init__.py create mode 100644 detection/mmdet/core/anchor/__init__.py create mode 100644 detection/mmdet/core/anchor/anchor_generator.py create mode 100644 detection/mmdet/core/anchor/builder.py create mode 100644 detection/mmdet/core/anchor/point_generator.py create mode 100644 detection/mmdet/core/anchor/utils.py create mode 100644 detection/mmdet/core/bbox/__init__.py create mode 100644 detection/mmdet/core/bbox/assigners/__init__.py create mode 100644 detection/mmdet/core/bbox/assigners/approx_max_iou_assigner.py create mode 100644 detection/mmdet/core/bbox/assigners/assign_result.py create mode 100644 detection/mmdet/core/bbox/assigners/atss_assigner.py create mode 100644 detection/mmdet/core/bbox/assigners/base_assigner.py create mode 100644 detection/mmdet/core/bbox/assigners/center_region_assigner.py create mode 100644 detection/mmdet/core/bbox/assigners/grid_assigner.py create mode 100644 detection/mmdet/core/bbox/assigners/hungarian_assigner.py create mode 100644 detection/mmdet/core/bbox/assigners/max_iou_assigner.py create mode 100644 detection/mmdet/core/bbox/assigners/point_assigner.py create mode 100644 detection/mmdet/core/bbox/assigners/region_assigner.py create mode 100644 detection/mmdet/core/bbox/builder.py create mode 100644 detection/mmdet/core/bbox/coder/__init__.py create mode 100644 detection/mmdet/core/bbox/coder/base_bbox_coder.py create mode 100644 detection/mmdet/core/bbox/coder/bucketing_bbox_coder.py create mode 100644 detection/mmdet/core/bbox/coder/delta_xywh_bbox_coder.py create mode 100644 detection/mmdet/core/bbox/coder/legacy_delta_xywh_bbox_coder.py create mode 100644 detection/mmdet/core/bbox/coder/pseudo_bbox_coder.py create mode 100644 detection/mmdet/core/bbox/coder/tblr_bbox_coder.py create mode 100644 detection/mmdet/core/bbox/coder/yolo_bbox_coder.py create mode 100644 detection/mmdet/core/bbox/demodata.py create mode 100644 detection/mmdet/core/bbox/iou_calculators/__init__.py create mode 100644 detection/mmdet/core/bbox/iou_calculators/builder.py create mode 100644 detection/mmdet/core/bbox/iou_calculators/iou2d_calculator.py create mode 100644 detection/mmdet/core/bbox/match_costs/__init__.py create mode 100644 detection/mmdet/core/bbox/match_costs/builder.py create mode 100644 detection/mmdet/core/bbox/match_costs/match_cost.py create mode 100644 detection/mmdet/core/bbox/samplers/__init__.py create mode 100644 detection/mmdet/core/bbox/samplers/base_sampler.py create mode 100644 detection/mmdet/core/bbox/samplers/combined_sampler.py create mode 100644 detection/mmdet/core/bbox/samplers/instance_balanced_pos_sampler.py create mode 100644 detection/mmdet/core/bbox/samplers/iou_balanced_neg_sampler.py create mode 100644 detection/mmdet/core/bbox/samplers/ohem_sampler.py create mode 100644 detection/mmdet/core/bbox/samplers/pseudo_sampler.py create mode 100644 detection/mmdet/core/bbox/samplers/random_sampler.py create mode 100644 detection/mmdet/core/bbox/samplers/sampling_result.py create mode 100644 detection/mmdet/core/bbox/samplers/score_hlr_sampler.py create mode 100644 detection/mmdet/core/bbox/transforms.py create mode 100644 detection/mmdet/core/evaluation/__init__.py create mode 100644 detection/mmdet/core/evaluation/bbox_overlaps.py create mode 100644 detection/mmdet/core/evaluation/class_names.py create mode 100644 detection/mmdet/core/evaluation/eval_hooks.py create mode 100644 detection/mmdet/core/evaluation/mean_ap.py create mode 100644 detection/mmdet/core/evaluation/recall.py create mode 100644 detection/mmdet/core/export/__init__.py create mode 100644 detection/mmdet/core/export/pytorch2onnx.py create mode 100644 detection/mmdet/core/mask/__init__.py create mode 100644 detection/mmdet/core/mask/mask_target.py create mode 100644 detection/mmdet/core/mask/structures.py create mode 100644 detection/mmdet/core/mask/utils.py create mode 100644 detection/mmdet/core/post_processing/__init__.py create mode 100644 detection/mmdet/core/post_processing/bbox_nms.py create mode 100644 detection/mmdet/core/post_processing/merge_augs.py create mode 100644 detection/mmdet/core/utils/__init__.py create mode 100644 detection/mmdet/core/utils/dist_utils.py create mode 100644 detection/mmdet/core/utils/misc.py create mode 100644 detection/mmdet/core/visualization/__init__.py create mode 100644 detection/mmdet/core/visualization/image.py create mode 100644 detection/mmdet/datasets/__init__.py create mode 100644 detection/mmdet/datasets/builder.py create mode 100644 detection/mmdet/datasets/cityscapes.py create mode 100644 detection/mmdet/datasets/coco.py create mode 100644 detection/mmdet/datasets/custom.py create mode 100644 detection/mmdet/datasets/dataset_wrappers.py create mode 100644 detection/mmdet/datasets/deepfashion.py create mode 100644 detection/mmdet/datasets/lvis.py create mode 100644 detection/mmdet/datasets/pipelines/__init__.py create mode 100644 detection/mmdet/datasets/pipelines/auto_augment.py create mode 100644 detection/mmdet/datasets/pipelines/compose.py create mode 100644 detection/mmdet/datasets/pipelines/formating.py create mode 100644 detection/mmdet/datasets/pipelines/instaboost.py create mode 100644 detection/mmdet/datasets/pipelines/loading.py create mode 100644 detection/mmdet/datasets/pipelines/test_time_aug.py create mode 100644 detection/mmdet/datasets/pipelines/transforms.py create mode 100644 detection/mmdet/datasets/samplers/__init__.py create mode 100644 detection/mmdet/datasets/samplers/distributed_sampler.py create mode 100644 detection/mmdet/datasets/samplers/group_sampler.py create mode 100644 detection/mmdet/datasets/utils.py create mode 100644 detection/mmdet/datasets/voc.py create mode 100644 detection/mmdet/datasets/wider_face.py create mode 100644 detection/mmdet/datasets/xml_style.py create mode 100644 detection/mmdet/models/__init__.py create mode 100644 detection/mmdet/models/backbones/__init__.py create mode 100644 detection/mmdet/models/backbones/darknet.py create mode 100644 detection/mmdet/models/backbones/detectors_resnet.py create mode 100644 detection/mmdet/models/backbones/detectors_resnext.py create mode 100644 detection/mmdet/models/backbones/hourglass.py create mode 100644 detection/mmdet/models/backbones/hrnet.py create mode 100644 detection/mmdet/models/backbones/litv2.py create mode 100644 detection/mmdet/models/backbones/regnet.py create mode 100644 detection/mmdet/models/backbones/res2net.py create mode 100644 detection/mmdet/models/backbones/resnest.py create mode 100644 detection/mmdet/models/backbones/resnet.py create mode 100644 detection/mmdet/models/backbones/resnext.py create mode 100644 detection/mmdet/models/backbones/ssd_vgg.py create mode 100644 detection/mmdet/models/backbones/trident_resnet.py create mode 100644 detection/mmdet/models/builder.py create mode 100644 detection/mmdet/models/dense_heads/__init__.py create mode 100644 detection/mmdet/models/dense_heads/anchor_free_head.py create mode 100644 detection/mmdet/models/dense_heads/anchor_head.py create mode 100644 detection/mmdet/models/dense_heads/atss_head.py create mode 100644 detection/mmdet/models/dense_heads/base_dense_head.py create mode 100644 detection/mmdet/models/dense_heads/cascade_rpn_head.py create mode 100644 detection/mmdet/models/dense_heads/centripetal_head.py create mode 100644 detection/mmdet/models/dense_heads/corner_head.py create mode 100644 detection/mmdet/models/dense_heads/dense_test_mixins.py create mode 100644 detection/mmdet/models/dense_heads/embedding_rpn_head.py create mode 100644 detection/mmdet/models/dense_heads/fcos_head.py create mode 100644 detection/mmdet/models/dense_heads/fovea_head.py create mode 100644 detection/mmdet/models/dense_heads/free_anchor_retina_head.py create mode 100644 detection/mmdet/models/dense_heads/fsaf_head.py create mode 100644 detection/mmdet/models/dense_heads/ga_retina_head.py create mode 100644 detection/mmdet/models/dense_heads/ga_rpn_head.py create mode 100644 detection/mmdet/models/dense_heads/gfl_head.py create mode 100644 detection/mmdet/models/dense_heads/guided_anchor_head.py create mode 100644 detection/mmdet/models/dense_heads/ld_head.py create mode 100644 detection/mmdet/models/dense_heads/nasfcos_head.py create mode 100644 detection/mmdet/models/dense_heads/paa_head.py create mode 100644 detection/mmdet/models/dense_heads/pisa_retinanet_head.py create mode 100644 detection/mmdet/models/dense_heads/pisa_ssd_head.py create mode 100644 detection/mmdet/models/dense_heads/reppoints_head.py create mode 100644 detection/mmdet/models/dense_heads/retina_head.py create mode 100644 detection/mmdet/models/dense_heads/retina_sepbn_head.py create mode 100644 detection/mmdet/models/dense_heads/rpn_head.py create mode 100644 detection/mmdet/models/dense_heads/rpn_test_mixin.py create mode 100644 detection/mmdet/models/dense_heads/sabl_retina_head.py create mode 100644 detection/mmdet/models/dense_heads/ssd_head.py create mode 100644 detection/mmdet/models/dense_heads/transformer_head.py create mode 100644 detection/mmdet/models/dense_heads/vfnet_head.py create mode 100644 detection/mmdet/models/dense_heads/yolact_head.py create mode 100644 detection/mmdet/models/dense_heads/yolo_head.py create mode 100644 detection/mmdet/models/detectors/__init__.py create mode 100644 detection/mmdet/models/detectors/atss.py create mode 100644 detection/mmdet/models/detectors/base.py create mode 100644 detection/mmdet/models/detectors/cascade_rcnn.py create mode 100644 detection/mmdet/models/detectors/cornernet.py create mode 100644 detection/mmdet/models/detectors/detr.py create mode 100644 detection/mmdet/models/detectors/fast_rcnn.py create mode 100644 detection/mmdet/models/detectors/faster_rcnn.py create mode 100644 detection/mmdet/models/detectors/fcos.py create mode 100644 detection/mmdet/models/detectors/fovea.py create mode 100644 detection/mmdet/models/detectors/fsaf.py create mode 100644 detection/mmdet/models/detectors/gfl.py create mode 100644 detection/mmdet/models/detectors/grid_rcnn.py create mode 100644 detection/mmdet/models/detectors/htc.py create mode 100644 detection/mmdet/models/detectors/kd_one_stage.py create mode 100644 detection/mmdet/models/detectors/mask_rcnn.py create mode 100644 detection/mmdet/models/detectors/mask_scoring_rcnn.py create mode 100644 detection/mmdet/models/detectors/nasfcos.py create mode 100644 detection/mmdet/models/detectors/paa.py create mode 100644 detection/mmdet/models/detectors/point_rend.py create mode 100644 detection/mmdet/models/detectors/reppoints_detector.py create mode 100644 detection/mmdet/models/detectors/retinanet.py create mode 100644 detection/mmdet/models/detectors/rpn.py create mode 100644 detection/mmdet/models/detectors/scnet.py create mode 100644 detection/mmdet/models/detectors/single_stage.py create mode 100644 detection/mmdet/models/detectors/sparse_rcnn.py create mode 100644 detection/mmdet/models/detectors/trident_faster_rcnn.py create mode 100644 detection/mmdet/models/detectors/two_stage.py create mode 100644 detection/mmdet/models/detectors/vfnet.py create mode 100644 detection/mmdet/models/detectors/yolact.py create mode 100644 detection/mmdet/models/detectors/yolo.py create mode 100644 detection/mmdet/models/losses/__init__.py create mode 100644 detection/mmdet/models/losses/accuracy.py create mode 100644 detection/mmdet/models/losses/ae_loss.py create mode 100644 detection/mmdet/models/losses/balanced_l1_loss.py create mode 100644 detection/mmdet/models/losses/cross_entropy_loss.py create mode 100644 detection/mmdet/models/losses/focal_loss.py create mode 100644 detection/mmdet/models/losses/gaussian_focal_loss.py create mode 100644 detection/mmdet/models/losses/gfocal_loss.py create mode 100644 detection/mmdet/models/losses/ghm_loss.py create mode 100644 detection/mmdet/models/losses/iou_loss.py create mode 100644 detection/mmdet/models/losses/kd_loss.py create mode 100644 detection/mmdet/models/losses/mse_loss.py create mode 100644 detection/mmdet/models/losses/pisa_loss.py create mode 100644 detection/mmdet/models/losses/smooth_l1_loss.py create mode 100644 detection/mmdet/models/losses/utils.py create mode 100644 detection/mmdet/models/losses/varifocal_loss.py create mode 100644 detection/mmdet/models/necks/__init__.py create mode 100644 detection/mmdet/models/necks/bfp.py create mode 100644 detection/mmdet/models/necks/channel_mapper.py create mode 100644 detection/mmdet/models/necks/fpg.py create mode 100644 detection/mmdet/models/necks/fpn.py create mode 100644 detection/mmdet/models/necks/fpn_carafe.py create mode 100644 detection/mmdet/models/necks/hrfpn.py create mode 100644 detection/mmdet/models/necks/nas_fpn.py create mode 100644 detection/mmdet/models/necks/nasfcos_fpn.py create mode 100644 detection/mmdet/models/necks/pafpn.py create mode 100644 detection/mmdet/models/necks/rfp.py create mode 100644 detection/mmdet/models/necks/yolo_neck.py create mode 100644 detection/mmdet/models/roi_heads/__init__.py create mode 100644 detection/mmdet/models/roi_heads/base_roi_head.py create mode 100644 detection/mmdet/models/roi_heads/bbox_heads/__init__.py create mode 100644 detection/mmdet/models/roi_heads/bbox_heads/bbox_head.py create mode 100644 detection/mmdet/models/roi_heads/bbox_heads/convfc_bbox_head.py create mode 100644 detection/mmdet/models/roi_heads/bbox_heads/dii_head.py create mode 100644 detection/mmdet/models/roi_heads/bbox_heads/double_bbox_head.py create mode 100644 detection/mmdet/models/roi_heads/bbox_heads/sabl_head.py create mode 100644 detection/mmdet/models/roi_heads/bbox_heads/scnet_bbox_head.py create mode 100644 detection/mmdet/models/roi_heads/cascade_roi_head.py create mode 100644 detection/mmdet/models/roi_heads/double_roi_head.py create mode 100644 detection/mmdet/models/roi_heads/dynamic_roi_head.py create mode 100644 detection/mmdet/models/roi_heads/grid_roi_head.py create mode 100644 detection/mmdet/models/roi_heads/htc_roi_head.py create mode 100644 detection/mmdet/models/roi_heads/mask_heads/__init__.py create mode 100644 detection/mmdet/models/roi_heads/mask_heads/coarse_mask_head.py create mode 100644 detection/mmdet/models/roi_heads/mask_heads/fcn_mask_head.py create mode 100644 detection/mmdet/models/roi_heads/mask_heads/feature_relay_head.py create mode 100644 detection/mmdet/models/roi_heads/mask_heads/fused_semantic_head.py create mode 100644 detection/mmdet/models/roi_heads/mask_heads/global_context_head.py create mode 100644 detection/mmdet/models/roi_heads/mask_heads/grid_head.py create mode 100644 detection/mmdet/models/roi_heads/mask_heads/htc_mask_head.py create mode 100644 detection/mmdet/models/roi_heads/mask_heads/mask_point_head.py create mode 100644 detection/mmdet/models/roi_heads/mask_heads/maskiou_head.py create mode 100644 detection/mmdet/models/roi_heads/mask_heads/scnet_mask_head.py create mode 100644 detection/mmdet/models/roi_heads/mask_heads/scnet_semantic_head.py create mode 100644 detection/mmdet/models/roi_heads/mask_scoring_roi_head.py create mode 100644 detection/mmdet/models/roi_heads/pisa_roi_head.py create mode 100644 detection/mmdet/models/roi_heads/point_rend_roi_head.py create mode 100644 detection/mmdet/models/roi_heads/roi_extractors/__init__.py create mode 100644 detection/mmdet/models/roi_heads/roi_extractors/base_roi_extractor.py create mode 100644 detection/mmdet/models/roi_heads/roi_extractors/generic_roi_extractor.py create mode 100644 detection/mmdet/models/roi_heads/roi_extractors/single_level_roi_extractor.py create mode 100644 detection/mmdet/models/roi_heads/scnet_roi_head.py create mode 100644 detection/mmdet/models/roi_heads/shared_heads/__init__.py create mode 100644 detection/mmdet/models/roi_heads/shared_heads/res_layer.py create mode 100644 detection/mmdet/models/roi_heads/sparse_roi_head.py create mode 100644 detection/mmdet/models/roi_heads/standard_roi_head.py create mode 100644 detection/mmdet/models/roi_heads/test_mixins.py create mode 100644 detection/mmdet/models/roi_heads/trident_roi_head.py create mode 100644 detection/mmdet/models/utils/__init__.py create mode 100644 detection/mmdet/models/utils/builder.py create mode 100644 detection/mmdet/models/utils/gaussian_target.py create mode 100644 detection/mmdet/models/utils/positional_encoding.py create mode 100644 detection/mmdet/models/utils/res_layer.py create mode 100644 detection/mmdet/models/utils/transformer.py create mode 100644 detection/mmdet/utils/__init__.py create mode 100644 detection/mmdet/utils/collect_env.py create mode 100644 detection/mmdet/utils/contextmanagers.py create mode 100644 detection/mmdet/utils/logger.py create mode 100644 detection/mmdet/utils/optimizer.py create mode 100644 detection/mmdet/utils/profiling.py create mode 100644 detection/mmdet/utils/util_mixins.py create mode 100644 detection/mmdet/utils/util_random.py create mode 100644 detection/mmdet/version.py create mode 100644 detection/requirements.txt create mode 100644 detection/setup.cfg create mode 100644 detection/setup.py create mode 100644 detection/tools/benchmark.py create mode 100644 detection/tools/dist_test.sh create mode 100644 detection/tools/dist_train.sh create mode 100644 detection/tools/get_flops.py create mode 100644 detection/tools/test.py create mode 100644 detection/tools/train.py create mode 100644 segmentation/.github/CODE_OF_CONDUCT.md create mode 100644 segmentation/.github/CONTRIBUTING.md create mode 100644 segmentation/.github/ISSUE_TEMPLATE/config.yml create mode 100644 segmentation/.github/ISSUE_TEMPLATE/error-report.md create mode 100644 segmentation/.github/ISSUE_TEMPLATE/feature_request.md create mode 100644 segmentation/.github/ISSUE_TEMPLATE/general_questions.md create mode 100644 segmentation/.gitignore create mode 100644 segmentation/LICENSE create mode 100644 segmentation/README.md create mode 100644 segmentation/configs/_base_/datasets/ade20k.py create mode 100644 segmentation/configs/_base_/default_runtime.py create mode 100644 segmentation/configs/_base_/models/fpn_r50_lit.py create mode 100644 segmentation/configs/_base_/schedules/schedule_160k.py create mode 100644 segmentation/configs/_base_/schedules/schedule_20k.py create mode 100644 segmentation/configs/_base_/schedules/schedule_40k.py create mode 100644 segmentation/configs/_base_/schedules/schedule_80k.py create mode 100644 segmentation/configs/litv2/litv2_b_fpn_r50_512x512_80k_ade20k.py create mode 100644 segmentation/configs/litv2/litv2_m_fpn_r50_512x512_80k_ade20k.py create mode 100644 segmentation/configs/litv2/litv2_s_fpn_r50_512x512_80k_ade20k.py create mode 100644 segmentation/mm_modules/DCN/deform_conv2d_naive.py create mode 100644 segmentation/mm_modules/DCN/functions/__init__.py create mode 100644 segmentation/mm_modules/DCN/functions/deform_conv2d_func.py create mode 100644 segmentation/mm_modules/DCN/functions/modulated_deform_conv2d_func.py create mode 100644 segmentation/mm_modules/DCN/make.sh create mode 100644 segmentation/mm_modules/DCN/make_dsai.sh create mode 100644 segmentation/mm_modules/DCN/modules/__init__.py create mode 100644 segmentation/mm_modules/DCN/modules/deform_conv2d.py create mode 100644 segmentation/mm_modules/DCN/modules/modulated_deform_conv2d.py create mode 100644 segmentation/mm_modules/DCN/setup.py create mode 100644 segmentation/mm_modules/DCN/src/cpu/deform_conv2d_cpu.cpp create mode 100644 segmentation/mm_modules/DCN/src/cpu/deform_conv2d_cpu.h create mode 100644 segmentation/mm_modules/DCN/src/cpu/modulated_deform_conv2d_cpu.cpp create mode 100644 segmentation/mm_modules/DCN/src/cpu/modulated_deform_conv2d_cpu.h create mode 100644 segmentation/mm_modules/DCN/src/cuda/deform_2d_im2col_cuda.cuh create mode 100644 segmentation/mm_modules/DCN/src/cuda/deform_conv2d_cuda.cu create mode 100644 segmentation/mm_modules/DCN/src/cuda/deform_conv2d_cuda.h create mode 100644 segmentation/mm_modules/DCN/src/cuda/modulated_deform_2d_im2col_cuda.cuh create mode 100644 segmentation/mm_modules/DCN/src/cuda/modulated_deform_conv2d_cuda.cu create mode 100644 segmentation/mm_modules/DCN/src/cuda/modulated_deform_conv2d_cuda.h create mode 100644 segmentation/mm_modules/DCN/src/deform_conv2d.h create mode 100644 segmentation/mm_modules/DCN/src/modulated_deform_conv2d.h create mode 100644 segmentation/mm_modules/DCN/src/vision.cpp create mode 100644 segmentation/mm_modules/__init__.py create mode 100644 segmentation/mm_modules/cocoapi_evaluator.py create mode 100644 segmentation/mm_modules/distributed_util.py create mode 100644 segmentation/mm_modules/fp16_utils/README.md create mode 100644 segmentation/mm_modules/fp16_utils/__init__.py create mode 100644 segmentation/mm_modules/fp16_utils/fp16_optimizer.py create mode 100644 segmentation/mm_modules/fp16_utils/fp16util.py create mode 100644 segmentation/mm_modules/fp16_utils/loss_scaler.py create mode 100644 segmentation/mm_modules/utils.py create mode 100644 segmentation/mm_modules/vis_utils.py create mode 100644 segmentation/mm_modules/voc_evaluator.py create mode 100644 segmentation/mmcv_custom/__init__.py create mode 100644 segmentation/mmcv_custom/checkpoint.py create mode 100644 segmentation/mmcv_custom/runner/__init__.py create mode 100644 segmentation/mmcv_custom/runner/checkpoint.py create mode 100644 segmentation/mmcv_custom/runner/epoch_based_runner.py create mode 100644 segmentation/mmseg/__init__.py create mode 100644 segmentation/mmseg/apis/__init__.py create mode 100644 segmentation/mmseg/apis/inference.py create mode 100644 segmentation/mmseg/apis/test.py create mode 100644 segmentation/mmseg/apis/train.py create mode 100644 segmentation/mmseg/core/__init__.py create mode 100644 segmentation/mmseg/core/evaluation/__init__.py create mode 100644 segmentation/mmseg/core/evaluation/class_names.py create mode 100644 segmentation/mmseg/core/evaluation/eval_hooks.py create mode 100644 segmentation/mmseg/core/evaluation/metrics.py create mode 100644 segmentation/mmseg/core/seg/__init__.py create mode 100644 segmentation/mmseg/core/seg/builder.py create mode 100644 segmentation/mmseg/core/seg/sampler/__init__.py create mode 100644 segmentation/mmseg/core/seg/sampler/base_pixel_sampler.py create mode 100644 segmentation/mmseg/core/seg/sampler/ohem_pixel_sampler.py create mode 100644 segmentation/mmseg/core/utils/__init__.py create mode 100644 segmentation/mmseg/core/utils/misc.py create mode 100644 segmentation/mmseg/datasets/__init__.py create mode 100644 segmentation/mmseg/datasets/ade.py create mode 100644 segmentation/mmseg/datasets/builder.py create mode 100644 segmentation/mmseg/datasets/chase_db1.py create mode 100644 segmentation/mmseg/datasets/cityscapes.py create mode 100644 segmentation/mmseg/datasets/custom.py create mode 100644 segmentation/mmseg/datasets/dataset_wrappers.py create mode 100644 segmentation/mmseg/datasets/drive.py create mode 100644 segmentation/mmseg/datasets/hrf.py create mode 100644 segmentation/mmseg/datasets/pascal_context.py create mode 100644 segmentation/mmseg/datasets/pipelines/__init__.py create mode 100644 segmentation/mmseg/datasets/pipelines/compose.py create mode 100644 segmentation/mmseg/datasets/pipelines/formating.py create mode 100644 segmentation/mmseg/datasets/pipelines/loading.py create mode 100644 segmentation/mmseg/datasets/pipelines/test_time_aug.py create mode 100644 segmentation/mmseg/datasets/pipelines/transforms.py create mode 100644 segmentation/mmseg/datasets/stare.py create mode 100644 segmentation/mmseg/datasets/voc.py create mode 100644 segmentation/mmseg/models/__init__.py create mode 100644 segmentation/mmseg/models/backbones/__init__.py create mode 100644 segmentation/mmseg/models/backbones/cgnet.py create mode 100644 segmentation/mmseg/models/backbones/fast_scnn.py create mode 100644 segmentation/mmseg/models/backbones/hrnet.py create mode 100644 segmentation/mmseg/models/backbones/litv2.py create mode 100644 segmentation/mmseg/models/backbones/mobilenet_v2.py create mode 100644 segmentation/mmseg/models/backbones/mobilenet_v3.py create mode 100644 segmentation/mmseg/models/backbones/resnest.py create mode 100644 segmentation/mmseg/models/backbones/resnet.py create mode 100644 segmentation/mmseg/models/backbones/resnext.py create mode 100644 segmentation/mmseg/models/backbones/unet.py create mode 100644 segmentation/mmseg/models/backbones/vit.py create mode 100644 segmentation/mmseg/models/builder.py create mode 100644 segmentation/mmseg/models/decode_heads/__init__.py create mode 100644 segmentation/mmseg/models/decode_heads/ann_head.py create mode 100644 segmentation/mmseg/models/decode_heads/apc_head.py create mode 100644 segmentation/mmseg/models/decode_heads/aspp_head.py create mode 100644 segmentation/mmseg/models/decode_heads/cascade_decode_head.py create mode 100644 segmentation/mmseg/models/decode_heads/cc_head.py create mode 100644 segmentation/mmseg/models/decode_heads/da_head.py create mode 100644 segmentation/mmseg/models/decode_heads/decode_head.py create mode 100644 segmentation/mmseg/models/decode_heads/dm_head.py create mode 100644 segmentation/mmseg/models/decode_heads/dnl_head.py create mode 100644 segmentation/mmseg/models/decode_heads/ema_head.py create mode 100644 segmentation/mmseg/models/decode_heads/enc_head.py create mode 100644 segmentation/mmseg/models/decode_heads/fcn_head.py create mode 100644 segmentation/mmseg/models/decode_heads/fpn_head.py create mode 100644 segmentation/mmseg/models/decode_heads/gc_head.py create mode 100644 segmentation/mmseg/models/decode_heads/lraspp_head.py create mode 100644 segmentation/mmseg/models/decode_heads/nl_head.py create mode 100644 segmentation/mmseg/models/decode_heads/ocr_head.py create mode 100644 segmentation/mmseg/models/decode_heads/point_head.py create mode 100644 segmentation/mmseg/models/decode_heads/psa_head.py create mode 100644 segmentation/mmseg/models/decode_heads/psp_head.py create mode 100644 segmentation/mmseg/models/decode_heads/sep_aspp_head.py create mode 100644 segmentation/mmseg/models/decode_heads/sep_fcn_head.py create mode 100644 segmentation/mmseg/models/decode_heads/uper_head.py create mode 100644 segmentation/mmseg/models/losses/__init__.py create mode 100644 segmentation/mmseg/models/losses/accuracy.py create mode 100644 segmentation/mmseg/models/losses/cross_entropy_loss.py create mode 100644 segmentation/mmseg/models/losses/dice_loss.py create mode 100644 segmentation/mmseg/models/losses/lovasz_loss.py create mode 100644 segmentation/mmseg/models/losses/utils.py create mode 100644 segmentation/mmseg/models/necks/__init__.py create mode 100644 segmentation/mmseg/models/necks/fpn.py create mode 100644 segmentation/mmseg/models/necks/multilevel_neck.py create mode 100644 segmentation/mmseg/models/segmentors/__init__.py create mode 100644 segmentation/mmseg/models/segmentors/base.py create mode 100644 segmentation/mmseg/models/segmentors/cascade_encoder_decoder.py create mode 100644 segmentation/mmseg/models/segmentors/encoder_decoder.py create mode 100644 segmentation/mmseg/models/utils/__init__.py create mode 100644 segmentation/mmseg/models/utils/drop.py create mode 100644 segmentation/mmseg/models/utils/inverted_residual.py create mode 100644 segmentation/mmseg/models/utils/make_divisible.py create mode 100644 segmentation/mmseg/models/utils/res_layer.py create mode 100644 segmentation/mmseg/models/utils/se_layer.py create mode 100644 segmentation/mmseg/models/utils/self_attention_block.py create mode 100644 segmentation/mmseg/models/utils/up_conv_block.py create mode 100644 segmentation/mmseg/models/utils/weight_init.py create mode 100644 segmentation/mmseg/ops/__init__.py create mode 100644 segmentation/mmseg/ops/encoding.py create mode 100644 segmentation/mmseg/ops/wrappers.py create mode 100644 segmentation/mmseg/utils/__init__.py create mode 100644 segmentation/mmseg/utils/collect_env.py create mode 100644 segmentation/mmseg/utils/logger.py create mode 100644 segmentation/mmseg/version.py create mode 100644 segmentation/requirements.txt create mode 100644 segmentation/requirements/docs.txt create mode 100644 segmentation/requirements/optional.txt create mode 100644 segmentation/requirements/readthedocs.txt create mode 100644 segmentation/requirements/runtime.txt create mode 100644 segmentation/requirements/tests.txt create mode 100644 segmentation/resources/mmseg-logo.png create mode 100644 segmentation/resources/seg_demo.gif create mode 100644 segmentation/setup.cfg create mode 100644 segmentation/setup.py create mode 100644 segmentation/tools/benchmark.py create mode 100644 segmentation/tools/dist_test.sh create mode 100644 segmentation/tools/dist_train.sh create mode 100644 segmentation/tools/get_flops.py create mode 100644 segmentation/tools/test.py create mode 100644 segmentation/tools/train.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6f0080a --- /dev/null +++ b/.gitignore @@ -0,0 +1,141 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +./outputs +outputs/* +figures/* +plots/* +.idea/* +checkpoints/ +*.out +*.err +.vscode/ +*.xml +*.pth \ No newline at end of file diff --git a/README.md b/README.md index 1e74110..936d833 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,15 @@ -# Fast Vision Transformers with HiLo Attention -Official PyTorch implementation of [Fast Vision Transformers with HiLo Attention](https://arxiv.org/abs/2205.13213). +# Fast Vision Transformers with HiLo Attention👋 +[![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) + + +This is the official PyTorch implementation of [Fast Vision Transformers with HiLo Attention](https://arxiv.org/abs/2205.13213). By [Zizheng Pan](https://scholar.google.com.au/citations?user=w_VMopoAAAAJ&hl=en), [Jianfei Cai](https://scholar.google.com/citations?user=N6czCoUAAAAJ&hl=en), and [Bohan Zhuang](https://scholar.google.com.au/citations?user=DFuDBBwAAAAJ). + + +## A Gentle Introduction + ![hilo](.github/arch.png) @@ -10,75 +17,154 @@ We introduce LITv2, a simple and effective ViT which performs favourably against ![hilo](.github/hilo.png) +The core of LITv2: **HiLo attention** HiLo is inspired by the insight that high frequencies in an image capture local fine details and low frequencies focus on global structures, whereas a multi-head self-attention layer neglects the characteristic of different frequencies. Therefore, we propose to disentangle the high/low frequency patterns in an attention layer by separating the heads into two groups, where one group encodes high frequencies via self-attention within each local window, and another group performs the attention to model the global relationship between the average-pooled low-frequency keys from each window and each query position in the input feature map. + + + +## News + +- **16/06/2022.** We release the source code for classification/detection/segmentation, along with the pretrained weights. Any issues are welcomed! + + + +## Installation + +### Requirements + +- Linux with Python ≥ 3.6 +- PyTorch 1.8.1 +- CUDA 11.1 +- An NVIDIA GPU + +### Conda environment setup + +**Note**: You can use the same environment to debug [LITv1](https://github.com/ziplab/LIT). Otherwise, you can create a new python virtual environment by the following script. + +```bash +conda create -n lit python=3.7 +conda activate lit + +# Install Pytorch and TorchVision +pip install torch==1.8.1+cu111 torchvision==0.9.1+cu111 torchaudio==0.8.1 -f https://download.pytorch.org/whl/torch_stable.html + +pip install timm==0.3.2 +pip install ninja +pip install tensorboard + +# Install NVIDIA apex +git clone https://github.com/NVIDIA/apex +cd apex +pip install -v --disable-pip-version-check --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" ./ +cd ../ +rm -rf apex/ + +# Build Deformable Convolution +cd mm_modules/DCN +python setup.py build install + +pip install opencv-python==4.4.0.46 termcolor==1.1.0 yacs==0.1.8 +``` + + + +# Getting Started + +For image classification on ImageNet, please refer to [classification](https://github.com/ziplab/LITv2/tree/main/classification). + +For object detection on COCO 2017, please refer to [detection](https://github.com/ziplab/LITv2/tree/main/detection). + +For semantic segmentation on ADE20K, please refer to [segmentation](https://github.com/ziplab/LITv2/tree/main/segmentation). + + -The core of LITv2: **HiLo attention**. HiLo is inspired by the insight that high frequencies in an image capture local fine details and low frequencies focus on global structures, whereas a multi-head self-attention layer neglects the characteristic of different frequencies. Therefore, we propose to disentangle the high/low frequency patterns in an attention layer by separating the heads into two groups, where one group encodes high frequencies via self-attention within each local window, and another group performs the attention to model the global relationship between the average-pooled low-frequency keys from each window and each query position in the input feature map. +## Results and Model Zoo +**Note:** For your convenience, you can download find all models and logs from [Google Drive](https://drive.google.com/drive/folders/1VAtrPWEqxi-6q6luwEVdYvkBYedApwbU?usp=sharing) (4.8G in total). Alternatively, we also provide download links from github. +### Image Classification on ImageNet-1K +All models are trained with 300 epochs with a total batch size of 1024 on 8 V100 GPUs. +| Model | Resolution | Params (M) | FLOPs (G) | Throughput (imgs/s) | Train Mem (GB) | Test Mem (GB) | Top-1 (%) | Download | +| ------- | ---------- | ---------- | --------- | ------------------- | -------------- | ------------- | --------- | ----------- | +| LITv2-S | 224 | 28 | 3.7 | 1,471 | 5.1 | 1.2 | 82.0 | model & log | +| LITv2-M | 224 | 49 | 7.5 | 812 | 8.8 | 1.4 | 83.3 | model & log | +| LITv2-B | 224 | 87 | 13.2 | 602 | 12.2 | 2.1 | 83.6 | model & log | +| LITv2-B | 384 | 87 | 39.7 | 198 | 35.8 | 4.6 | 84.7 | model | -## Usage +> By default, the throughput and memory footprint are tested on one RTX 3090 based on a batch size of 64. Memory is measured by the peak memory usage with `torch.cuda.max_memory_allocated()`. Throughput is averaged over 30 runs. -Code and pretrained weights will be released soon. +### Object Detection on COCO 2017 +All models are trained with 1x schedule (12 epochs) with a total batch size of 16 on 8 V100 GPUs. +#### RetinaNet -## Image Classification on ImageNet-1K +| Backbone | Window Size | Params (M) | FLOPs (G) | FPS | box AP | Config | Download | +| -------- | ----------- | ---------- | --------- | ---- | ------ | ------ | ----------- | +| LITv2-S | 2 | 38 | 242 | 18.7 | 44.0 | link | model & log | +| LITv2-S | 4 | 38 | 230 | 20.4 | 43.7 | link | model & log | +| LITv2-M | 2 | 59 | 348 | 12.2 | 46.0 | link | model & log | +| LITv2-M | 4 | 59 | 312 | 14.8 | 45.8 | link | model & log | +| LITv2-B | 2 | 97 | 481 | 9.5 | 46.7 | link | model & log | +| LITv2-B | 4 | 97 | 430 | 11.8 | 46.3 | link | model & log | -| Model | Resolution | Params (M) | FLOPs (G) | Throughput (imgs/s) | Train Mem (GB) | Test Mem (GB) | Top-1 (%) | -| ------- | ---------- | ---------- | --------- | ------------------- | -------------- | ------------- | --------- | -| LITv2-S | 224 | 28 | 3.7 | 1,471 | 5.1 | 1.2 | 82.0 | -| LITv2-M | 224 | 49 | 7.5 | 812 | 8.8 | 1.4 | 83.3 | -| LITv2-B | 224 | 87 | 13.2 | 602 | 12.2 | 2.1 | 83.6 | -| LITv2-B | 384 | 87 | 39.7 | 198 | 35.8 | 4.6 | 84.7 | +#### Mask R-CNN -> Throughput and memory footprint are tested on one RTX 3090 based on a batch size of 64. Memory is measured by the peak memory usage with `torch.cuda.max_memory_allocated()`. +| Backbone | Window Size | Params (M) | FLOPs (G) | FPS | box AP | mask AP | Config | Download | +| -------- | ----------- | ---------- | --------- | ---- | ------ | ------- | ------ | ----------- | +| LITv2-S | 2 | 47 | 261 | 18.7 | 44.9 | 40.8 | link | model & log | +| LITv2-S | 4 | 47 | 249 | 21.9 | 44.7 | 40.7 | link | model & log | +| LITv2-M | 2 | 68 | 367 | 12.6 | 46.8 | 42.3 | link | model & log | +| LITv2-M | 4 | 68 | 315 | 16.0 | 46.5 | 42.0 | link | model & log | +| LITv2-B | 2 | 106 | 500 | 9.3 | 47.3 | 42.6 | link | model & log | +| LITv2-B | 4 | 106 | 449 | 11.5 | 46.8 | 42.3 | link | model & log | -## Object Detection on COCO +### Semantic Segmentation on ADE20K -### RetinaNet +All models are trained with 80K iterations with a total batch size of 16 on 8 V100 GPUs. -| Backbone | Window Size | Params (M) | FLOPs (G) | FPS | box AP | -| -------- | ----------- | ---------- | --------- | ---- | ------ | -| LITv2-S | 2 | 38 | 242 | 18.7 | 44.0 | -| LITv2-S | 4 | 38 | 230 | 20.4 | 43.7 | -| LITv2-M | 2 | 59 | 348 | 12.2 | 46.0 | -| LITv2-M | 4 | 59 | 312 | 14.8 | 45.8 | -| LITv2-B | 2 | 97 | 481 | 9.5 | 46.7 | -| LITv2-B | 4 | 97 | 430 | 11.8 | 46.3 | +| Backbone | Params (M) | FLOPs (G) | FPS | mIoU | Config | Download | +| -------- | ---------- | --------- | ---- | ---- | ------ | ----------- | +| LITv2-S | 31 | 41 | 42.6 | 44.3 | link | model & log | +| LITv2-M | 52 | 63 | 28.5 | 45.7 | link | model & log | +| LITv2-B | 90 | 93 | 27.5 | 47.2 | link | model & log | -### Mask R-CNN -| Backbone | Window Size | Params (M) | FLOPs (G) | FPS | box AP | mask AP | -| -------- | ----------- | ---------- | --------- | ---- | ------ | ------- | -| LITv2-S | 2 | 47 | 261 | 18.7 | 44.9 | 40.8 | -| LITv2-S | 4 | 47 | 249 | 21.9 | 44.7 | 40.7 | -| LITv2-M | 2 | 68 | 367 | 12.6 | 46.8 | 42.3 | -| LITv2-M | 4 | 68 | 315 | 16.0 | 46.5 | 42.0 | -| LITv2-B | 2 | 106 | 500 | 9.3 | 47.3 | 42.6 | -| LITv2-B | 4 | 106 | 449 | 11.5 | 46.8 | 42.3 | +## Citation +If you use LITv2 in your research, please consider the following BibTeX entry and giving us a star 🌟. -## Semantic Segmentation on ADE20K +```BibTeX +@article{pan2022hilo + title={Fast Vision Transformers with HiLo Attention}, + author={Pan, Zizheng and Cai, Jianfei and Zhuang, Bohan}, + journal={arXiv preprint arXiv:2205.13213}, + year={2022} +} +``` -### Semantic FPN +If you find the code useful, please also consider the following BibTeX entry -| Backbone | Params (M) | FLOPs (G) | FPS | mIoU | -| -------- | ---------- | --------- | ---- | ---- | -| LITv2-S | 31 | 41 | 42.6 | 44.3 | -| LITv2-M | 52 | 63 | 28.5 | 45.7 | -| LITv2-B | 90 | 93 | 27.5 | 47.2 | +```BibTeX +@inproceedings{pan2022litv1, + title={Less is More: Pay Less Attention in Vision Transformers}, + author={Pan, Zizheng and Zhuang, Bohan and He, Haoyu and Liu, Jing and Cai, Jianfei}, + booktitle = {AAAI}, + year={2022} +} +``` -## License +# License -This repository is released under the Apache 2.0 license as found in the [LICENSE](https://github.com/zip-group/LITv2/blob/main/LICENSE) file. +This repository is released under the Apache 2.0 license as found in the [LICENSE](https://github.com/ziplab/LITv2/blob/main/LICENSE) file. ## Acknowledgement -This repository is built upon [DeiT](https://github.com/facebookresearch/deit), [Swin](https://github.com/microsoft/Swin-Transformer) and [LIT](https://github.com/zip-group/LIT), we thank the authors for their open-sourced code. +This repository is built upon [DeiT](https://github.com/facebookresearch/deit), [Swin](https://github.com/microsoft/Swin-Transformer) and [LIT](https://github.com/ziplab/LIT), we thank the authors for their open-sourced code. diff --git a/classification/README.md b/classification/README.md new file mode 100644 index 0000000..b47191d --- /dev/null +++ b/classification/README.md @@ -0,0 +1,83 @@ +# Classification Code for LITv2 + +## Dataset Preparation + +Download the ImageNet 2012 dataset from [here](http://image-net.org/), and prepare the dataset based on this [script](https://gist.github.com/BIGBALLON/8a71d225eff18d88e469e6ea9b39cef4). The file structure should look like: + +``` +imagenet +├── train +│ ├── class1 +│ │ ├── img1.jpeg +│ │ ├── img2.jpeg +│ │ └── ... +│ ├── class2 +│ │ ├── img3.jpeg +│ │ └── ... +│ └── ... +└── val + ├── class1 + │ ├── img4.jpeg + │ ├── img5.jpeg + │ └── ... + ├── class2 + │ ├── img6.jpeg + │ └── ... + └── ... +``` + + + +## Training + +First, activate your python environment + +```bash +conda activate lit +``` + +Make sure you have the correct ImageNet `DATA_PATH` in `config/*.yaml`. + +To train a model on ImageNet: + +```bash +python -m torch.distributed.launch --nproc_per_node [num_gpus] --master_port 13335 main.py --cfg [path/to/config] +``` + +For example, you can train LITv2-S with 8 GPUs by + +```bash +python -m torch.distributed.launch --nproc_per_node 8 --master_port 13335 main.py --cfg configs/litv2-small.yaml +``` + +Note: In our experiments, we train all models ImageNet-1K with 8 GPUs under a total batch size of 1024. + +## Evaluation + +To evaluate a model, you can run + +```bash +python -m torch.distributed.launch --nproc_per_node [num_gpus] --master_port 13335 main.py --cfg [path/to/config] --eval +``` + +For example, to evaluate LIT-S with 1 GPU, you can run: + +```bash +python -m torch.distributed.launch --nproc_per_node 1 --master_port 13335 main.py --cfg configs/litv2-small.yaml --eval +``` + +This should give + +``` +* Acc@1 82.044 Acc@5 95.666 +Accuracy of the network on the 50000 test images: 82.0% +``` + +> Result could be slightly different based on you environment. + +To test the throughput, you can run + +```bash +python -m torch.distributed.launch --nproc_per_node 1 --master_port 13335 main.py --cfg configs/lit-small.yaml --throughput +``` + diff --git a/classification/config.py b/classification/config.py new file mode 100644 index 0000000..be6eaa8 --- /dev/null +++ b/classification/config.py @@ -0,0 +1,235 @@ +import os +import yaml +from yacs.config import CfgNode as CN + +_C = CN() + +# Base config files +_C.BASE = [''] + +# ----------------------------------------------------------------------------- +# Data settings +# ----------------------------------------------------------------------------- +_C.DATA = CN() +# Batch size for a single GPU, could be overwritten by command line argument +_C.DATA.BATCH_SIZE = 128 +# Path to dataset, could be overwritten by command line argument +_C.DATA.DATA_PATH = '' +# Dataset name +_C.DATA.DATASET = 'imagenet' +# Input image size +_C.DATA.IMG_SIZE = 224 +# Interpolation to resize image (random, bilinear, bicubic) +_C.DATA.INTERPOLATION = 'bicubic' +# Use zipped dataset instead of folder dataset +# could be overwritten by command line argument +_C.DATA.ZIP_MODE = False +# Cache Data in Memory, could be overwritten by command line argument +_C.DATA.CACHE_MODE = 'part' +# Pin CPU memory in DataLoader for more efficient (sometimes) transfer to GPU. +_C.DATA.PIN_MEMORY = True +# Number of data loading threads +_C.DATA.NUM_WORKERS = 8 + +# ----------------------------------------------------------------------------- +# Model settings +# ----------------------------------------------------------------------------- +_C.MODEL = CN() +# Model type +_C.MODEL.TYPE = 'lit' +# Model name +_C.MODEL.NAME = 'lit-s' +# Checkpoint to resume, could be overwritten by command line argument +_C.MODEL.RESUME = '' +# pretrained model on imagenet-1k 224 +_C.MODEL.PRETRAINED = '' +# Number of classes, overwritten in data preparation +_C.MODEL.NUM_CLASSES = 1000 +# Dropout rate +_C.MODEL.DROP_RATE = 0.0 +# Drop path rate +_C.MODEL.DROP_PATH_RATE = 0.1 +# Label Smoothing +_C.MODEL.LABEL_SMOOTHING = 0.1 + +_C.MODEL.OFFSET_LR_MULTI = 1.0 + +# LIT parameters +_C.MODEL.LIT = CN() +_C.MODEL.LIT.PATCH_SIZE = 4 +_C.MODEL.LIT.IN_CHANS = 3 +_C.MODEL.LIT.EMBED_DIM = 96 +_C.MODEL.LIT.DEPTHS = [2, 2, 6, 2] +_C.MODEL.LIT.NUM_HEADS = [3, 6, 12, 24] +_C.MODEL.LIT.WINDOW_SIZE = 7 +_C.MODEL.LIT.MLP_RATIO = 4. +_C.MODEL.LIT.QKV_BIAS = True +_C.MODEL.LIT.QK_SCALE = None +_C.MODEL.LIT.APE = False +_C.MODEL.LIT.PATCH_NORM = True +_C.MODEL.LIT.HAS_MSA = [0, 0, 1, 1] +_C.MODEL.LIT.ALPHA = 0.5 +_C.MODEL.LIT.LOCAL_WS = [0, 0, 2, 1] +# ----------------------------------------------------------------------------- +# Training settings +# ----------------------------------------------------------------------------- +_C.TRAIN = CN() +_C.TRAIN.START_EPOCH = 0 +_C.TRAIN.EPOCHS = 300 +_C.TRAIN.WARMUP_EPOCHS = 20 +_C.TRAIN.WEIGHT_DECAY = 0.05 +_C.TRAIN.BASE_LR = 5e-4 +_C.TRAIN.WARMUP_LR = 5e-7 +_C.TRAIN.MIN_LR = 5e-6 +# Clip gradient norm +_C.TRAIN.CLIP_GRAD = 5.0 +# Auto resume from latest checkpoint +_C.TRAIN.AUTO_RESUME = True +# Gradient accumulation steps +# could be overwritten by command line argument +_C.TRAIN.ACCUMULATION_STEPS = 0 +# Whether to use gradient checkpointing to save memory +# could be overwritten by command line argument +_C.TRAIN.USE_CHECKPOINT = False + +# LR scheduler +_C.TRAIN.LR_SCHEDULER = CN() +_C.TRAIN.LR_SCHEDULER.NAME = 'cosine' +# Epoch interval to decay LR, used in StepLRScheduler +_C.TRAIN.LR_SCHEDULER.DECAY_EPOCHS = 30 +# LR decay rate, used in StepLRScheduler +_C.TRAIN.LR_SCHEDULER.DECAY_RATE = 0.1 + +# Optimizer +_C.TRAIN.OPTIMIZER = CN() +_C.TRAIN.OPTIMIZER.NAME = 'adamw' +# Optimizer Epsilon +_C.TRAIN.OPTIMIZER.EPS = 1e-8 +# Optimizer Betas +_C.TRAIN.OPTIMIZER.BETAS = (0.9, 0.999) +# SGD momentum +_C.TRAIN.OPTIMIZER.MOMENTUM = 0.9 + +# ----------------------------------------------------------------------------- +# Augmentation settings +# ----------------------------------------------------------------------------- +_C.AUG = CN() +# Color jitter factor +_C.AUG.COLOR_JITTER = 0.4 +# Use AutoAugment policy. "v0" or "original" +_C.AUG.AUTO_AUGMENT = 'rand-m9-mstd0.5-inc1' +# Random erase prob +_C.AUG.REPROB = 0.25 +# Random erase mode +_C.AUG.REMODE = 'pixel' +# Random erase count +_C.AUG.RECOUNT = 1 +# Mixup alpha, mixup enabled if > 0 +_C.AUG.MIXUP = 0.8 +# Cutmix alpha, cutmix enabled if > 0 +_C.AUG.CUTMIX = 1.0 +# Cutmix min/max ratio, overrides alpha and enables cutmix if set +_C.AUG.CUTMIX_MINMAX = None +# Probability of performing mixup or cutmix when either/both is enabled +_C.AUG.MIXUP_PROB = 1.0 +# Probability of switching to cutmix when both mixup and cutmix enabled +_C.AUG.MIXUP_SWITCH_PROB = 0.5 +# How to apply mixup/cutmix params. Per "batch", "pair", or "elem" +_C.AUG.MIXUP_MODE = 'batch' + +# ----------------------------------------------------------------------------- +# Testing settings +# ----------------------------------------------------------------------------- +_C.TEST = CN() +# Whether to use center crop when testing +_C.TEST.CROP = True + +# ----------------------------------------------------------------------------- +# Misc +# ----------------------------------------------------------------------------- +# Mixed precision opt level, if O0, no amp is used ('O0', 'O1', 'O2') +# overwritten by command line argument +_C.AMP_OPT_LEVEL = '' +# Path to output folder, overwritten by command line argument +_C.OUTPUT = '' +# Tag of experiment, overwritten by command line argument +_C.TAG = 'default' +# Frequency to save checkpoint +_C.SAVE_FREQ = 1 +# Frequency to logging info +_C.PRINT_FREQ = 10 +# Fixed random seed +_C.SEED = 0 +# Perform evaluation only, overwritten by command line argument +_C.EVAL_MODE = False +# Test throughput only, overwritten by command line argument +_C.THROUGHPUT_MODE = False +# local rank for DistributedDataParallel, given by command line argument +_C.LOCAL_RANK = 0 + + +def _update_config_from_file(config, cfg_file): + config.defrost() + with open(cfg_file, 'r') as f: + yaml_cfg = yaml.load(f, Loader=yaml.FullLoader) + + for cfg in yaml_cfg.setdefault('BASE', ['']): + if cfg: + _update_config_from_file( + config, os.path.join(os.path.dirname(cfg_file), cfg) + ) + print('=> merge config from {}'.format(cfg_file)) + config.merge_from_file(cfg_file) + config.freeze() + + +def update_config(config, args): + _update_config_from_file(config, args.cfg) + + config.defrost() + if args.opts: + config.merge_from_list(args.opts) + + # merge from specific arguments + if args.batch_size: + config.DATA.BATCH_SIZE = args.batch_size + if args.data_path: + config.DATA.DATA_PATH = args.data_path + if args.zip: + config.DATA.ZIP_MODE = True + if args.cache_mode: + config.DATA.CACHE_MODE = args.cache_mode + if args.resume: + config.MODEL.RESUME = args.resume + if args.accumulation_steps: + config.TRAIN.ACCUMULATION_STEPS = args.accumulation_steps + if args.use_checkpoint: + config.TRAIN.USE_CHECKPOINT = True + if args.amp_opt_level: + config.AMP_OPT_LEVEL = args.amp_opt_level + if args.output: + config.OUTPUT = args.output + if args.tag: + config.TAG = args.tag + if args.eval: + config.EVAL_MODE = True + if args.throughput: + config.THROUGHPUT_MODE = True + + # set local rank for distributed training + config.LOCAL_RANK = args.local_rank + + # output folder + config.OUTPUT = os.path.join(config.OUTPUT, config.MODEL.NAME, config.TAG) + + config.freeze() + + +def get_config(args): + """Get a yacs CfgNode object with default values.""" + # Return a clone so that the defaults will not be altered + # This is for the "local variable" use pattern + config = _C.clone() + update_config(config, args) + + return config diff --git a/classification/configs/litv2-base-finetune-384.yaml b/classification/configs/litv2-base-finetune-384.yaml new file mode 100644 index 0000000..eb90804 --- /dev/null +++ b/classification/configs/litv2-base-finetune-384.yaml @@ -0,0 +1,27 @@ +MODEL: + TYPE: litv2 + NAME: litv2_base_384 + DROP_PATH_RATE: 0.5 + OFFSET_LR_MULTI: 0.01 + PRETRAINED: ckpt_epoch_best.pth + LIT: + EMBED_DIM: 128 + DEPTHS: [ 2, 2, 18, 2 ] + NUM_HEADS: [ 4, 8, 16, 32 ] + ALPHA: 0.9 + LOCAL_WS: [ 0, 0, 2, 1 ] +DATA: + IMG_SIZE: 384 + NUM_WORKERS: 10 + BATCH_SIZE: 128 + DATA_PATH: /home/datasets/imagenet + DATASET: imagenet +TRAIN: + EPOCHS: 30 + WARMUP_EPOCHS: 5 + WEIGHT_DECAY: 1e-8 + BASE_LR: 2e-05 + WARMUP_LR: 2e-08 + MIN_LR: 2e-07 +TEST: + CROP: False \ No newline at end of file diff --git a/classification/configs/litv2-base.yaml b/classification/configs/litv2-base.yaml new file mode 100644 index 0000000..551526d --- /dev/null +++ b/classification/configs/litv2-base.yaml @@ -0,0 +1,16 @@ +MODEL: + TYPE: litv2 + NAME: litv2_base + DROP_PATH_RATE: 0.5 + OFFSET_LR_MULTI: 0.01 + LIT: + EMBED_DIM: 128 + DEPTHS: [ 2, 2, 18, 2 ] + NUM_HEADS: [ 4, 8, 16, 32 ] + ALPHA: 0.9 + LOCAL_WS: [ 0, 0, 2, 1 ] +DATA: + NUM_WORKERS: 10 + BATCH_SIZE: 128 + DATA_PATH: /home/datasets/imagenet + DATASET: imagenet \ No newline at end of file diff --git a/classification/configs/litv2-medium.yaml b/classification/configs/litv2-medium.yaml new file mode 100644 index 0000000..c7149bd --- /dev/null +++ b/classification/configs/litv2-medium.yaml @@ -0,0 +1,16 @@ +MODEL: + TYPE: litv2 + NAME: litv2_medium + DROP_PATH_RATE: 0.3 + OFFSET_LR_MULTI: 0.01 + LIT: + EMBED_DIM: 96 + DEPTHS: [ 2, 2, 18, 2 ] + NUM_HEADS: [ 3, 6, 12, 24 ] + ALPHA: 0.9 + LOCAL_WS: [ 0, 0, 2, 1 ] +DATA: + NUM_WORKERS: 10 + BATCH_SIZE: 128 + DATA_PATH: /home/datasets/imagenet + DATASET: imagenet diff --git a/classification/configs/litv2-small.yaml b/classification/configs/litv2-small.yaml new file mode 100644 index 0000000..60215c5 --- /dev/null +++ b/classification/configs/litv2-small.yaml @@ -0,0 +1,16 @@ +MODEL: + TYPE: litv2 + NAME: litv2_small + DROP_PATH_RATE: 0.2 + OFFSET_LR_MULTI: 0.01 + LIT: + EMBED_DIM: 96 + DEPTHS: [ 2, 2, 6, 2 ] + NUM_HEADS: [ 3, 6, 12, 24 ] + ALPHA: 0.9 + LOCAL_WS: [0, 0, 2, 1] +DATA: + NUM_WORKERS: 10 + BATCH_SIZE: 128 + DATA_PATH: /home/datasets/imagenet + DATASET: imagenet \ No newline at end of file diff --git a/classification/data/__init__.py b/classification/data/__init__.py new file mode 100644 index 0000000..70c633c --- /dev/null +++ b/classification/data/__init__.py @@ -0,0 +1 @@ +from .build import build_loader \ No newline at end of file diff --git a/classification/data/build.py b/classification/data/build.py new file mode 100644 index 0000000..e2574fa --- /dev/null +++ b/classification/data/build.py @@ -0,0 +1,132 @@ +import os +import torch +import numpy as np +import torch.distributed as dist +from torchvision import datasets, transforms +from timm.data.constants import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD +from timm.data import Mixup +from timm.data import create_transform +from timm.data.transforms import _pil_interp + +from .cached_image_folder import CachedImageFolder +from .samplers import SubsetRandomSampler + + +def build_loader(config): + config.defrost() + dataset_train, config.MODEL.NUM_CLASSES = build_dataset(is_train=True, config=config) + config.freeze() + print(f"local rank {config.LOCAL_RANK} / global rank {dist.get_rank()} successfully build train dataset") + dataset_val, _ = build_dataset(is_train=False, config=config) + print(f"local rank {config.LOCAL_RANK} / global rank {dist.get_rank()} successfully build val dataset") + + num_tasks = dist.get_world_size() + global_rank = dist.get_rank() + if config.DATA.ZIP_MODE and config.DATA.CACHE_MODE == 'part': + indices = np.arange(dist.get_rank(), len(dataset_train), dist.get_world_size()) + sampler_train = SubsetRandomSampler(indices) + else: + sampler_train = torch.utils.data.DistributedSampler( + dataset_train, num_replicas=num_tasks, rank=global_rank, shuffle=True + ) + + indices = np.arange(dist.get_rank(), len(dataset_val), dist.get_world_size()) + sampler_val = SubsetRandomSampler(indices) + + data_loader_train = torch.utils.data.DataLoader( + dataset_train, sampler=sampler_train, + batch_size=config.DATA.BATCH_SIZE, + num_workers=config.DATA.NUM_WORKERS, + pin_memory=config.DATA.PIN_MEMORY, + drop_last=True, + ) + + data_loader_val = torch.utils.data.DataLoader( + dataset_val, sampler=sampler_val, + batch_size=config.DATA.BATCH_SIZE, + shuffle=False, + num_workers=config.DATA.NUM_WORKERS, + pin_memory=config.DATA.PIN_MEMORY, + drop_last=False + ) + + # setup mixup / cutmix + mixup_fn = None + mixup_active = config.AUG.MIXUP > 0 or config.AUG.CUTMIX > 0. or config.AUG.CUTMIX_MINMAX is not None + if mixup_active: + mixup_fn = Mixup( + mixup_alpha=config.AUG.MIXUP, cutmix_alpha=config.AUG.CUTMIX, cutmix_minmax=config.AUG.CUTMIX_MINMAX, + prob=config.AUG.MIXUP_PROB, switch_prob=config.AUG.MIXUP_SWITCH_PROB, mode=config.AUG.MIXUP_MODE, + label_smoothing=config.MODEL.LABEL_SMOOTHING, num_classes=config.MODEL.NUM_CLASSES) + + return dataset_train, dataset_val, data_loader_train, data_loader_val, mixup_fn + + +def build_dataset(is_train, config): + transform = build_transform(is_train, config) + if config.DATA.DATASET == 'imagenet': + prefix = 'train' if is_train else 'val' + if config.DATA.ZIP_MODE: + ann_file = prefix + "_map.txt" + prefix = prefix + ".zip@/" + dataset = CachedImageFolder(config.DATA.DATA_PATH, ann_file, prefix, transform, + cache_mode=config.DATA.CACHE_MODE if is_train else 'part') + else: + root = os.path.join(config.DATA.DATA_PATH, prefix) + dataset = datasets.ImageFolder(root, transform=transform) + nb_classes = 1000 + elif config.DATA.DATASET == 'imagenet100': + prefix = 'train' if is_train else 'val' + root = os.path.join(config.DATA.DATA_PATH, prefix) + dataset = datasets.ImageFolder(root, transform=transform) + nb_classes = 100 + elif config.DATA.DATASET == 'cifar': + dataset = datasets.CIFAR100(config.DATA.DATA_PATH, train=is_train, transform=transform) + nb_classes = 100 + elif config.DATA.DATASET == 'cifar10': + dataset = datasets.CIFAR10(config.DATA.DATA_PATH, train=is_train, transform=transform) + nb_classes = 10 + else: + raise NotImplementedError("We only support ImageNet Now.") + + return dataset, nb_classes + + +def build_transform(is_train, config): + resize_im = config.DATA.IMG_SIZE > 32 + if is_train: + # this should always dispatch to transforms_imagenet_train + transform = create_transform( + input_size=config.DATA.IMG_SIZE, + is_training=True, + color_jitter=config.AUG.COLOR_JITTER if config.AUG.COLOR_JITTER > 0 else None, + auto_augment=config.AUG.AUTO_AUGMENT if config.AUG.AUTO_AUGMENT != 'none' else None, + re_prob=config.AUG.REPROB, + re_mode=config.AUG.REMODE, + re_count=config.AUG.RECOUNT, + interpolation=config.DATA.INTERPOLATION, + ) + if not resize_im: + # replace RandomResizedCropAndInterpolation with + # RandomCrop + transform.transforms[0] = transforms.RandomCrop(config.DATA.IMG_SIZE, padding=4) + return transform + + t = [] + if resize_im: + if config.TEST.CROP: + size = int((256 / 224) * config.DATA.IMG_SIZE) + t.append( + transforms.Resize(size, interpolation=_pil_interp(config.DATA.INTERPOLATION)), + # to maintain same ratio w.r.t. 224 images + ) + t.append(transforms.CenterCrop(config.DATA.IMG_SIZE)) + else: + t.append( + transforms.Resize((config.DATA.IMG_SIZE, config.DATA.IMG_SIZE), + interpolation=_pil_interp(config.DATA.INTERPOLATION)) + ) + + t.append(transforms.ToTensor()) + t.append(transforms.Normalize(IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD)) + return transforms.Compose(t) diff --git a/classification/data/cached_image_folder.py b/classification/data/cached_image_folder.py new file mode 100644 index 0000000..2f3d013 --- /dev/null +++ b/classification/data/cached_image_folder.py @@ -0,0 +1,244 @@ +import io +import os +import time +import torch.distributed as dist +import torch.utils.data as data +from PIL import Image + +from .zipreader import is_zip_path, ZipReader + + +def has_file_allowed_extension(filename, extensions): + """Checks if a file is an allowed extension. + Args: + filename (string): path to a file + Returns: + bool: True if the filename ends with a known image extension + """ + filename_lower = filename.lower() + return any(filename_lower.endswith(ext) for ext in extensions) + + +def find_classes(dir): + classes = [d for d in os.listdir(dir) if os.path.isdir(os.path.join(dir, d))] + classes.sort() + class_to_idx = {classes[i]: i for i in range(len(classes))} + return classes, class_to_idx + + +def make_dataset(dir, class_to_idx, extensions): + images = [] + dir = os.path.expanduser(dir) + for target in sorted(os.listdir(dir)): + d = os.path.join(dir, target) + if not os.path.isdir(d): + continue + + for root, _, fnames in sorted(os.walk(d)): + for fname in sorted(fnames): + if has_file_allowed_extension(fname, extensions): + path = os.path.join(root, fname) + item = (path, class_to_idx[target]) + images.append(item) + + return images + + +def make_dataset_with_ann(ann_file, img_prefix, extensions): + images = [] + with open(ann_file, "r") as f: + contents = f.readlines() + for line_str in contents: + path_contents = [c for c in line_str.split('\t')] + im_file_name = path_contents[0] + class_index = int(path_contents[1]) + + assert str.lower(os.path.splitext(im_file_name)[-1]) in extensions + item = (os.path.join(img_prefix, im_file_name), class_index) + + images.append(item) + + return images + + +class DatasetFolder(data.Dataset): + """A generic data loader where the samples are arranged in this way: :: + root/class_x/xxx.ext + root/class_x/xxy.ext + root/class_x/xxz.ext + root/class_y/123.ext + root/class_y/nsdf3.ext + root/class_y/asd932_.ext + Args: + root (string): Root directory path. + loader (callable): A function to load a sample given its path. + extensions (list[string]): A list of allowed extensions. + transform (callable, optional): A function/transform that takes in + a sample and returns a transformed version. + E.g, ``transforms.RandomCrop`` for images. + target_transform (callable, optional): A function/transform that takes + in the target and transforms it. + Attributes: + samples (list): List of (sample path, class_index) tuples + """ + + def __init__(self, root, loader, extensions, ann_file='', img_prefix='', transform=None, target_transform=None, + cache_mode="no"): + # image folder mode + if ann_file == '': + _, class_to_idx = find_classes(root) + samples = make_dataset(root, class_to_idx, extensions) + # zip mode + else: + samples = make_dataset_with_ann(os.path.join(root, ann_file), + os.path.join(root, img_prefix), + extensions) + + if len(samples) == 0: + raise (RuntimeError("Found 0 files in subfolders of: " + root + "\n" + + "Supported extensions are: " + ",".join(extensions))) + + self.root = root + self.loader = loader + self.extensions = extensions + + self.samples = samples + self.labels = [y_1k for _, y_1k in samples] + self.classes = list(set(self.labels)) + + self.transform = transform + self.target_transform = target_transform + + self.cache_mode = cache_mode + if self.cache_mode != "no": + self.init_cache() + + def init_cache(self): + assert self.cache_mode in ["part", "full"] + n_sample = len(self.samples) + global_rank = dist.get_rank() + world_size = dist.get_world_size() + + samples_bytes = [None for _ in range(n_sample)] + start_time = time.time() + for index in range(n_sample): + if index % (n_sample // 10) == 0: + t = time.time() - start_time + print(f'global_rank {dist.get_rank()} cached {index}/{n_sample} takes {t:.2f}s per block') + start_time = time.time() + path, target = self.samples[index] + if self.cache_mode == "full": + samples_bytes[index] = (ZipReader.read(path), target) + elif self.cache_mode == "part" and index % world_size == global_rank: + samples_bytes[index] = (ZipReader.read(path), target) + else: + samples_bytes[index] = (path, target) + self.samples = samples_bytes + + def __getitem__(self, index): + """ + Args: + index (int): Index + Returns: + tuple: (sample, target) where target is class_index of the target class. + """ + path, target = self.samples[index] + sample = self.loader(path) + if self.transform is not None: + sample = self.transform(sample) + if self.target_transform is not None: + target = self.target_transform(target) + + return sample, target + + def __len__(self): + return len(self.samples) + + def __repr__(self): + fmt_str = 'Dataset ' + self.__class__.__name__ + '\n' + fmt_str += ' Number of datapoints: {}\n'.format(self.__len__()) + fmt_str += ' Root Location: {}\n'.format(self.root) + tmp = ' Transforms (if any): ' + fmt_str += '{0}{1}\n'.format(tmp, self.transform.__repr__().replace('\n', '\n' + ' ' * len(tmp))) + tmp = ' Target Transforms (if any): ' + fmt_str += '{0}{1}'.format(tmp, self.target_transform.__repr__().replace('\n', '\n' + ' ' * len(tmp))) + return fmt_str + + +IMG_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif'] + + +def pil_loader(path): + # open path as file to avoid ResourceWarning (https://github.com/python-pillow/Pillow/issues/835) + if isinstance(path, bytes): + img = Image.open(io.BytesIO(path)) + elif is_zip_path(path): + data = ZipReader.read(path) + img = Image.open(io.BytesIO(data)) + else: + with open(path, 'rb') as f: + img = Image.open(f) + return img.convert('RGB') + + +def accimage_loader(path): + import accimage + try: + return accimage.Image(path) + except IOError: + # Potentially a decoding problem, fall back to PIL.Image + return pil_loader(path) + + +def default_img_loader(path): + from torchvision import get_image_backend + if get_image_backend() == 'accimage': + return accimage_loader(path) + else: + return pil_loader(path) + + +class CachedImageFolder(DatasetFolder): + """A generic data loader where the images are arranged in this way: :: + root/dog/xxx.png + root/dog/xxy.png + root/dog/xxz.png + root/cat/123.png + root/cat/nsdf3.png + root/cat/asd932_.png + Args: + root (string): Root directory path. + transform (callable, optional): A function/transform that takes in an PIL image + and returns a transformed version. E.g, ``transforms.RandomCrop`` + target_transform (callable, optional): A function/transform that takes in the + target and transforms it. + loader (callable, optional): A function to load an image given its path. + Attributes: + imgs (list): List of (image path, class_index) tuples + """ + + def __init__(self, root, ann_file='', img_prefix='', transform=None, target_transform=None, + loader=default_img_loader, cache_mode="no"): + super(CachedImageFolder, self).__init__(root, loader, IMG_EXTENSIONS, + ann_file=ann_file, img_prefix=img_prefix, + transform=transform, target_transform=target_transform, + cache_mode=cache_mode) + self.imgs = self.samples + + def __getitem__(self, index): + """ + Args: + index (int): Index + Returns: + tuple: (image, target) where target is class_index of the target class. + """ + path, target = self.samples[index] + image = self.loader(path) + if self.transform is not None: + img = self.transform(image) + else: + img = image + if self.target_transform is not None: + target = self.target_transform(target) + + return img, target diff --git a/classification/data/samplers.py b/classification/data/samplers.py new file mode 100644 index 0000000..229904f --- /dev/null +++ b/classification/data/samplers.py @@ -0,0 +1,22 @@ +import torch + + +class SubsetRandomSampler(torch.utils.data.Sampler): + r"""Samples elements randomly from a given list of indices, without replacement. + + Arguments: + indices (sequence): a sequence of indices + """ + + def __init__(self, indices): + self.epoch = 0 + self.indices = indices + + def __iter__(self): + return (self.indices[i] for i in torch.randperm(len(self.indices))) + + def __len__(self): + return len(self.indices) + + def set_epoch(self, epoch): + self.epoch = epoch diff --git a/classification/data/zipreader.py b/classification/data/zipreader.py new file mode 100644 index 0000000..9d773c3 --- /dev/null +++ b/classification/data/zipreader.py @@ -0,0 +1,96 @@ +import os +import zipfile +import io +import numpy as np +from PIL import Image +from PIL import ImageFile + +ImageFile.LOAD_TRUNCATED_IMAGES = True + + +def is_zip_path(img_or_path): + """judge if this is a zip path""" + return '.zip@' in img_or_path + + +class ZipReader(object): + """A class to read zipped files""" + zip_bank = dict() + + def __init__(self): + super(ZipReader, self).__init__() + + @staticmethod + def get_zipfile(path): + zip_bank = ZipReader.zip_bank + if path not in zip_bank: + zfile = zipfile.ZipFile(path, 'r') + zip_bank[path] = zfile + return zip_bank[path] + + @staticmethod + def split_zip_style_path(path): + pos_at = path.index('@') + assert pos_at != -1, "character '@' is not found from the given path '%s'" % path + + zip_path = path[0: pos_at] + folder_path = path[pos_at + 1:] + folder_path = str.strip(folder_path, '/') + return zip_path, folder_path + + @staticmethod + def list_folder(path): + zip_path, folder_path = ZipReader.split_zip_style_path(path) + + zfile = ZipReader.get_zipfile(zip_path) + folder_list = [] + for file_foler_name in zfile.namelist(): + file_foler_name = str.strip(file_foler_name, '/') + if file_foler_name.startswith(folder_path) and \ + len(os.path.splitext(file_foler_name)[-1]) == 0 and \ + file_foler_name != folder_path: + if len(folder_path) == 0: + folder_list.append(file_foler_name) + else: + folder_list.append(file_foler_name[len(folder_path) + 1:]) + + return folder_list + + @staticmethod + def list_files(path, extension=None): + if extension is None: + extension = ['.*'] + zip_path, folder_path = ZipReader.split_zip_style_path(path) + + zfile = ZipReader.get_zipfile(zip_path) + file_lists = [] + for file_foler_name in zfile.namelist(): + file_foler_name = str.strip(file_foler_name, '/') + if file_foler_name.startswith(folder_path) and \ + str.lower(os.path.splitext(file_foler_name)[-1]) in extension: + if len(folder_path) == 0: + file_lists.append(file_foler_name) + else: + file_lists.append(file_foler_name[len(folder_path) + 1:]) + + return file_lists + + @staticmethod + def read(path): + zip_path, path_img = ZipReader.split_zip_style_path(path) + zfile = ZipReader.get_zipfile(zip_path) + data = zfile.read(path_img) + return data + + @staticmethod + def imread(path): + zip_path, path_img = ZipReader.split_zip_style_path(path) + zfile = ZipReader.get_zipfile(zip_path) + data = zfile.read(path_img) + try: + im = Image.open(io.BytesIO(data)) + except: + print("ERROR IMG LOADED: ", path_img) + random_img = np.random.rand(224, 224, 3) * 255 + im = Image.fromarray(np.uint8(random_img)) + return im diff --git a/classification/logger.py b/classification/logger.py new file mode 100644 index 0000000..3e48e0f --- /dev/null +++ b/classification/logger.py @@ -0,0 +1,34 @@ +import os +import sys +import logging +import functools +from termcolor import colored + + +@functools.lru_cache() +def create_logger(output_dir, dist_rank=0, name=''): + # create logger + logger = logging.getLogger(name) + logger.setLevel(logging.DEBUG) + logger.propagate = False + + # create formatter + fmt = '[%(asctime)s %(name)s] (%(filename)s %(lineno)d): %(levelname)s %(message)s' + color_fmt = colored('[%(asctime)s %(name)s]', 'green') + \ + colored('(%(filename)s %(lineno)d)', 'yellow') + ': %(levelname)s %(message)s' + + # create console handlers for master process + if dist_rank == 0: + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setLevel(logging.DEBUG) + console_handler.setFormatter( + logging.Formatter(fmt=color_fmt, datefmt='%Y-%m-%d %H:%M:%S')) + logger.addHandler(console_handler) + + # create file handlers + file_handler = logging.FileHandler(os.path.join(output_dir, f'log_rank{dist_rank}.txt'), mode='a') + file_handler.setLevel(logging.DEBUG) + file_handler.setFormatter(logging.Formatter(fmt=fmt, datefmt='%Y-%m-%d %H:%M:%S')) + logger.addHandler(file_handler) + + return logger diff --git a/classification/lr_scheduler.py b/classification/lr_scheduler.py new file mode 100644 index 0000000..2513935 --- /dev/null +++ b/classification/lr_scheduler.py @@ -0,0 +1,95 @@ +import torch +from timm.scheduler.cosine_lr import CosineLRScheduler +from timm.scheduler.step_lr import StepLRScheduler +from timm.scheduler.scheduler import Scheduler + + +def build_scheduler(config, optimizer, n_iter_per_epoch): + num_steps = int(config.TRAIN.EPOCHS * n_iter_per_epoch) + warmup_steps = int(config.TRAIN.WARMUP_EPOCHS * n_iter_per_epoch) + decay_steps = int(config.TRAIN.LR_SCHEDULER.DECAY_EPOCHS * n_iter_per_epoch) + + lr_scheduler = None + if config.TRAIN.LR_SCHEDULER.NAME == 'cosine': + lr_scheduler = CosineLRScheduler( + optimizer, + t_initial=num_steps, + t_mul=1., + lr_min=config.TRAIN.MIN_LR, + warmup_lr_init=config.TRAIN.WARMUP_LR, + warmup_t=warmup_steps, + cycle_limit=1, + t_in_epochs=False, + ) + elif config.TRAIN.LR_SCHEDULER.NAME == 'linear': + lr_scheduler = LinearLRScheduler( + optimizer, + t_initial=num_steps, + lr_min_rate=0.01, + warmup_lr_init=config.TRAIN.WARMUP_LR, + warmup_t=warmup_steps, + t_in_epochs=False, + ) + elif config.TRAIN.LR_SCHEDULER.NAME == 'step': + lr_scheduler = StepLRScheduler( + optimizer, + decay_t=decay_steps, + decay_rate=config.TRAIN.LR_SCHEDULER.DECAY_RATE, + warmup_lr_init=config.TRAIN.WARMUP_LR, + warmup_t=warmup_steps, + t_in_epochs=False, + ) + + return lr_scheduler + + +class LinearLRScheduler(Scheduler): + def __init__(self, + optimizer: torch.optim.Optimizer, + t_initial: int, + lr_min_rate: float, + warmup_t=0, + warmup_lr_init=0., + t_in_epochs=True, + noise_range_t=None, + noise_pct=0.67, + noise_std=1.0, + noise_seed=42, + initialize=True, + ) -> None: + super().__init__( + optimizer, param_group_field="lr", + noise_range_t=noise_range_t, noise_pct=noise_pct, noise_std=noise_std, noise_seed=noise_seed, + initialize=initialize) + + self.t_initial = t_initial + self.lr_min_rate = lr_min_rate + self.warmup_t = warmup_t + self.warmup_lr_init = warmup_lr_init + self.t_in_epochs = t_in_epochs + if self.warmup_t: + self.warmup_steps = [(v - warmup_lr_init) / self.warmup_t for v in self.base_values] + super().update_groups(self.warmup_lr_init) + else: + self.warmup_steps = [1 for _ in self.base_values] + + def _get_lr(self, t): + if t < self.warmup_t: + lrs = [self.warmup_lr_init + t * s for s in self.warmup_steps] + else: + t = t - self.warmup_t + total_t = self.t_initial - self.warmup_t + lrs = [v - ((v - v * self.lr_min_rate) * (t / total_t)) for v in self.base_values] + return lrs + + def get_epoch_values(self, epoch: int): + if self.t_in_epochs: + return self._get_lr(epoch) + else: + return None + + def get_update_values(self, num_updates: int): + if not self.t_in_epochs: + return self._get_lr(num_updates) + else: + return None diff --git a/classification/main.py b/classification/main.py new file mode 100644 index 0000000..17353ca --- /dev/null +++ b/classification/main.py @@ -0,0 +1,362 @@ +import os +import time +import argparse +import datetime +# import matplotlib.pyplot as plt +import numpy as np + +import torch +import torch.backends.cudnn as cudnn +import torch.distributed as dist + +from timm.loss import LabelSmoothingCrossEntropy, SoftTargetCrossEntropy +from timm.utils import accuracy, AverageMeter + +from config import get_config +from models import build_model +from data import build_loader +from lr_scheduler import build_scheduler +from optimizer import build_optimizer +from logger import create_logger +from utils import load_checkpoint, load_pretrained, save_checkpoint, get_grad_norm, auto_resume_helper, reduce_tensor, save_best_checkpoint, save_last_checkpoint + +try: + # noinspection PyUnresolvedReferences + from apex import amp +except ImportError: + amp = None + + +def parse_option(): + parser = argparse.ArgumentParser('LIT training and evaluation script', add_help=False) + parser.add_argument('--cfg', type=str, required=True, metavar="FILE", help='path to config file', ) + parser.add_argument( + "--opts", + help="Modify config options by adding 'KEY VALUE' pairs. ", + default=None, + nargs='+', + ) + + # easy config modification + parser.add_argument('--batch-size', type=int, help="batch size for single GPU") + parser.add_argument('--data-path', type=str, help='path to dataset') + parser.add_argument('--zip', action='store_true', help='use zipped dataset instead of folder dataset') + parser.add_argument('--cache-mode', type=str, default='part', choices=['no', 'full', 'part'], + help='no: no cache, ' + 'full: cache all data, ' + 'part: sharding the dataset into nonoverlapping pieces and only cache one piece') + parser.add_argument('--resume', help='resume from checkpoint') + parser.add_argument('--accumulation-steps', type=int, help="gradient accumulation steps") + parser.add_argument('--use-checkpoint', action='store_true', + help="whether to use gradient checkpointing to save memory") + parser.add_argument('--amp-opt-level', type=str, default='O1', choices=['O0', 'O1', 'O2'], + help='mixed precision opt level, if O0, no amp is used') + parser.add_argument('--output', default='output', type=str, metavar='PATH', + help='root of output folder, the full path is // (default: output)') + parser.add_argument('--tag', help='tag of experiment') + parser.add_argument('--eval', action='store_true', help='Perform evaluation only') + parser.add_argument('--throughput', action='store_true', help='Test throughput only') + + # distributed training + parser.add_argument("--local_rank", type=int, required=True, help='local rank for DistributedDataParallel') + parser.add_argument('--pretrained', help='pretrained weight from checkpoint, could be imagenet22k pretrained weight') + + args, unparsed = parser.parse_known_args() + + config = get_config(args) + + return args, config + + +def main(config): + dataset_train, dataset_val, data_loader_train, data_loader_val, mixup_fn = build_loader(config) + + logger.info(f"Creating model:{config.MODEL.TYPE}/{config.MODEL.NAME}") + model = build_model(config) + + # for visualization + for name, module in model.named_modules(): + module.name = name + + model.cuda() + logger.info(str(model)) + + optimizer = build_optimizer(config, model) + if config.AMP_OPT_LEVEL != "O0": + model, optimizer = amp.initialize(model, optimizer, opt_level=config.AMP_OPT_LEVEL) + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[config.LOCAL_RANK], broadcast_buffers=False) + model_without_ddp = model.module + + n_parameters = sum(p.numel() for p in model.parameters() if p.requires_grad) + logger.info(f"number of params: {n_parameters}") + if hasattr(model_without_ddp, 'flops'): + flops = model_without_ddp.flops() + logger.info(f"number of GFLOPs: {flops / 1e9}") + + lr_scheduler = build_scheduler(config, optimizer, len(data_loader_train)) + + if config.AUG.MIXUP > 0.: + # smoothing is handled with mixup label transform + criterion = SoftTargetCrossEntropy() + elif config.MODEL.LABEL_SMOOTHING > 0.: + criterion = LabelSmoothingCrossEntropy(smoothing=config.MODEL.LABEL_SMOOTHING) + else: + criterion = torch.nn.CrossEntropyLoss() + + max_accuracy = 0.0 + + if config.TRAIN.AUTO_RESUME: + resume_file = auto_resume_helper(config.OUTPUT) + if resume_file: + if config.MODEL.RESUME: + logger.warning(f"auto-resume changing resume file from {config.MODEL.RESUME} to {resume_file}") + config.defrost() + config.MODEL.RESUME = resume_file + config.freeze() + logger.info(f'auto resuming from {resume_file}') + else: + logger.info(f'no checkpoint found in {config.OUTPUT}, ignoring auto resume') + + if config.THROUGHPUT_MODE: + throughput(data_loader_val, model, logger) + return + + if config.MODEL.RESUME: + max_accuracy = load_checkpoint(config, model_without_ddp, optimizer, lr_scheduler, logger) + acc1, acc5, loss = validate(config, data_loader_val, model) + logger.info(f"Accuracy of the network on the {len(dataset_val)} test images: {acc1:.1f}%") + if config.EVAL_MODE: + return + + if config.MODEL.PRETRAINED and (not config.MODEL.RESUME): + max_accuracy = load_pretrained(config, model_without_ddp, optimizer, lr_scheduler, logger) + acc1, acc5, loss = validate(config, data_loader_val, model) + logger.info(f"Accuracy of the network on the {len(dataset_val)} test images: {acc1:.1f}%") + if config.EVAL_MODE: + return + + if config.EVAL_MODE: + acc1, acc5, loss = validate(config, data_loader_val, model) + logger.info(f"Accuracy of the network on the {len(dataset_val)} test images: {acc1:.1f}%") + return + + + logger.info("Start training") + start_time = time.time() + for epoch in range(config.TRAIN.START_EPOCH, config.TRAIN.EPOCHS): + data_loader_train.sampler.set_epoch(epoch) + + train_one_epoch(config, model, criterion, data_loader_train, optimizer, epoch, mixup_fn, lr_scheduler) + if dist.get_rank() == 0 and (epoch % config.SAVE_FREQ == 0 or epoch == (config.TRAIN.EPOCHS - 1)): + save_last_checkpoint(config, epoch, model_without_ddp, max_accuracy, optimizer, lr_scheduler, logger) + + acc1, acc5, loss = validate(config, data_loader_val, model) + logger.info(f"Accuracy of the network on the {len(dataset_val)} test images: {acc1:.1f}%") + if dist.get_rank() == 0 and max_accuracy < acc1: + save_best_checkpoint(config, epoch, model_without_ddp, max_accuracy, optimizer, lr_scheduler, logger) + max_accuracy = max(max_accuracy, acc1) + logger.info(f'Max accuracy: {max_accuracy:.2f}%') + + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + logger.info('Training time {}'.format(total_time_str)) + + +def train_one_epoch(config, model, criterion, data_loader, optimizer, epoch, mixup_fn, lr_scheduler): + model.train() + optimizer.zero_grad() + + num_steps = len(data_loader) + batch_time = AverageMeter() + loss_meter = AverageMeter() + norm_meter = AverageMeter() + + start = time.time() + end = time.time() + for idx, (samples, targets) in enumerate(data_loader): + samples = samples.cuda(non_blocking=True) + targets = targets.cuda(non_blocking=True) + + if mixup_fn is not None: + samples, targets = mixup_fn(samples, targets) + + outputs = model(samples) + + if config.TRAIN.ACCUMULATION_STEPS > 1: + loss = criterion(outputs, targets) + loss = loss / config.TRAIN.ACCUMULATION_STEPS + if config.AMP_OPT_LEVEL != "O0": + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + if config.TRAIN.CLIP_GRAD: + grad_norm = torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), config.TRAIN.CLIP_GRAD) + else: + grad_norm = get_grad_norm(amp.master_params(optimizer)) + else: + loss.backward() + if config.TRAIN.CLIP_GRAD: + grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), config.TRAIN.CLIP_GRAD) + else: + grad_norm = get_grad_norm(model.parameters()) + if (idx + 1) % config.TRAIN.ACCUMULATION_STEPS == 0: + optimizer.step() + optimizer.zero_grad() + lr_scheduler.step_update(epoch * num_steps + idx) + else: + loss = criterion(outputs, targets) + optimizer.zero_grad() + if config.AMP_OPT_LEVEL != "O0": + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + if config.TRAIN.CLIP_GRAD: + grad_norm = torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), config.TRAIN.CLIP_GRAD) + else: + grad_norm = get_grad_norm(amp.master_params(optimizer)) + else: + loss.backward() + if config.TRAIN.CLIP_GRAD: + grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), config.TRAIN.CLIP_GRAD) + else: + grad_norm = get_grad_norm(model.parameters()) + optimizer.step() + lr_scheduler.step_update(epoch * num_steps + idx) + + torch.cuda.synchronize() + + loss_meter.update(loss.item(), targets.size(0)) + norm_meter.update(grad_norm) + batch_time.update(time.time() - end) + end = time.time() + + if idx % config.PRINT_FREQ == 0: + lr = optimizer.param_groups[0]['lr'] + memory_used = torch.cuda.max_memory_allocated() / (1024.0 * 1024.0) + etas = batch_time.avg * (num_steps - idx) + logger.info( + f'Train: [{epoch}/{config.TRAIN.EPOCHS}][{idx}/{num_steps}]\t' + f'eta {datetime.timedelta(seconds=int(etas))} lr {lr:.6f}\t' + f'time {batch_time.val:.4f} ({batch_time.avg:.4f})\t' + f'loss {loss_meter.val:.4f} ({loss_meter.avg:.4f})\t' + f'grad_norm {norm_meter.val:.4f} ({norm_meter.avg:.4f})\t' + f'mem {memory_used:.0f}MB') + epoch_time = time.time() - start + logger.info(f"EPOCH {epoch} training takes {datetime.timedelta(seconds=int(epoch_time))}") + + +@torch.no_grad() +def validate(config, data_loader, model): + criterion = torch.nn.CrossEntropyLoss() + model.eval() + + batch_time = AverageMeter() + loss_meter = AverageMeter() + acc1_meter = AverageMeter() + acc5_meter = AverageMeter() + + end = time.time() + for idx, (images, target) in enumerate(data_loader): + images = images.cuda(non_blocking=True) + target = target.cuda(non_blocking=True) + + # compute output + output = model(images) + # break + # measure accuracy and record loss + loss = criterion(output, target) + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + + acc1 = reduce_tensor(acc1) + acc5 = reduce_tensor(acc5) + loss = reduce_tensor(loss) + + loss_meter.update(loss.item(), target.size(0)) + acc1_meter.update(acc1.item(), target.size(0)) + acc5_meter.update(acc5.item(), target.size(0)) + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if idx % config.PRINT_FREQ == 0: + memory_used = torch.cuda.max_memory_allocated() / (1024.0 * 1024.0) + logger.info( + f'Test: [{idx}/{len(data_loader)}]\t' + f'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' + f'Loss {loss_meter.val:.4f} ({loss_meter.avg:.4f})\t' + f'Acc@1 {acc1_meter.val:.3f} ({acc1_meter.avg:.3f})\t' + f'Acc@5 {acc5_meter.val:.3f} ({acc5_meter.avg:.3f})\t' + f'Mem {memory_used:.0f}MB') + logger.info(f' * Acc@1 {acc1_meter.avg:.3f} Acc@5 {acc5_meter.avg:.3f}') + return acc1_meter.avg, acc5_meter.avg, loss_meter.avg + + +@torch.no_grad() +def throughput(data_loader, model, logger): + model.eval() + + for idx, (images, _) in enumerate(data_loader): + images = images.cuda(non_blocking=True) + batch_size = images.shape[0] + for i in range(50): + model(images) + torch.cuda.synchronize() + logger.info(f"throughput averaged with 30 times") + tic1 = time.time() + for i in range(30): + model(images) + torch.cuda.synchronize() + tic2 = time.time() + logger.info(f"batch_size {batch_size} throughput {30 * batch_size / (tic2 - tic1)}") + return + + +if __name__ == '__main__': + _, config = parse_option() + + if config.AMP_OPT_LEVEL != "O0": + assert amp is not None, "amp not installed!" + + if 'RANK' in os.environ and 'WORLD_SIZE' in os.environ: + rank = int(os.environ["RANK"]) + world_size = int(os.environ['WORLD_SIZE']) + print(f"RANK and WORLD_SIZE in environ: {rank}/{world_size}") + else: + rank = -1 + world_size = -1 + torch.cuda.set_device(config.LOCAL_RANK) + torch.distributed.init_process_group(backend='nccl', init_method='env://', world_size=world_size, rank=rank) + torch.distributed.barrier() + + seed = config.SEED + dist.get_rank() + torch.manual_seed(seed) + np.random.seed(seed) + cudnn.benchmark = True + + # linear scale the learning rate according to total batch size, may not be optimal + linear_scaled_lr = config.TRAIN.BASE_LR * config.DATA.BATCH_SIZE * dist.get_world_size() / 512.0 + linear_scaled_warmup_lr = config.TRAIN.WARMUP_LR * config.DATA.BATCH_SIZE * dist.get_world_size() / 512.0 + linear_scaled_min_lr = config.TRAIN.MIN_LR * config.DATA.BATCH_SIZE * dist.get_world_size() / 512.0 + # gradient accumulation also need to scale the learning rate + if config.TRAIN.ACCUMULATION_STEPS > 1: + linear_scaled_lr = linear_scaled_lr * config.TRAIN.ACCUMULATION_STEPS + linear_scaled_warmup_lr = linear_scaled_warmup_lr * config.TRAIN.ACCUMULATION_STEPS + linear_scaled_min_lr = linear_scaled_min_lr * config.TRAIN.ACCUMULATION_STEPS + config.defrost() + config.TRAIN.BASE_LR = linear_scaled_lr + config.TRAIN.WARMUP_LR = linear_scaled_warmup_lr + config.TRAIN.MIN_LR = linear_scaled_min_lr + config.freeze() + + os.makedirs(config.OUTPUT, exist_ok=True) + logger = create_logger(output_dir=config.OUTPUT, dist_rank=dist.get_rank(), name=f"{config.MODEL.NAME}") + + if dist.get_rank() == 0: + path = os.path.join(config.OUTPUT, "config.json") + with open(path, "w") as f: + f.write(config.dump()) + logger.info(f"Full config saved to {path}") + + # print config + logger.info(config.dump()) + + main(config) diff --git a/classification/mm_modules/DCN/deform_conv2d_naive.py b/classification/mm_modules/DCN/deform_conv2d_naive.py new file mode 100644 index 0000000..100ce74 --- /dev/null +++ b/classification/mm_modules/DCN/deform_conv2d_naive.py @@ -0,0 +1,93 @@ +import torch +import torch.nn as nn +from torch.nn import init +import math +import numpy as np +from torch.nn.modules.module import Module +import torch.nn.functional as F +from torch.nn.modules.utils import _pair + +class deform_conv2d_naive(Module): + def __init__(self, in_channels, out_channels, + kernel_size, stride, padding, dilation=1, groups=1, deformable_groups=1, bias=True): + super(deform_conv2d_naive, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = _pair(kernel_size) + self.stride = _pair(stride) + self.padding = _pair(padding) + self.dilation = _pair(dilation) + self.groups = groups + self.deformable_groups = deformable_groups + self.use_bias = bias + + self.weight = nn.Parameter(torch.Tensor( + out_channels, in_channels//groups, *self.kernel_size)) + self.bias = nn.Parameter(torch.Tensor(out_channels)) + self.reset_parameters() + if not self.use_bias: + self.bias.requires_grad = False + self.bias.data.zero_() + + def reset_parameters(self): + n = self.in_channels + init.kaiming_uniform_(self.weight, a=math.sqrt(5)) + if self.bias is not None: + fan_in, _ = init._calculate_fan_in_and_fan_out(self.weight) + bound = 1 / math.sqrt(fan_in) + init.uniform_(self.bias, -bound, bound) + + def forward(self, input, offset): + N = input.size(0) + in_channels = self.in_channels + out_channels = self.out_channels + in_h = input.size(2) + in_w = input.size(3) + out_h = offset.size(2) + out_w = offset.size(3) + kernel_h = self.kernel_size[0] + kernel_w = self.kernel_size[1] + # [1, kernel_h * kernel_w, out_h, out_w, 2] + mesh = self.compute_mesh_grid(in_h, in_w).cuda(input.get_device()) + offset = offset.view(N, self.deformable_groups, kernel_h, kernel_w, 2, out_h, out_w) + # [N * dg * kernel_h * kernel_w, out_h, out_w, 2] + offset = offset.permute(0, 1, 2, 3, 5, 6, 4).contiguous().view(N * self.deformable_groups * kernel_h * kernel_w, out_h, out_w, 2) + offset_x_normalize = (offset[:, :, :, 1]) / ((in_w - 1) * 1.0 / 2) + offset_y_normalize = (offset[:, :, :, 0]) / ((in_h - 1) * 1.0 / 2) + # [N * dg * kernel_h * kernel_w, out_h, out_w, 2] + offset = torch.cat([offset_x_normalize[..., None], offset_y_normalize[..., None]], dim=3) + # [N * dg * kernel_h * kernel_w, out_h, out_w, 2] + grid = mesh.expand(N * self.deformable_groups, -1, -1, -1, -1).contiguous().view(-1, out_h, out_w, 2) + offset + # [N * kernel_h * kernel_w * dg, in_channels/dg, in_h, in_w] + input = input[:, None, ...].expand(-1, kernel_h * kernel_w, -1, -1, -1).contiguous().view( + N * kernel_h * kernel_w * self.deformable_groups, in_channels // self.deformable_groups, in_h, in_w) + sampled_feat = F.grid_sample(input, grid).view(N, kernel_h * kernel_w, in_channels, out_h, out_w).permute(2, 1, 0, 3, 4).contiguous().view(in_channels * kernel_h * kernel_w, -1) + output_feat = torch.matmul(self.weight.view(self.weight.size(0), -1), sampled_feat).view(out_channels, N, out_h, out_w).permute(1,0,2,3) + return output_feat + + def compute_mesh_grid(self, in_h, in_w): + kernel_h, kernel_w = self.kernel_size + stride_h, stride_w = self.stride + dilation_h, dilation_w = self.dilation + padding_h, padding_w = self.padding + out_h = (in_h + 2 * padding_h - (dilation_h * (kernel_h - 1) + 1)) // stride_h + 1 + out_w = (in_w + 2 * padding_w - (dilation_w * (kernel_w - 1) + 1)) // stride_w + 1 + # [out_h, out_w] + mesh_y, mesh_x = torch.meshgrid(torch.arange(out_h), torch.arange(out_w)) + mesh_y = mesh_y * stride_h - padding_h + mesh_x = mesh_x * stride_w - padding_w + # [1, out_h, out_w] + mesh_y = mesh_y.unsqueeze(0).float() + mesh_x = mesh_x.unsqueeze(0).float() + # [kernel_h, kernel_w] + kernel_offset_y, kernel_offset_x = torch.meshgrid(torch.arange(kernel_h), torch.arange(kernel_w)) + # [kernel_h * kernel_w, 1, 1] + kernel_offset_y = kernel_offset_y.float().view(kernel_h * kernel_w, 1, 1) * dilation_h + kernel_offset_x = kernel_offset_x.float().view(kernel_h * kernel_w, 1, 1) * dilation_w + # [kernel_h * kernel_w, out_h, out_w] + mesh_y = mesh_y + kernel_offset_y + mesh_x = mesh_x + kernel_offset_x + mesh_y = (mesh_y - (in_h - 1) / 2.) / ((in_h - 1) / 2.) + mesh_x = (mesh_x - (in_w - 1) / 2.) / ((in_w - 1) / 2.) + mesh = torch.cat([mesh_x[None, ..., None], mesh_y[None, ..., None]], dim=4) + return mesh diff --git a/classification/mm_modules/DCN/functions/__init__.py b/classification/mm_modules/DCN/functions/__init__.py new file mode 100644 index 0000000..a80bf4d --- /dev/null +++ b/classification/mm_modules/DCN/functions/__init__.py @@ -0,0 +1,2 @@ +from .deform_conv2d_func import DeformConv2dFunction +from .modulated_deform_conv2d_func import ModulatedDeformConv2dFunction diff --git a/classification/mm_modules/DCN/functions/deform_conv2d_func.py b/classification/mm_modules/DCN/functions/deform_conv2d_func.py new file mode 100644 index 0000000..689c695 --- /dev/null +++ b/classification/mm_modules/DCN/functions/deform_conv2d_func.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import math +import torch +from torch import nn +from torch.autograd import Function +from torch.nn.modules.utils import _pair +from torch.autograd.function import once_differentiable +try: + from apex import amp +except ImportError: + raise ImportError("Please install apex from https://www.github.com/nvidia/apex to run this example.") +# from torch.cuda import amp +import DCN + +class DeformConv2dFunction(Function): + @staticmethod + @amp.float_function + def forward(ctx, input, offset, weight, bias, + stride, padding, dilation, group, deformable_groups, im2col_step): + ctx.stride = _pair(stride) + ctx.padding = _pair(padding) + ctx.dilation = _pair(dilation) + ctx.kernel_size = _pair(weight.shape[2:4]) + ctx.group = group + ctx.deformable_groups = deformable_groups + ctx.im2col_step = im2col_step + output = DCN.deform_conv2d_forward(input, weight, bias, + offset, + ctx.kernel_size[0], ctx.kernel_size[1], + ctx.stride[0], ctx.stride[1], + ctx.padding[0], ctx.padding[1], + ctx.dilation[0], ctx.dilation[1], + ctx.group, + ctx.deformable_groups, + ctx.im2col_step) + ctx.save_for_backward(input, offset, weight, bias) + return output + + @staticmethod + @once_differentiable + @amp.float_function + def backward(ctx, grad_output): + input, offset, weight, bias = ctx.saved_tensors + grad_input, grad_offset, grad_weight, grad_bias = \ + DCN.deform_conv2d_backward(input, weight, + bias, + offset, + grad_output, + ctx.kernel_size[0], ctx.kernel_size[1], + ctx.stride[0], ctx.stride[1], + ctx.padding[0], ctx.padding[1], + ctx.dilation[0], ctx.dilation[1], + ctx.group, + ctx.deformable_groups, + ctx.im2col_step) + + return grad_input, grad_offset, grad_weight, grad_bias,\ + None, None, None, None, None, None diff --git a/classification/mm_modules/DCN/functions/modulated_deform_conv2d_func.py b/classification/mm_modules/DCN/functions/modulated_deform_conv2d_func.py new file mode 100644 index 0000000..f52d807 --- /dev/null +++ b/classification/mm_modules/DCN/functions/modulated_deform_conv2d_func.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import math +import torch +from torch import nn +from torch.autograd import Function +from torch.nn.modules.utils import _pair +from torch.autograd.function import once_differentiable +# from torch.cuda import amp +import DCN +try: + from apex import amp +except ImportError: + raise ImportError("Please install apex from https://www.github.com/nvidia/apex to run this example.") + +class ModulatedDeformConv2dFunction(Function): + @staticmethod + @amp.float_function + def forward(ctx, input, offset, mask, weight, bias, + stride, padding, dilation, groups, deformable_groups, im2col_step): + ctx.stride = _pair(stride) + ctx.padding = _pair(padding) + ctx.dilation = _pair(dilation) + ctx.kernel_size = _pair(weight.shape[2:4]) + ctx.groups = groups + ctx.deformable_groups = deformable_groups + ctx.im2col_step = im2col_step + output = DCN.modulated_deform_conv2d_forward(input, weight, bias, + offset, mask, + ctx.kernel_size[0], ctx.kernel_size[1], + ctx.stride[0], ctx.stride[1], + ctx.padding[0], ctx.padding[1], + ctx.dilation[0], ctx.dilation[1], + ctx.groups, + ctx.deformable_groups, + ctx.im2col_step) + ctx.save_for_backward(input, offset, mask, weight, bias) + return output + + @staticmethod + @once_differentiable + @amp.float_function + def backward(ctx, grad_output): + input, offset, mask, weight, bias = ctx.saved_tensors + grad_input, grad_offset, grad_mask, grad_weight, grad_bias = \ + DCN.modulated_deform_conv2d_backward(input, weight, + bias, + offset, mask, + grad_output, + ctx.kernel_size[0], ctx.kernel_size[1], + ctx.stride[0], ctx.stride[1], + ctx.padding[0], ctx.padding[1], + ctx.dilation[0], ctx.dilation[1], + ctx.groups, + ctx.deformable_groups, + ctx.im2col_step) + + return grad_input, grad_offset, grad_mask, grad_weight, grad_bias,\ + None, None, None, None, None, None diff --git a/classification/mm_modules/DCN/modules/__init__.py b/classification/mm_modules/DCN/modules/__init__.py new file mode 100644 index 0000000..552cca0 --- /dev/null +++ b/classification/mm_modules/DCN/modules/__init__.py @@ -0,0 +1,2 @@ +from .deform_conv2d import DeformConv2d, _DeformConv2d, DeformConv2dPack, DeformConv2dPackMore +from .modulated_deform_conv2d import ModulatedDeformConv2d, _ModulatedDeformConv2d, ModulatedDeformConv2dPack diff --git a/classification/mm_modules/DCN/modules/deform_conv2d.py b/classification/mm_modules/DCN/modules/deform_conv2d.py new file mode 100644 index 0000000..edc12e3 --- /dev/null +++ b/classification/mm_modules/DCN/modules/deform_conv2d.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import torch +import math +from torch import nn +from torch.nn import init +from torch.nn.modules.utils import _pair + +from ..functions.deform_conv2d_func import DeformConv2dFunction + +class DeformConv2d(nn.Module): + + def __init__(self, in_channels, out_channels, + kernel_size, stride, padding, dilation=1, groups=1, deformable_groups=1, im2col_step=32, bias=True): + super(DeformConv2d, self).__init__() + + if in_channels % groups != 0: + raise ValueError('in_channels {} must be divisible by groups {}'.format(in_channels, groups)) + if out_channels % groups != 0: + raise ValueError('out_channels {} must be divisible by groups {}'.format(out_channels, groups)) + + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = _pair(kernel_size) + self.stride = _pair(stride) + self.padding = _pair(padding) + self.dilation = _pair(dilation) + self.groups = groups + self.deformable_groups = deformable_groups + self.im2col_step = im2col_step + self.use_bias = bias + + self.weight = nn.Parameter(torch.Tensor( + out_channels, in_channels//groups, *self.kernel_size)) + self.bias = nn.Parameter(torch.Tensor(out_channels)) + self.reset_parameters() + if not self.use_bias: + self.bias.requires_grad = False + self.bias.data.zero_() + + def reset_parameters(self): + n = self.in_channels + init.kaiming_uniform_(self.weight, a=math.sqrt(5)) + if self.bias is not None: + fan_in, _ = init._calculate_fan_in_and_fan_out(self.weight) + bound = 1 / math.sqrt(fan_in) + init.uniform_(self.bias, -bound, bound) + + def forward(self, input, offset): + assert 2 * self.deformable_groups * self.kernel_size[0] * self.kernel_size[1] == \ + offset.shape[1] + return DeformConv2dFunction.apply(input, offset, + self.weight, + self.bias, + self.stride, + self.padding, + self.dilation, + self.groups, + self.deformable_groups, + self.im2col_step) + +_DeformConv2d = DeformConv2dFunction.apply + +class DeformConv2dPack(DeformConv2d): + + def __init__(self, in_channels, out_channels, + kernel_size, stride, padding, + dilation=1, groups=1, deformable_groups=1, im2col_step=32, bias=True, lr_mult=0.1): + super(DeformConv2dPack, self).__init__(in_channels, out_channels, + kernel_size, stride, padding, dilation, groups, deformable_groups, im2col_step, bias) + + out_channels = self.deformable_groups * 2 * self.kernel_size[0] * self.kernel_size[1] + self.conv_offset = nn.Conv2d(self.in_channels, + out_channels, + kernel_size=self.kernel_size, + stride=self.stride, + padding=self.padding, + bias=True) + self.conv_offset.lr_mult = lr_mult + self.conv_offset.inited = True + self.init_offset() + + def init_offset(self): + self.conv_offset.weight.data.zero_() + self.conv_offset.bias.data.zero_() + + def forward(self, input, return_offset=False): + offset = self.conv_offset(input) + bs = input.size()[0] + im2col_step = bs // 2 if bs > 1 else 1 + out = DeformConv2dFunction.apply(input, offset, + self.weight, + self.bias, + self.stride, + self.padding, + self.dilation, + self.groups, + self.deformable_groups, + im2col_step) + if return_offset: + return out, offset + return out, None + + +class DeformConv2dPackMore(DeformConv2d): + + def __init__(self, in_channels, out_channels, + kernel_size, stride, padding, + dilation=1, groups=1, deformable_groups=1, im2col_step=64, bias=True, lr_mult=0.1): + super(DeformConv2dPackMore, self).__init__(in_channels, out_channels, + kernel_size, stride, padding, dilation, groups, deformable_groups, im2col_step, bias) + + out_channels = self.deformable_groups * 2 * self.kernel_size[0] * self.kernel_size[1] + self.conv_offset = nn.Sequential( + nn.Conv2d(self.in_channels, self.in_channels//4, kernel_size=1, bias=False), + nn.BatchNorm2d(self.in_channels//4), + nn.ReLU(inplace=True), + nn.Conv2d(self.in_channels//4, out_channels, kernel_size=self.kernel_size, stride=self.stride, padding=self.padding, bias=True) + ) + self.conv_offset[-1].lr_mult = lr_mult + self.conv_offset[-1].inited = True + self.init_offset() + + def init_offset(self): + self.conv_offset[-1].weight.data.zero_() + self.conv_offset[-1].bias.data.zero_() + + def forward(self, input): + offset = self.conv_offset(input) + bs = input.size()[0] + im2col_step = bs // 2 + return DeformConv2dFunction.apply(input, offset, + self.weight, + self.bias, + self.stride, + self.padding, + self.dilation, + self.groups, + self.deformable_groups, + im2col_step) diff --git a/classification/mm_modules/DCN/modules/modulated_deform_conv2d.py b/classification/mm_modules/DCN/modules/modulated_deform_conv2d.py new file mode 100644 index 0000000..2052051 --- /dev/null +++ b/classification/mm_modules/DCN/modules/modulated_deform_conv2d.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import torch +import math +from torch import nn +from torch.nn import init +from torch.nn.modules.utils import _pair + +from ..functions.modulated_deform_conv2d_func import ModulatedDeformConv2dFunction + +class ModulatedDeformConv2d(nn.Module): + + def __init__(self, in_channels, out_channels, + kernel_size, stride, padding, dilation=1, groups=1, deformable_groups=1, im2col_step=64, bias=True): + super(ModulatedDeformConv2d, self).__init__() + + if in_channels % groups != 0: + raise ValueError('in_channels {} must be divisible by groups {}'.format(in_channels, groups)) + if out_channels % groups != 0: + raise ValueError('out_channels {} must be divisible by groups {}'.format(out_channels, groups)) + + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = _pair(kernel_size) + self.stride = _pair(stride) + self.padding = _pair(padding) + self.dilation = _pair(dilation) + self.groups = groups + self.deformable_groups = deformable_groups + self.im2col_step = im2col_step + self.use_bias = bias + + self.weight = nn.Parameter(torch.Tensor( + out_channels, in_channels//groups, *self.kernel_size)) + self.bias = nn.Parameter(torch.Tensor(out_channels)) + self.reset_parameters() + if not self.use_bias: + self.bias.requires_grad = False + + def reset_parameters(self): + n = self.in_channels + init.kaiming_uniform_(self.weight, a=math.sqrt(5)) + if self.bias is not None: + fan_in, _ = init._calculate_fan_in_and_fan_out(self.weight) + bound = 1 / math.sqrt(fan_in) + init.uniform_(self.bias, -bound, bound) + + def forward(self, input, offset, mask): + assert 2 * self.deformable_groups * self.kernel_size[0] * self.kernel_size[1] == \ + offset.shape[1] + assert self.deformable_groups * self.kernel_size[0] * self.kernel_size[1] == \ + mask.shape[1] + return ModulatedDeformConv2dFunction.apply(input, offset, mask, + self.weight, + self.bias, + self.stride, + self.padding, + self.dilation, + self.groups, + self.deformable_groups, + self.im2col_step) + +_ModulatedDeformConv2d = ModulatedDeformConv2dFunction.apply + +class ModulatedDeformConv2dPack(ModulatedDeformConv2d): + + def __init__(self, in_channels, out_channels, + kernel_size, stride, padding, + dilation=1, groups=1, deformable_groups=1, im2col_step=64, bias=True, lr_mult=0.1): + super(ModulatedDeformConv2dPack, self).__init__(in_channels, out_channels, + kernel_size, stride, padding, dilation, groups, deformable_groups, im2col_step, bias) + + out_channels = self.deformable_groups * 3 * self.kernel_size[0] * self.kernel_size[1] + self.conv_offset_mask = nn.Conv2d(self.in_channels, + out_channels, + kernel_size=self.kernel_size, + stride=self.stride, + padding=self.padding, + bias=True) + self.conv_offset_mask.lr_mult = lr_mult + self.conv_offset_mask.inited = True + self.init_offset() + + def init_offset(self): + self.conv_offset_mask.weight.data.zero_() + self.conv_offset_mask.bias.data.zero_() + + def forward(self, input, return_offset=False): + out = self.conv_offset_mask(input) + o1, o2, mask = torch.chunk(out, 3, dim=1) + offset = torch.cat((o1, o2), dim=1) + mask = torch.sigmoid(mask) + + bs = input.size()[0] + im2col_step = bs // 2 + + out = ModulatedDeformConv2dFunction.apply(input, offset, mask, + self.weight, + self.bias, + self.stride, + self.padding, + self.dilation, + self.groups, + self.deformable_groups, + im2col_step) + if return_offset: + return out, offset + return out, None diff --git a/classification/mm_modules/DCN/setup.py b/classification/mm_modules/DCN/setup.py new file mode 100644 index 0000000..55cf53d --- /dev/null +++ b/classification/mm_modules/DCN/setup.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python + +import os +import glob + +import torch + +from torch.utils.cpp_extension import CUDA_HOME +from torch.utils.cpp_extension import CppExtension +from torch.utils.cpp_extension import CUDAExtension + +from setuptools import find_packages +from setuptools import setup + +requirements = ["torch", "torchvision"] + +def get_extensions(): + this_dir = os.path.dirname(os.path.abspath(__file__)) + extensions_dir = os.path.join(this_dir, "src") + + main_file = glob.glob(os.path.join(extensions_dir, "*.cpp")) + source_cpu = glob.glob(os.path.join(extensions_dir, "cpu", "*.cpp")) + source_cuda = glob.glob(os.path.join(extensions_dir, "cuda", "*.cu")) + + sources = main_file + source_cpu + extension = CppExtension + extra_compile_args = {"cxx": []} + define_macros = [] + + if torch.cuda.is_available() and CUDA_HOME is not None: + extension = CUDAExtension + sources += source_cuda + define_macros += [("WITH_CUDA", None)] + extra_compile_args["nvcc"] = [ + "-DCUDA_HAS_FP16=1", + "-D__CUDA_NO_HALF_OPERATORS__", + "-D__CUDA_NO_HALF_CONVERSIONS__", + "-D__CUDA_NO_HALF2_OPERATORS__", + ] + else: + raise NotImplementedError('Cuda is not availabel') + + sources = [os.path.join(extensions_dir, s) for s in sources] + include_dirs = [extensions_dir] + ext_modules = [ + extension( + "DCN", + sources, + include_dirs=include_dirs, + define_macros=define_macros, + extra_compile_args=extra_compile_args, + ) + ] + return ext_modules + +setup( + name="DCN", + version="1.0", + description="deformable convolutional networks", + packages=find_packages(exclude=("configs", "tests",)), + ext_modules=get_extensions(), + cmdclass={"build_ext": torch.utils.cpp_extension.BuildExtension}, +) diff --git a/classification/mm_modules/DCN/src/cpu/deform_conv2d_cpu.cpp b/classification/mm_modules/DCN/src/cpu/deform_conv2d_cpu.cpp new file mode 100644 index 0000000..64a67bb --- /dev/null +++ b/classification/mm_modules/DCN/src/cpu/deform_conv2d_cpu.cpp @@ -0,0 +1,47 @@ +#include + +#include +#include + + +at::Tensor +deform_conv2d_cpu_forward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + AT_ERROR("Not implement on cpu"); +} + +std::vector +deform_conv2d_cpu_backward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &grad_output, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + AT_ERROR("Not implement on cpu"); +} + diff --git a/classification/mm_modules/DCN/src/cpu/deform_conv2d_cpu.h b/classification/mm_modules/DCN/src/cpu/deform_conv2d_cpu.h new file mode 100644 index 0000000..585d3d8 --- /dev/null +++ b/classification/mm_modules/DCN/src/cpu/deform_conv2d_cpu.h @@ -0,0 +1,39 @@ +#pragma once +#include + +at::Tensor +deform_conv2d_cpu_forward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step); + +std::vector +deform_conv2d_cpu_backward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &grad_output, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step); + + diff --git a/classification/mm_modules/DCN/src/cpu/modulated_deform_conv2d_cpu.cpp b/classification/mm_modules/DCN/src/cpu/modulated_deform_conv2d_cpu.cpp new file mode 100644 index 0000000..b712a19 --- /dev/null +++ b/classification/mm_modules/DCN/src/cpu/modulated_deform_conv2d_cpu.cpp @@ -0,0 +1,49 @@ +#include + +#include +#include + + +at::Tensor +modulated_deform_conv2d_cpu_forward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &mask, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + AT_ERROR("Not implement on cpu"); +} + +std::vector +modulated_deform_conv2d_cpu_backward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &mask, + const at::Tensor &grad_output, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + AT_ERROR("Not implement on cpu"); +} + diff --git a/classification/mm_modules/DCN/src/cpu/modulated_deform_conv2d_cpu.h b/classification/mm_modules/DCN/src/cpu/modulated_deform_conv2d_cpu.h new file mode 100644 index 0000000..8f54d0e --- /dev/null +++ b/classification/mm_modules/DCN/src/cpu/modulated_deform_conv2d_cpu.h @@ -0,0 +1,41 @@ +#pragma once +#include + +at::Tensor +modulated_deform_conv2d_cpu_forward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &mask, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step); + +std::vector +modulated_deform_conv2d_cpu_backward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &mask, + const at::Tensor &grad_output, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step); + + diff --git a/classification/mm_modules/DCN/src/cuda/deform_2d_im2col_cuda.cuh b/classification/mm_modules/DCN/src/cuda/deform_2d_im2col_cuda.cuh new file mode 100644 index 0000000..266f3a2 --- /dev/null +++ b/classification/mm_modules/DCN/src/cuda/deform_2d_im2col_cuda.cuh @@ -0,0 +1,391 @@ +#include +#include +#include + +#include +#include + +// #include +#include +// #include + +#define CUDA_KERNEL_LOOP(i, n) \ + for (int i = blockIdx.x * blockDim.x + threadIdx.x; \ + i < (n); \ + i += blockDim.x * gridDim.x) + +const int CUDA_NUM_THREADS = 1024; +inline int GET_BLOCKS(const int N) +{ + return (N + CUDA_NUM_THREADS - 1) / CUDA_NUM_THREADS; +} + +template +__device__ scalar_t dmcn_2d_im2col_bilinear(const scalar_t *bottom_data, const int data_width, + const int height, const int width, scalar_t h, scalar_t w) +{ + int h_low = floor(h); + int w_low = floor(w); + int h_high = h_low + 1; + int w_high = w_low + 1; + + scalar_t lh = h - h_low; + scalar_t lw = w - w_low; + scalar_t hh = 1 - lh, hw = 1 - lw; + + scalar_t v1 = 0; + if (h_low >= 0 && w_low >= 0) + v1 = bottom_data[h_low * data_width + w_low]; + scalar_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) + v2 = bottom_data[h_low * data_width + w_high]; + scalar_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) + v3 = bottom_data[h_high * data_width + w_low]; + scalar_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) + v4 = bottom_data[h_high * data_width + w_high]; + + scalar_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + + scalar_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + return val; +} + +template +__device__ scalar_t dmcn_2d_get_gradient_weight(scalar_t argmax_h, scalar_t argmax_w, + const int h, const int w, const int height, const int width) +{ + if (argmax_h <= -1 || argmax_h >= height || argmax_w <= -1 || argmax_w >= width) + { + //empty + return 0; + } + + int argmax_h_low = floor(argmax_h); + int argmax_w_low = floor(argmax_w); + int argmax_h_high = argmax_h_low + 1; + int argmax_w_high = argmax_w_low + 1; + + scalar_t weight = 0; + if (h == argmax_h_low && w == argmax_w_low) + weight = (h + 1 - argmax_h) * (w + 1 - argmax_w); + if (h == argmax_h_low && w == argmax_w_high) + weight = (h + 1 - argmax_h) * (argmax_w + 1 - w); + if (h == argmax_h_high && w == argmax_w_low) + weight = (argmax_h + 1 - h) * (w + 1 - argmax_w); + if (h == argmax_h_high && w == argmax_w_high) + weight = (argmax_h + 1 - h) * (argmax_w + 1 - w); + return weight; +} + +template +__device__ scalar_t dmcn_2d_get_coordinate_weight(scalar_t argmax_h, scalar_t argmax_w, + const int height, const int width, const scalar_t *im_data, + const int data_width, const int bp_dir) +{ + if (argmax_h <= -1 || argmax_h >= height || argmax_w <= -1 || argmax_w >= width) + { + //empty + return 0; + } + + int argmax_h_low = floor(argmax_h); + int argmax_w_low = floor(argmax_w); + int argmax_h_high = argmax_h_low + 1; + int argmax_w_high = argmax_w_low + 1; + + scalar_t weight = 0; + + if (bp_dir == 0) + { + if (argmax_h_low >= 0 && argmax_w_low >= 0) + weight += -1 * (argmax_w_low + 1 - argmax_w) * im_data[argmax_h_low * data_width + argmax_w_low]; + if (argmax_h_low >= 0 && argmax_w_high <= width - 1) + weight += -1 * (argmax_w - argmax_w_low) * im_data[argmax_h_low * data_width + argmax_w_high]; + if (argmax_h_high <= height - 1 && argmax_w_low >= 0) + weight += (argmax_w_low + 1 - argmax_w) * im_data[argmax_h_high * data_width + argmax_w_low]; + if (argmax_h_high <= height - 1 && argmax_w_high <= width - 1) + weight += (argmax_w - argmax_w_low) * im_data[argmax_h_high * data_width + argmax_w_high]; + } + else if (bp_dir == 1) + { + if (argmax_h_low >= 0 && argmax_w_low >= 0) + weight += -1 * (argmax_h_low + 1 - argmax_h) * im_data[argmax_h_low * data_width + argmax_w_low]; + if (argmax_h_low >= 0 && argmax_w_high <= width - 1) + weight += (argmax_h_low + 1 - argmax_h) * im_data[argmax_h_low * data_width + argmax_w_high]; + if (argmax_h_high <= height - 1 && argmax_w_low >= 0) + weight += -1 * (argmax_h - argmax_h_low) * im_data[argmax_h_high * data_width + argmax_w_low]; + if (argmax_h_high <= height - 1 && argmax_w_high <= width - 1) + weight += (argmax_h - argmax_h_low) * im_data[argmax_h_high * data_width + argmax_w_high]; + } + + return weight; +} + +template +__global__ void deformable_2d_im2col_gpu_kernel(const int n, + const scalar_t *data_im, const scalar_t *data_offset, + const int height, const int width, const int kernel_h, + const int kernel_w, + const int pad_h, const int pad_w, + const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, const int num_channels, + const int deformable_group, + const int height_col, const int width_col, + scalar_t *data_col) +{ + // launch channels * batch_size * height_col * width_col cores + CUDA_KERNEL_LOOP(index, n) + { + // NOTE(CharlesShang): different from Dai Jifeng's MXNet implementation, col_buffer is of shape (c*kw*kh, N, oh, ow) + // here columns is of shape (N, c*kw*kh, oh * ow), need to adapt axis + // NOTE(Jiarui XU): different from CharlesShang's implementation, col_buffer is of shape (N, c*kw*kh, oh * ow) + // here columns is of shape (c*kw*kh, N, oh, ow), need to adapt axis + + // index index of output matrix + const int w_col = index % width_col; + const int h_col = (index / width_col) % height_col; + const int b_col = (index / width_col / height_col) % batch_size; + const int c_im = (index / width_col / height_col) / batch_size; + const int c_col = c_im * kernel_h * kernel_w; + + // compute deformable group index + const int deformable_group_index = c_im / channel_per_deformable_group; + + const int h_in = h_col * stride_h - pad_h; + const int w_in = w_col * stride_w - pad_w; + + scalar_t *data_col_ptr = data_col + ((c_col * batch_size + b_col) * height_col + h_col) * width_col + w_col; + // const scalar_t* data_im_ptr = data_im + ((b_col * num_channels + c_im) * height + h_in) * width + w_in; + const scalar_t *data_im_ptr = data_im + (b_col * num_channels + c_im) * height * width; + const scalar_t *data_offset_ptr = data_offset + (b_col * deformable_group + deformable_group_index) * 2 * kernel_h * kernel_w * height_col * width_col; + + for (int i = 0; i < kernel_h; ++i) + { + for (int j = 0; j < kernel_w; ++j) + { + const int data_offset_h_ptr = ((2 * (i * kernel_w + j)) * height_col + h_col) * width_col + w_col; + const int data_offset_w_ptr = ((2 * (i * kernel_w + j) + 1) * height_col + h_col) * width_col + w_col; + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + scalar_t val = static_cast(0); + const scalar_t h_im = h_in + i * dilation_h + offset_h; + const scalar_t w_im = w_in + j * dilation_w + offset_w; + if (h_im > -1 && w_im > -1 && h_im < height && w_im < width) + { + //const scalar_t map_h = i * dilation_h + offset_h; + //const scalar_t map_w = j * dilation_w + offset_w; + //const int cur_height = height - h_in; + //const int cur_width = width - w_in; + //val = dmcn_im2col_bilinear(data_im_ptr, width, cur_height, cur_width, map_h, map_w); + val = dmcn_2d_im2col_bilinear(data_im_ptr, width, height, width, h_im, w_im); + } + *data_col_ptr = val; + data_col_ptr += batch_size * height_col * width_col; + } + } + } +} + +template +__global__ void deformable_2d_col2im_gpu_kernel(const int n, + const scalar_t *data_col, const scalar_t *data_offset, + const int channels, const int height, const int width, + const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, + const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, const int deformable_group, + const int height_col, const int width_col, + scalar_t *grad_im) +{ + CUDA_KERNEL_LOOP(index, n) + { + const int j = (index / width_col / height_col / batch_size) % kernel_w; + const int i = (index / width_col / height_col / batch_size / kernel_w) % kernel_h; + const int c = index / width_col / height_col / batch_size / kernel_w / kernel_h; + // compute the start and end of the output + + const int deformable_group_index = c / channel_per_deformable_group; + + int w_out = index % width_col; + int h_out = (index / width_col) % height_col; + int b = (index / width_col / height_col) % batch_size; + int w_in = w_out * stride_w - pad_w; + int h_in = h_out * stride_h - pad_h; + + const scalar_t *data_offset_ptr = data_offset + (b * deformable_group + deformable_group_index) * 2 * kernel_h * kernel_w * height_col * width_col; + const int data_offset_h_ptr = ((2 * (i * kernel_w + j)) * height_col + h_out) * width_col + w_out; + const int data_offset_w_ptr = ((2 * (i * kernel_w + j) + 1) * height_col + h_out) * width_col + w_out; + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + const scalar_t cur_inv_h_data = h_in + i * dilation_h + offset_h; + const scalar_t cur_inv_w_data = w_in + j * dilation_w + offset_w; + + const scalar_t cur_top_grad = data_col[index]; + const int cur_h = (int)cur_inv_h_data; + const int cur_w = (int)cur_inv_w_data; + for (int dy = -2; dy <= 2; dy++) + { + for (int dx = -2; dx <= 2; dx++) + { + if (cur_h + dy >= 0 && cur_h + dy < height && + cur_w + dx >= 0 && cur_w + dx < width && + abs(cur_inv_h_data - (cur_h + dy)) < 1 && + abs(cur_inv_w_data - (cur_w + dx)) < 1) + { + int cur_bottom_grad_pos = ((b * channels + c) * height + cur_h + dy) * width + cur_w + dx; + scalar_t weight = dmcn_2d_get_gradient_weight(cur_inv_h_data, cur_inv_w_data, cur_h + dy, cur_w + dx, height, width); + atomicAdd(grad_im + cur_bottom_grad_pos, weight * cur_top_grad); + } + } + } + } +} + +template +__global__ void deformable_2d_col2im_coord_gpu_kernel(const int n, + const scalar_t *data_col, const scalar_t *data_im, + const scalar_t *data_offset, + const int channels, const int height, const int width, + const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, + const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, const int offset_channels, + const int deformable_group, + const int height_col, const int width_col, + scalar_t *grad_offset) +{ + CUDA_KERNEL_LOOP(index, n) + { + scalar_t val = 0; + int w = index % width_col; + int h = (index / width_col) % height_col; + int c = (index / width_col / height_col) % offset_channels; + int b = (index / width_col / height_col) / offset_channels; + // compute the start and end of the output + + const int deformable_group_index = c / (2 * kernel_h * kernel_w); + const int col_step = kernel_h * kernel_w; + int cnt = 0; + const scalar_t *data_col_ptr = data_col + deformable_group_index * channel_per_deformable_group * batch_size * width_col * height_col; + const scalar_t *data_im_ptr = data_im + (b * deformable_group + deformable_group_index) * channel_per_deformable_group / kernel_h / kernel_w * height * width; + const scalar_t *data_offset_ptr = data_offset + (b * deformable_group + deformable_group_index) * 2 * kernel_h * kernel_w * height_col * width_col; + + const int offset_c = c - deformable_group_index * 2 * kernel_h * kernel_w; + + for (int col_c = (offset_c / 2); col_c < channel_per_deformable_group; col_c += col_step) + { + const int col_pos = (((col_c * batch_size + b) * height_col) + h) * width_col + w; + const int bp_dir = offset_c % 2; + + int j = (col_pos / width_col / height_col / batch_size) % kernel_w; + int i = (col_pos / width_col / height_col / batch_size / kernel_w) % kernel_h; + int w_out = col_pos % width_col; + int h_out = (col_pos / width_col) % height_col; + int w_in = w_out * stride_w - pad_w; + int h_in = h_out * stride_h - pad_h; + const int data_offset_h_ptr = (((2 * (i * kernel_w + j)) * height_col + h_out) * width_col + w_out); + const int data_offset_w_ptr = (((2 * (i * kernel_w + j) + 1) * height_col + h_out) * width_col + w_out); + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + scalar_t inv_h = h_in + i * dilation_h + offset_h; + scalar_t inv_w = w_in + j * dilation_w + offset_w; + if (inv_h <= -1 || inv_w <= -1 || inv_h >= height || inv_w >= width) + { + inv_h = inv_w = -2; + } + const scalar_t weight = dmcn_2d_get_coordinate_weight( + inv_h, inv_w, + height, width, data_im_ptr + cnt * height * width, width, bp_dir); + val += weight * data_col_ptr[col_pos]; + cnt += 1; + } + // KERNEL_ASSIGN(grad_offset[index], offset_req, val); + grad_offset[index] = val; + } +} + +template +void deformable_2d_im2col_cuda(cudaStream_t stream, + const scalar_t *data_im, const scalar_t *data_offset, + const int batch_size, const int channels, const int height_im, const int width_im, + const int height_col, const int width_col, const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int deformable_group, scalar_t *data_col) { + // num_axes should be smaller than block size + const int channel_per_deformable_group = channels / deformable_group; + const int num_kernels = channels * batch_size * height_col * width_col; + deformable_2d_im2col_gpu_kernel + <<>>( + num_kernels, data_im, data_offset, height_im, width_im, kernel_h, kernel_w, + pad_h, pad_w, stride_h, stride_w, dilation_h, dilation_w, channel_per_deformable_group, + batch_size, channels, deformable_group, height_col, width_col, data_col); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in deformable_im2col_cuda: %s\n", cudaGetErrorString(err)); + } + +} + +template +void deformable_2d_col2im_cuda(cudaStream_t stream, + const scalar_t *data_col, const scalar_t *data_offset, + const int batch_size, const int channels, const int height_im, const int width_im, + const int height_col, const int width_col, const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int deformable_group, scalar_t *grad_im){ + + const int channel_per_deformable_group = channels / deformable_group; + const int num_kernels = channels * kernel_h * kernel_w * batch_size * height_col * width_col; + deformable_2d_col2im_gpu_kernel + <<>>( + num_kernels, data_col, data_offset, channels, height_im, width_im, + kernel_h, kernel_w, pad_h, pad_h, stride_h, stride_w, + dilation_h, dilation_w, channel_per_deformable_group, + batch_size, deformable_group, height_col, width_col, grad_im); + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in deformable_col2im_cuda: %s\n", cudaGetErrorString(err)); + } + +} + +template +void deformable_2d_col2im_coord_cuda(cudaStream_t stream, + const scalar_t *data_col, const scalar_t *data_im, const scalar_t *data_offset, + const int batch_size, const int channels, const int height_im, const int width_im, + const int height_col, const int width_col, const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int deformable_group, + scalar_t *grad_offset) { + const int num_kernels = batch_size * height_col * width_col * 2 * kernel_h * kernel_w * deformable_group; + const int channel_per_deformable_group = channels * kernel_h * kernel_w / deformable_group; + deformable_2d_col2im_coord_gpu_kernel + <<>>( + num_kernels, data_col, data_im, data_offset, channels, height_im, width_im, + kernel_h, kernel_w, pad_h, pad_w, stride_h, stride_w, + dilation_h, dilation_w, channel_per_deformable_group, + batch_size, 2 * kernel_h * kernel_w * deformable_group, deformable_group, height_col, width_col, + grad_offset); + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in deformable_col2im_coord_cuda: %s\n", cudaGetErrorString(err)); + } +} \ No newline at end of file diff --git a/classification/mm_modules/DCN/src/cuda/deform_conv2d_cuda.cu b/classification/mm_modules/DCN/src/cuda/deform_conv2d_cuda.cu new file mode 100644 index 0000000..0493211 --- /dev/null +++ b/classification/mm_modules/DCN/src/cuda/deform_conv2d_cuda.cu @@ -0,0 +1,273 @@ +#include +#include "cuda/deform_2d_im2col_cuda.cuh" + +#include +#include +#include +#include + +// #include +// #include +// #include + +// extern THCState *state; + +// author: Charles Shang +// https://github.com/torch/cunn/blob/master/lib/THCUNN/generic/SpatialConvolutionMM.cu + + +at::Tensor +deform_conv2d_cuda_forward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + // THCAssertSameGPU(THCudaTensor_checkGPU(state, 5, input, weight, bias, offset, mask)); + + AT_ASSERTM(input.is_contiguous(), "input tensor has to be contiguous"); + AT_ASSERTM(weight.is_contiguous(), "weight tensor has to be contiguous"); + + AT_ASSERTM(input.type().is_cuda(), "input must be a CUDA tensor"); + AT_ASSERTM(weight.type().is_cuda(), "weight must be a CUDA tensor"); + AT_ASSERTM(bias.type().is_cuda(), "bias must be a CUDA tensor"); + AT_ASSERTM(offset.type().is_cuda(), "offset must be a CUDA tensor"); + + const int batch = input.size(0); + const int channels = input.size(1); + const int height = input.size(2); + const int width = input.size(3); + + const int channels_out = weight.size(0); + const int channels_kernel = weight.size(1); + const int kernel_h_ = weight.size(2); + const int kernel_w_ = weight.size(3); + + const int im2col_step_ = std::min(batch, im2col_step); + // if (batch % im2col_step_ != 0) { + // printf("batch: %d im2col_step_: %d\n", batch, im2col_step_); + // } + + AT_ASSERTM(batch % im2col_step_ == 0, "batch(%d) must divide im2col_step(%d)", batch, im2col_step_); + AT_ASSERTM((channels % group == 0) && (channels_out % group == 0), + "channels(%d) and channels_out(%d) must divide group(%d)", channels, channels_out, group); + + // printf("Kernels: %d %d %d %d\n", kernel_h_, kernel_w_, kernel_w, kernel_h); + // printf("Channels: %d %d\n", channels, channels_kernel); + // printf("Channels: %d %d\n", channels_out, channels_kernel); + + AT_ASSERTM(kernel_h_ == kernel_h && kernel_w_ == kernel_w, + "Input shape and kernel shape wont match: (%d x %d vs %d x %d).", kernel_h_, kernel_w, kernel_h_, kernel_w_); + + AT_ASSERTM(channels == (channels_kernel * group), + "Input shape and kernel channels wont match: (%d vs %d).", channels, channels_kernel * group); + + const int height_out = (height + 2 * pad_h - (dilation_h * (kernel_h - 1) + 1)) / stride_h + 1; + const int width_out = (width + 2 * pad_w - (dilation_w * (kernel_w - 1) + 1)) / stride_w + 1; + + auto output = at::empty({batch * height_out * width_out, channels_out}, input.options()); + + // prepare group weight and bias + auto weight_g = weight.view({group, channels_out/group, channels_kernel, kernel_h, kernel_w}); + auto bias_g = bias.view({group, channels_out/group}); + + // define alias for easy use + const int batch_n = im2col_step_; + const int per_input_size = channels * height * width; + const int per_offset_size = offset.size(1) * offset.size(2) * offset.size(3); + auto output_n = output.view({batch/im2col_step_, batch_n * height_out * width_out, channels_out}); + for (int n = 0; n < batch/im2col_step_; ++n) + { + auto columns = at::empty({channels * kernel_h * kernel_w, batch_n * height_out * width_out}, input.options()); + AT_DISPATCH_FLOATING_TYPES(input.type(), "deform_conv_forward_cuda", ([&] { + deformable_2d_im2col_cuda(at::cuda::getCurrentCUDAStream(), + input.data() + n * im2col_step_ * per_input_size, + offset.data() + n * im2col_step_ * per_offset_size, + batch_n, channels, height, width, + height_out, width_out, kernel_h, kernel_w, + pad_h, pad_w, stride_h, stride_w, dilation_h, dilation_w, + deformable_group, + columns.data()); + + })); + + // auto columns_m = columns.t(); + // auto weight_m = weight.view({channels_out, channels_kernel * kernel_h * kernel_w}).t(); + // output = at::addmm(bias, columns_m, weight_m); + auto columns_g = columns.view({group, channels/group * kernel_h * kernel_w, batch_n * height_out * width_out}); + auto output_g = output_n.select(0, n).view({batch_n * height_out * width_out, group, channels_out/group}); + for (int g = 0; g < group; ++g) + { + auto columns_gm = columns_g.select(0, g).t(); + auto weight_gm = weight_g.select(0, g).view({channels_out/group, channels_kernel * kernel_h * kernel_w}).t(); + auto output_m = at::addmm(bias_g.select(0, g), columns_gm, weight_gm); + output_g.select(1, g) = output_m.view({batch_n * height_out * width_out, channels_out/group}); + } + + } + + output = output.view({batch, height_out, width_out, channels_out}).permute({0, 3, 1, 2}).contiguous(); + + return output; +} + +std::vector deform_conv2d_cuda_backward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &grad_output, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + + AT_ASSERTM(input.is_contiguous(), "input tensor has to be contiguous"); + AT_ASSERTM(weight.is_contiguous(), "weight tensor has to be contiguous"); + + AT_ASSERTM(input.type().is_cuda(), "input must be a CUDA tensor"); + AT_ASSERTM(weight.type().is_cuda(), "weight must be a CUDA tensor"); + AT_ASSERTM(bias.type().is_cuda(), "bias must be a CUDA tensor"); + AT_ASSERTM(offset.type().is_cuda(), "offset must be a CUDA tensor"); + + const int batch = input.size(0); + const int channels = input.size(1); + const int height = input.size(2); + const int width = input.size(3); + + const int channels_out = weight.size(0); + const int channels_kernel = weight.size(1); + const int kernel_h_ = weight.size(2); + const int kernel_w_ = weight.size(3); + + const int batch_ = grad_output.size(0); + const int channels_out_ = grad_output.size(1); + const int height_out_ = grad_output.size(2); + const int width_out_ = grad_output.size(3); + + const int im2col_step_ = std::min(im2col_step, batch); + + AT_ASSERTM(batch % im2col_step_ == 0, "batch(%d) must divide im2col_step(%d)", batch, im2col_step_); + + AT_ASSERTM((channels % group == 0) && (channels_out % group == 0), + "channels(%d) and channels_out(%d) must divide group(%d)", channels, channels_out, group); + + AT_ASSERTM(kernel_h_ == kernel_h && kernel_w_ == kernel_w, + "Input shape and kernel shape wont match: (%d x %d vs %d x %d).", kernel_h_, kernel_w, kernel_h_, kernel_w_); + + AT_ASSERTM(channels == (channels_kernel * group), + "Input shape and kernel channels wont match: (%d vs %d).", channels, channels_kernel * group); + + const int height_out = (height + 2 * pad_h - (dilation_h * (kernel_h - 1) + 1)) / stride_h + 1; + const int width_out = (width + 2 * pad_w - (dilation_w * (kernel_w - 1) + 1)) / stride_w + 1; + + AT_ASSERTM(batch == batch_, + "Input shape and grad_out batch wont match: (%d vs %d).", batch, batch_); + + AT_ASSERTM(channels_out == channels_out_, + "Input shape and grad_out channels_out wont match: (%d vs %d).", channels_out, channels_out_); + + AT_ASSERTM(height_out == height_out_ && width_out == width_out_, + "Input shape and grad_out shape wont match: (%d x %d vs %d x %d).", height_out, height_out_, width_out, width_out_); + + auto grad_input = at::zeros_like(input); + auto grad_offset = at::zeros_like(offset); + auto grad_weight = at::zeros_like(weight); + auto grad_bias = at::zeros_like(bias); + + // auto grad_output_m = grad_output.permute({1, 0, 2, 3}).contiguous().view({channels_out, batch * height_out * width_out}); + // auto weight_m = weight.view({channels_out, channels_kernel * kernel_h * kernel_w}).t(); + // columns = at::mm(weight_m, grad_output_m); + + // prepare group weight and bias + auto weight_g = weight.view({group, channels_out/group, channels_kernel, kernel_h, kernel_w}); + auto grad_weight_g = grad_weight.view({group, channels_out/group, channels_kernel, kernel_h, kernel_w}); + auto grad_bias_g = grad_bias.view({group, channels_out/group}); + + const int batch_n = im2col_step_; + const int per_input_size = channels * height * width; + const int per_offset_size = offset.size(1) * offset.size(2) * offset.size(3); + auto grad_output_n = grad_output.view({batch/im2col_step_, batch_n, channels_out, height_out, width_out}); + for (int n = 0; n < batch/im2col_step_; ++n) + { + auto grad_output_g = grad_output_n.select(0, n).view({batch_n, group, channels_out/group, height_out, width_out}); + auto ones = at::ones({batch_n * height_out * width_out}, input.options()); + auto columns = at::empty({channels * kernel_h * kernel_w, batch_n * 1 * height_out * width_out}, input.options()); + auto columns_g = columns.view({group, channels/group * kernel_h * kernel_w, batch_n * height_out * width_out}); + for (int g = 0; g < group; ++g) + { + auto grad_output_gm = grad_output_g.select(1, g).permute({1, 0, 2, 3}).contiguous().view({channels_out/group, batch_n * height_out * width_out}); + auto weight_gm = weight_g.select(0, g).view({channels_out/group, channels_kernel * kernel_h * kernel_w}).t(); + columns_g.select(0, g) = at::mm(weight_gm, grad_output_gm); + } + + AT_DISPATCH_FLOATING_TYPES(input.type(), "deform_conv_backward_cuda", ([&] { + deformable_2d_col2im_coord_cuda(at::cuda::getCurrentCUDAStream(), + columns.data(), + input.data() + n * im2col_step_ * per_input_size, + offset.data() + n * im2col_step_ * per_offset_size, + batch_n, channels, height, width, + height_out, width_out, kernel_h, kernel_w, + pad_h, pad_w, stride_h, stride_w, + dilation_h, dilation_w, deformable_group, + grad_offset.data() + n * im2col_step_ * per_offset_size); + // gradient w.r.t. input data + deformable_2d_col2im_cuda(at::cuda::getCurrentCUDAStream(), + columns.data(), + offset.data() + n * im2col_step_ * per_offset_size, + batch_n, channels, height, width, + height_out, width_out, kernel_h, kernel_w, + pad_h, pad_w, stride_h, stride_w, + dilation_h, dilation_w, deformable_group, + grad_input.data() + n * im2col_step_ * per_input_size); + + // gradient w.r.t. weight, dWeight should accumulate across the batch and group + deformable_2d_im2col_cuda(at::cuda::getCurrentCUDAStream(), + input.data() + n * im2col_step_ * per_input_size, + offset.data() + n * im2col_step_ * per_offset_size, + batch_n, channels, height, width, + height_out, width_out, kernel_h, kernel_w, + pad_h, pad_w, stride_h, stride_w, + dilation_h, dilation_w, deformable_group, + columns.data()); + + })); + + // auto grad_output_m = grad_output.permute({1, 0, 2, 3}).contiguous().view({channels_out, batch * height_out * width_out}); + // grad_weight = at::mm(grad_output_m, columns.t()).view_as(weight); + // grad_bias = at::mv(grad_output_m, ones); + // auto grad_output_g = grad_output.view({batch, group, channels_out/group, height_out, width_out}); + // auto columns_g = columns.view({group, channels/group * kernel_h * kernel_w, batch * height_out * width_out}); + for (int g = 0; g < group; ++g) + { + auto grad_output_gm = grad_output_g.select(1, g).permute({1, 0, 2, 3}).contiguous().view({channels_out/group, batch_n * height_out * width_out}); + auto columns_gm = columns_g.select(0, g).t(); + auto grad_weight_gm = grad_weight_g.select(0, g).view({channels_out/group, channels_kernel * kernel_h * kernel_w}); + auto grad_bias_gm = grad_bias_g.select(0, g); + grad_weight_g.select(0, g) = at::addmm(grad_weight_gm, grad_output_gm, columns_gm).view_as(grad_weight_g.select(0, g)); + grad_bias_g.select(0, g) = at::addmv(grad_bias_gm, grad_output_gm, ones); + } + + } + + return { + grad_input, grad_offset, grad_weight, grad_bias + }; +} diff --git a/classification/mm_modules/DCN/src/cuda/deform_conv2d_cuda.h b/classification/mm_modules/DCN/src/cuda/deform_conv2d_cuda.h new file mode 100644 index 0000000..0958453 --- /dev/null +++ b/classification/mm_modules/DCN/src/cuda/deform_conv2d_cuda.h @@ -0,0 +1,38 @@ +#pragma once +#include + +at::Tensor +deform_conv2d_cuda_forward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step); + +std::vector +deform_conv2d_cuda_backward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &grad_output, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step); + diff --git a/classification/mm_modules/DCN/src/cuda/modulated_deform_2d_im2col_cuda.cuh b/classification/mm_modules/DCN/src/cuda/modulated_deform_2d_im2col_cuda.cuh new file mode 100644 index 0000000..0ff18cb --- /dev/null +++ b/classification/mm_modules/DCN/src/cuda/modulated_deform_2d_im2col_cuda.cuh @@ -0,0 +1,420 @@ +#include +#include +#include + +#include +#include + +// #include +#include +// #include + +#define CUDA_KERNEL_LOOP(i, n) \ + for (int i = blockIdx.x * blockDim.x + threadIdx.x; \ + i < (n); \ + i += blockDim.x * gridDim.x) + +const int CUDA_NUM_THREADS = 1024; +inline int GET_BLOCKS(const int N) +{ + return (N + CUDA_NUM_THREADS - 1) / CUDA_NUM_THREADS; +} + + +template +__device__ scalar_t mdmcn_2d_im2col_bilinear(const scalar_t *bottom_data, const int data_width, + const int height, const int width, scalar_t h, scalar_t w) +{ + int h_low = floor(h); + int w_low = floor(w); + int h_high = h_low + 1; + int w_high = w_low + 1; + + scalar_t lh = h - h_low; + scalar_t lw = w - w_low; + scalar_t hh = 1 - lh, hw = 1 - lw; + + scalar_t v1 = 0; + if (h_low >= 0 && w_low >= 0) + v1 = bottom_data[h_low * data_width + w_low]; + scalar_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) + v2 = bottom_data[h_low * data_width + w_high]; + scalar_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) + v3 = bottom_data[h_high * data_width + w_low]; + scalar_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) + v4 = bottom_data[h_high * data_width + w_high]; + + scalar_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + + scalar_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + return val; +} + +template +__device__ scalar_t mdmcn_2d_get_gradient_weight(scalar_t argmax_h, scalar_t argmax_w, + const int h, const int w, const int height, const int width) +{ + if (argmax_h <= -1 || argmax_h >= height || argmax_w <= -1 || argmax_w >= width) + { + //empty + return 0; + } + + int argmax_h_low = floor(argmax_h); + int argmax_w_low = floor(argmax_w); + int argmax_h_high = argmax_h_low + 1; + int argmax_w_high = argmax_w_low + 1; + + scalar_t weight = 0; + if (h == argmax_h_low && w == argmax_w_low) + weight = (h + 1 - argmax_h) * (w + 1 - argmax_w); + if (h == argmax_h_low && w == argmax_w_high) + weight = (h + 1 - argmax_h) * (argmax_w + 1 - w); + if (h == argmax_h_high && w == argmax_w_low) + weight = (argmax_h + 1 - h) * (w + 1 - argmax_w); + if (h == argmax_h_high && w == argmax_w_high) + weight = (argmax_h + 1 - h) * (argmax_w + 1 - w); + return weight; +} + +template +__device__ scalar_t mdmcn_2d_get_coordinate_weight(scalar_t argmax_h, scalar_t argmax_w, + const int height, const int width, const scalar_t *im_data, + const int data_width, const int bp_dir) +{ + if (argmax_h <= -1 || argmax_h >= height || argmax_w <= -1 || argmax_w >= width) + { + //empty + return 0; + } + + int argmax_h_low = floor(argmax_h); + int argmax_w_low = floor(argmax_w); + int argmax_h_high = argmax_h_low + 1; + int argmax_w_high = argmax_w_low + 1; + + scalar_t weight = 0; + + if (bp_dir == 0) + { + if (argmax_h_low >= 0 && argmax_w_low >= 0) + weight += -1 * (argmax_w_low + 1 - argmax_w) * im_data[argmax_h_low * data_width + argmax_w_low]; + if (argmax_h_low >= 0 && argmax_w_high <= width - 1) + weight += -1 * (argmax_w - argmax_w_low) * im_data[argmax_h_low * data_width + argmax_w_high]; + if (argmax_h_high <= height - 1 && argmax_w_low >= 0) + weight += (argmax_w_low + 1 - argmax_w) * im_data[argmax_h_high * data_width + argmax_w_low]; + if (argmax_h_high <= height - 1 && argmax_w_high <= width - 1) + weight += (argmax_w - argmax_w_low) * im_data[argmax_h_high * data_width + argmax_w_high]; + } + else if (bp_dir == 1) + { + if (argmax_h_low >= 0 && argmax_w_low >= 0) + weight += -1 * (argmax_h_low + 1 - argmax_h) * im_data[argmax_h_low * data_width + argmax_w_low]; + if (argmax_h_low >= 0 && argmax_w_high <= width - 1) + weight += (argmax_h_low + 1 - argmax_h) * im_data[argmax_h_low * data_width + argmax_w_high]; + if (argmax_h_high <= height - 1 && argmax_w_low >= 0) + weight += -1 * (argmax_h - argmax_h_low) * im_data[argmax_h_high * data_width + argmax_w_low]; + if (argmax_h_high <= height - 1 && argmax_w_high <= width - 1) + weight += (argmax_h - argmax_h_low) * im_data[argmax_h_high * data_width + argmax_w_high]; + } + + return weight; +} + +template +__global__ void modulated_deformable_2d_im2col_gpu_kernel(const int n, + const scalar_t *data_im, const scalar_t *data_offset, + const scalar_t *data_mask, + const int height, const int width, const int kernel_h, + const int kernel_w, + const int pad_h, const int pad_w, + const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, const int num_channels, + const int deformable_group, + const int height_col, const int width_col, + scalar_t *data_col) +{ + // launch channels * batch_size * height_col * width_col cores + CUDA_KERNEL_LOOP(index, n) + { + // NOTE(CharlesShang): different from Dai Jifeng's MXNet implementation, col_buffer is of shape (c*kw*kh, N, oh, ow) + // here columns is of shape (N, c*kw*kh, oh * ow), need to adapt axis + // NOTE(Jiarui XU): different from CharlesShang's implementation, col_buffer is of shape (N, c*kw*kh, oh * ow) + // here columns is of shape (c*kw*kh, N, oh, ow), need to adapt axis + + // index index of output matrix + const int w_col = index % width_col; + const int h_col = (index / width_col) % height_col; + const int b_col = (index / width_col / height_col) % batch_size; + const int c_im = (index / width_col / height_col) / batch_size; + const int c_col = c_im * kernel_h * kernel_w; + + // compute deformable group index + const int deformable_group_index = c_im / channel_per_deformable_group; + + const int h_in = h_col * stride_h - pad_h; + const int w_in = w_col * stride_w - pad_w; + + scalar_t *data_col_ptr = data_col + ((c_col * batch_size + b_col) * height_col + h_col) * width_col + w_col; + //const scalar_t* data_im_ptr = data_im + ((b_col * num_channels + c_im) * height + h_in) * width + w_in; + const scalar_t *data_im_ptr = data_im + (b_col * num_channels + c_im) * height * width; + const scalar_t *data_offset_ptr = data_offset + (b_col * deformable_group + deformable_group_index) * 2 * kernel_h * kernel_w * height_col * width_col; + + const scalar_t *data_mask_ptr = data_mask + (b_col * deformable_group + deformable_group_index) * kernel_h * kernel_w * height_col * width_col; + + for (int i = 0; i < kernel_h; ++i) + { + for (int j = 0; j < kernel_w; ++j) + { + const int data_offset_h_ptr = ((2 * (i * kernel_w + j)) * height_col + h_col) * width_col + w_col; + const int data_offset_w_ptr = ((2 * (i * kernel_w + j) + 1) * height_col + h_col) * width_col + w_col; + const int data_mask_hw_ptr = ((i * kernel_w + j) * height_col + h_col) * width_col + w_col; + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + const scalar_t mask = data_mask_ptr[data_mask_hw_ptr]; + scalar_t val = static_cast(0); + const scalar_t h_im = h_in + i * dilation_h + offset_h; + const scalar_t w_im = w_in + j * dilation_w + offset_w; + if (h_im > -1 && w_im > -1 && h_im < height && w_im < width) + { + //const scalar_t map_h = i * dilation_h + offset_h; + //const scalar_t map_w = j * dilation_w + offset_w; + //const int cur_height = height - h_in; + //const int cur_width = width - w_in; + //val = dmcn_im2col_bilinear(data_im_ptr, width, cur_height, cur_width, map_h, map_w); + val = mdmcn_2d_im2col_bilinear(data_im_ptr, width, height, width, h_im, w_im); + } + *data_col_ptr = val * mask; + data_col_ptr += batch_size * height_col * width_col; + } + } + } +} + +template +__global__ void modulated_deformable_2d_col2im_gpu_kernel(const int n, + const scalar_t *data_col, const scalar_t *data_offset, + const scalar_t *data_mask, + const int channels, const int height, const int width, + const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, + const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, const int deformable_group, + const int height_col, const int width_col, + scalar_t *grad_im) +{ + CUDA_KERNEL_LOOP(index, n) + { + const int j = (index / width_col / height_col / batch_size) % kernel_w; + const int i = (index / width_col / height_col / batch_size / kernel_w) % kernel_h; + const int c = index / width_col / height_col / batch_size / kernel_w / kernel_h; + // compute the start and end of the output + + const int deformable_group_index = c / channel_per_deformable_group; + + int w_out = index % width_col; + int h_out = (index / width_col) % height_col; + int b = (index / width_col / height_col) % batch_size; + int w_in = w_out * stride_w - pad_w; + int h_in = h_out * stride_h - pad_h; + + const scalar_t *data_offset_ptr = data_offset + (b * deformable_group + deformable_group_index) * 2 * kernel_h * kernel_w * height_col * width_col; + const scalar_t *data_mask_ptr = data_mask + (b * deformable_group + deformable_group_index) * kernel_h * kernel_w * height_col * width_col; + const int data_offset_h_ptr = ((2 * (i * kernel_w + j)) * height_col + h_out) * width_col + w_out; + const int data_offset_w_ptr = ((2 * (i * kernel_w + j) + 1) * height_col + h_out) * width_col + w_out; + const int data_mask_hw_ptr = ((i * kernel_w + j) * height_col + h_out) * width_col + w_out; + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + const scalar_t mask = data_mask_ptr[data_mask_hw_ptr]; + const scalar_t cur_inv_h_data = h_in + i * dilation_h + offset_h; + const scalar_t cur_inv_w_data = w_in + j * dilation_w + offset_w; + + const scalar_t cur_top_grad = data_col[index] * mask; + const int cur_h = (int)cur_inv_h_data; + const int cur_w = (int)cur_inv_w_data; + for (int dy = -2; dy <= 2; dy++) + { + for (int dx = -2; dx <= 2; dx++) + { + if (cur_h + dy >= 0 && cur_h + dy < height && + cur_w + dx >= 0 && cur_w + dx < width && + abs(cur_inv_h_data - (cur_h + dy)) < 1 && + abs(cur_inv_w_data - (cur_w + dx)) < 1) + { + int cur_bottom_grad_pos = ((b * channels + c) * height + cur_h + dy) * width + cur_w + dx; + scalar_t weight = mdmcn_2d_get_gradient_weight(cur_inv_h_data, cur_inv_w_data, cur_h + dy, cur_w + dx, height, width); + atomicAdd(grad_im + cur_bottom_grad_pos, weight * cur_top_grad); + } + } + } + } +} + +template +__global__ void modulated_deformable_2d_col2im_coord_gpu_kernel(const int n, + const scalar_t *data_col, const scalar_t *data_im, + const scalar_t *data_offset, const scalar_t *data_mask, + const int channels, const int height, const int width, + const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, + const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, const int offset_channels, + const int deformable_group, + const int height_col, const int width_col, + scalar_t *grad_offset, scalar_t *grad_mask) +{ + CUDA_KERNEL_LOOP(index, n) + { + scalar_t val = 0, mval = 0; + int w = index % width_col; + int h = (index / width_col) % height_col; + int c = (index / width_col / height_col) % offset_channels; + int b = (index / width_col / height_col) / offset_channels; + // compute the start and end of the output + + const int deformable_group_index = c / (2 * kernel_h * kernel_w); + const int col_step = kernel_h * kernel_w; + int cnt = 0; + const scalar_t *data_col_ptr = data_col + deformable_group_index * channel_per_deformable_group * batch_size * width_col * height_col; + const scalar_t *data_im_ptr = data_im + (b * deformable_group + deformable_group_index) * channel_per_deformable_group / kernel_h / kernel_w * height * width; + const scalar_t *data_offset_ptr = data_offset + (b * deformable_group + deformable_group_index) * 2 * kernel_h * kernel_w * height_col * width_col; + const scalar_t *data_mask_ptr = data_mask + (b * deformable_group + deformable_group_index) * kernel_h * kernel_w * height_col * width_col; + + const int offset_c = c - deformable_group_index * 2 * kernel_h * kernel_w; + + for (int col_c = (offset_c / 2); col_c < channel_per_deformable_group; col_c += col_step) + { + const int col_pos = (((col_c * batch_size + b) * height_col) + h) * width_col + w; + const int bp_dir = offset_c % 2; + + int j = (col_pos / width_col / height_col / batch_size) % kernel_w; + int i = (col_pos / width_col / height_col / batch_size / kernel_w) % kernel_h; + int w_out = col_pos % width_col; + int h_out = (col_pos / width_col) % height_col; + int w_in = w_out * stride_w - pad_w; + int h_in = h_out * stride_h - pad_h; + const int data_offset_h_ptr = (((2 * (i * kernel_w + j)) * height_col + h_out) * width_col + w_out); + const int data_offset_w_ptr = (((2 * (i * kernel_w + j) + 1) * height_col + h_out) * width_col + w_out); + const int data_mask_hw_ptr = (((i * kernel_w + j) * height_col + h_out) * width_col + w_out); + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + const scalar_t mask = data_mask_ptr[data_mask_hw_ptr]; + scalar_t inv_h = h_in + i * dilation_h + offset_h; + scalar_t inv_w = w_in + j * dilation_w + offset_w; + if (inv_h <= -1 || inv_w <= -1 || inv_h >= height || inv_w >= width) + { + inv_h = inv_w = -2; + } + else + { + mval += data_col_ptr[col_pos] * mdmcn_2d_im2col_bilinear(data_im_ptr + cnt * height * width, width, height, width, inv_h, inv_w); + } + const scalar_t weight = mdmcn_2d_get_coordinate_weight( + inv_h, inv_w, + height, width, data_im_ptr + cnt * height * width, width, bp_dir); + val += weight * data_col_ptr[col_pos] * mask; + cnt += 1; + } + // KERNEL_ASSIGN(grad_offset[index], offset_req, val); + grad_offset[index] = val; + if (offset_c % 2 == 0) + // KERNEL_ASSIGN(grad_mask[(((b * deformable_group + deformable_group_index) * kernel_h * kernel_w + offset_c / 2) * height_col + h) * width_col + w], mask_req, mval); + grad_mask[(((b * deformable_group + deformable_group_index) * kernel_h * kernel_w + offset_c / 2) * height_col + h) * width_col + w] = mval; + } +} + +template +void modulated_deformable_2d_im2col_cuda(cudaStream_t stream, + const scalar_t *data_im, const scalar_t *data_offset, + const scalar_t *data_mask, + const int batch_size, const int channels, const int height_im, + const int width_im, + const int height_col, const int width_col, const int kernel_h, + const int kernel_w, + const int pad_h, const int pad_w, const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int deformable_group, scalar_t *data_col) { + // num_axes should be smaller than block size + const int channel_per_deformable_group = channels / deformable_group; + const int num_kernels = channels * batch_size * height_col * width_col; + modulated_deformable_2d_im2col_gpu_kernel + <<>>( + num_kernels, data_im, data_offset, data_mask, height_im, width_im, kernel_h, kernel_w, + pad_h, pad_w, stride_h, stride_w, dilation_h, dilation_w, channel_per_deformable_group, + batch_size, channels, deformable_group, height_col, width_col, data_col); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in modulated_deformable_im2col_cuda: %s\n", cudaGetErrorString(err)); + } + +} + +template +void modulated_deformable_2d_col2im_cuda(cudaStream_t stream, + const scalar_t *data_col, const scalar_t *data_offset, + const scalar_t *data_mask, + const int batch_size, const int channels, const int height_im, + const int width_im, + const int height_col, const int width_col, const int kernel_h, + const int kernel_w, + const int pad_h, const int pad_w, const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int deformable_group, scalar_t *grad_im){ + + const int channel_per_deformable_group = channels / deformable_group; + const int num_kernels = channels * kernel_h * kernel_w * batch_size * height_col * width_col; + modulated_deformable_2d_col2im_gpu_kernel + <<>>( + num_kernels, data_col, data_offset, data_mask, channels, height_im, width_im, + kernel_h, kernel_w, pad_h, pad_h, stride_h, stride_w, + dilation_h, dilation_w, channel_per_deformable_group, + batch_size, deformable_group, height_col, width_col, grad_im); + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in modulated_deformable_col2im_cuda: %s\n", cudaGetErrorString(err)); + } + +} + +template +void modulated_deformable_2d_col2im_coord_cuda(cudaStream_t stream, + const scalar_t *data_col, const scalar_t *data_im, + const scalar_t *data_offset, const scalar_t *data_mask, + const int batch_size, const int channels, const int height_im, + const int width_im, + const int height_col, const int width_col, const int kernel_h, + const int kernel_w, + const int pad_h, const int pad_w, const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int deformable_group, + scalar_t *grad_offset, scalar_t *grad_mask) { + const int num_kernels = batch_size * height_col * width_col * 2 * kernel_h * kernel_w * deformable_group; + const int channel_per_deformable_group = channels * kernel_h * kernel_w / deformable_group; + modulated_deformable_2d_col2im_coord_gpu_kernel + <<>>( + num_kernels, data_col, data_im, data_offset, data_mask, channels, height_im, width_im, + kernel_h, kernel_w, pad_h, pad_w, stride_h, stride_w, + dilation_h, dilation_w, channel_per_deformable_group, + batch_size, 2 * kernel_h * kernel_w * deformable_group, deformable_group, height_col, width_col, + grad_offset, grad_mask); + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in modulated_deformable_col2im_coord_cuda: %s\n", cudaGetErrorString(err)); + } +} \ No newline at end of file diff --git a/classification/mm_modules/DCN/src/cuda/modulated_deform_conv2d_cuda.cu b/classification/mm_modules/DCN/src/cuda/modulated_deform_conv2d_cuda.cu new file mode 100644 index 0000000..18ec02b --- /dev/null +++ b/classification/mm_modules/DCN/src/cuda/modulated_deform_conv2d_cuda.cu @@ -0,0 +1,280 @@ +#include +#include "cuda/modulated_deform_2d_im2col_cuda.cuh" + +#include +#include +#include +#include + +// #include +// #include +// #include + +// extern THCState *state; + +// author: Charles Shang +// https://github.com/torch/cunn/blob/master/lib/THCUNN/generic/SpatialConvolutionMM.cu + + +at::Tensor +modulated_deform_conv2d_cuda_forward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &mask, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + // THCAssertSameGPU(THCudaTensor_checkGPU(state, 5, input, weight, bias, offset, mask)); + + AT_ASSERTM(input.is_contiguous(), "input tensor has to be contiguous"); + AT_ASSERTM(weight.is_contiguous(), "weight tensor has to be contiguous"); + + AT_ASSERTM(input.type().is_cuda(), "input must be a CUDA tensor"); + AT_ASSERTM(weight.type().is_cuda(), "weight must be a CUDA tensor"); + AT_ASSERTM(bias.type().is_cuda(), "bias must be a CUDA tensor"); + AT_ASSERTM(offset.type().is_cuda(), "offset must be a CUDA tensor"); + AT_ASSERTM(mask.type().is_cuda(), "mask must be a CUDA tensor"); + + const int batch = input.size(0); + const int channels = input.size(1); + const int height = input.size(2); + const int width = input.size(3); + + const int channels_out = weight.size(0); + const int channels_kernel = weight.size(1); + const int kernel_h_ = weight.size(2); + const int kernel_w_ = weight.size(3); + + const int im2col_step_ = std::min(batch, im2col_step); + + AT_ASSERTM(batch % im2col_step_ == 0, "batch(%d) must divide im2col_step(%d)", batch, im2col_step_); + + AT_ASSERTM((channels % group == 0) && (channels_out % group == 0), + "channels(%d) and channels_out(%d) must divide group(%d)", channels, channels_out, group); + + // printf("Kernels: %d %d %d %d\n", kernel_h_, kernel_w_, kernel_w, kernel_h); + // printf("Channels: %d %d\n", channels, channels_kernel); + // printf("Channels: %d %d\n", channels_out, channels_kernel); + + AT_ASSERTM(kernel_h_ == kernel_h && kernel_w_ == kernel_w, + "Input shape and kernel shape wont match: (%d x %d vs %d x %d).", kernel_h_, kernel_w, kernel_h_, kernel_w_); + + AT_ASSERTM(channels == (channels_kernel * group), + "Input shape and kernel channels wont match: (%d vs %d).", channels, channels_kernel * group); + + const int height_out = (height + 2 * pad_h - (dilation_h * (kernel_h - 1) + 1)) / stride_h + 1; + const int width_out = (width + 2 * pad_w - (dilation_w * (kernel_w - 1) + 1)) / stride_w + 1; + + auto output = at::empty({batch * height_out * width_out, channels_out}, input.options()); + + // prepare group weight and bias + auto weight_g = weight.view({group, channels_out/group, channels_kernel, kernel_h, kernel_w}); + auto bias_g = bias.view({group, channels_out/group}); + + // define alias for easy use + const int batch_n = im2col_step_; + const int per_input_size = channels * height * width; + const int per_offset_size = offset.size(1) * offset.size(2) * offset.size(3); + const int per_mask_size = mask.size(1) * mask.size(2) * mask.size(3); + auto output_n = output.view({batch/im2col_step_, batch_n * height_out * width_out, channels_out}); + for (int n = 0; n < batch/im2col_step_; ++n) + { + auto columns = at::empty({channels * kernel_h * kernel_w, batch_n * height_out * width_out}, input.options()); + AT_DISPATCH_FLOATING_TYPES(input.type(), "deform_conv_forward_cuda", ([&] { + modulated_deformable_2d_im2col_cuda(at::cuda::getCurrentCUDAStream(), + input.data() + n * im2col_step_ * per_input_size, + offset.data() + n * im2col_step_ * per_offset_size, + mask.data() + n * im2col_step_ * per_mask_size, + batch_n, channels, height, width, + height_out, width_out, kernel_h, kernel_w, + pad_h, pad_w, stride_h, stride_w, dilation_h, dilation_w, + deformable_group, + columns.data()); + + })); + + auto columns_g = columns.view({group, channels/group * kernel_h * kernel_w, batch_n * height_out * width_out}); + auto output_g = output_n.select(0, n).view({batch_n * height_out * width_out, group, channels_out/group}); + for (int g = 0; g < group; ++g) + { + auto columns_gm = columns_g.select(0, g).t(); + auto weight_gm = weight_g.select(0, g).view({channels_out/group, channels_kernel * kernel_h * kernel_w}).t(); + auto output_m = at::addmm(bias_g.select(0, g), columns_gm, weight_gm); + output_g.select(1, g) = output_m.view({batch_n * height_out * width_out, channels_out/group}); + } + + } + + output = output.view({batch, height_out, width_out, channels_out}).permute({0, 3, 1, 2}).contiguous(); + + return output; +} + + +std::vector modulated_deform_conv2d_cuda_backward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &mask, + const at::Tensor &grad_output, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + + AT_ASSERTM(input.is_contiguous(), "input tensor has to be contiguous"); + AT_ASSERTM(weight.is_contiguous(), "weight tensor has to be contiguous"); + + AT_ASSERTM(input.type().is_cuda(), "input must be a CUDA tensor"); + AT_ASSERTM(weight.type().is_cuda(), "weight must be a CUDA tensor"); + AT_ASSERTM(bias.type().is_cuda(), "bias must be a CUDA tensor"); + AT_ASSERTM(offset.type().is_cuda(), "offset must be a CUDA tensor"); + AT_ASSERTM(mask.type().is_cuda(), "mask must be a CUDA tensor"); + + const int batch = input.size(0); + const int channels = input.size(1); + const int height = input.size(2); + const int width = input.size(3); + + const int channels_out = weight.size(0); + const int channels_kernel = weight.size(1); + const int kernel_h_ = weight.size(2); + const int kernel_w_ = weight.size(3); + + const int batch_ = grad_output.size(0); + const int channels_out_ = grad_output.size(1); + const int height_out_ = grad_output.size(2); + const int width_out_ = grad_output.size(3); + + const int im2col_step_ = std::min(im2col_step, batch); + + AT_ASSERTM(batch % im2col_step_ == 0, "batch(%d) must divide im2col_step(%d)", batch, im2col_step_); + + AT_ASSERTM((channels % group == 0) && (channels_out % group == 0), + "channels(%d) and channels_out(%d) must divide group(%d)", channels, channels_out, group); + + AT_ASSERTM(kernel_h_ == kernel_h && kernel_w_ == kernel_w, + "Input shape and kernel shape wont match: (%d x %d vs %d x %d).", kernel_h_, kernel_w, kernel_h_, kernel_w_); + + AT_ASSERTM(channels == (channels_kernel * group), + "Input shape and kernel channels wont match: (%d vs %d).", channels, channels_kernel * group); + + const int height_out = (height + 2 * pad_h - (dilation_h * (kernel_h - 1) + 1)) / stride_h + 1; + const int width_out = (width + 2 * pad_w - (dilation_w * (kernel_w - 1) + 1)) / stride_w + 1; + + AT_ASSERTM(batch == batch_, + "Input shape and grad_out batch wont match: (%d vs %d).", batch, batch_); + + AT_ASSERTM(channels_out == channels_out_, + "Input shape and grad_out channels_out wont match: (%d vs %d).", channels_out, channels_out_); + + AT_ASSERTM(height_out == height_out_ && width_out == width_out_, + "Input shape and grad_out shape wont match: (%d x %d vs %d x %d).", height_out, height_out_, width_out, width_out_); + + auto ones = at::ones({batch * height_out * width_out}, input.options()); + auto columns = at::empty({channels * kernel_h * kernel_w, batch * 1 * height_out * width_out}, input.options()); + + auto grad_input = at::zeros_like(input); + auto grad_weight = at::zeros_like(weight); + auto grad_bias = at::zeros_like(bias); + auto grad_offset = at::zeros_like(offset); + auto grad_mask = at::zeros_like(mask); + + // prepare group weight and bias + auto weight_g = weight.view({group, channels_out/group, channels_kernel, kernel_h, kernel_w}); + auto grad_weight_g = grad_weight.view({group, channels_out/group, channels_kernel, kernel_h, kernel_w}); + auto grad_bias_g = grad_bias.view({group, channels_out/group}); + + const int batch_n = im2col_step_; + const int per_input_size = channels * height * width; + const int per_offset_size = offset.size(1) * offset.size(2) * offset.size(3); + const int per_mask_size = mask.size(1) * mask.size(2) * mask.size(3); + auto grad_output_n = grad_output.view({batch/im2col_step_, batch_n, channels_out, height_out, width_out}); + for (int n = 0; n < batch/im2col_step_; ++n) + { + auto grad_output_g = grad_output_n.select(0, n).view({batch_n, group, channels_out/group, height_out, width_out}); + auto ones = at::ones({batch_n * height_out * width_out}, input.options()); + auto columns = at::empty({channels * kernel_h * kernel_w, batch_n * 1 * height_out * width_out}, input.options()); + auto columns_g = columns.view({group, channels/group * kernel_h * kernel_w, batch_n * height_out * width_out}); + for (int g = 0; g < group; ++g) + { + auto grad_output_gm = grad_output_g.select(1, g).permute({1, 0, 2, 3}).contiguous().view({channels_out/group, batch_n * height_out * width_out}); + auto weight_gm = weight_g.select(0, g).view({channels_out/group, channels_kernel * kernel_h * kernel_w}).t(); + columns_g.select(0, g) = at::mm(weight_gm, grad_output_gm); + } + + AT_DISPATCH_FLOATING_TYPES(input.type(), "deform_conv_backward_cuda", ([&] { + modulated_deformable_2d_col2im_coord_cuda(at::cuda::getCurrentCUDAStream(), + columns.data(), + input.data() + n * im2col_step_ * per_input_size, + offset.data() + n * im2col_step_ * per_offset_size, + mask.data() + n * im2col_step_ * per_mask_size, + batch_n, channels, height, width, + height_out, width_out, kernel_h, kernel_w, + pad_h, pad_w, stride_h, stride_w, + dilation_h, dilation_w, deformable_group, + grad_offset.data() + n * im2col_step_ * per_offset_size, + grad_mask.data() + n * im2col_step_ * per_mask_size); + // gradient w.r.t. input data + modulated_deformable_2d_col2im_cuda(at::cuda::getCurrentCUDAStream(), + columns.data(), + offset.data() + n * im2col_step_ * per_offset_size, + mask.data() + n * im2col_step_ * per_mask_size, + batch_n, channels, height, width, + height_out, width_out, kernel_h, kernel_w, + pad_h, pad_w, stride_h, stride_w, + dilation_h, dilation_w, deformable_group, + grad_input.data() + n * im2col_step_ * per_input_size); + + // gradient w.r.t. weight, dWeight should accumulate across the batch and group + modulated_deformable_2d_im2col_cuda(at::cuda::getCurrentCUDAStream(), + input.data() + n * im2col_step_ * per_input_size, + offset.data() + n * im2col_step_ * per_offset_size, + mask.data() + n * im2col_step_ * per_mask_size, + batch_n, channels, height, width, + height_out, width_out, kernel_h, kernel_w, + pad_h, pad_w, stride_h, stride_w, + dilation_h, dilation_w, deformable_group, + columns.data()); + + })); + + // auto grad_output_m = grad_output.permute({1, 0, 2, 3}).contiguous().view({channels_out, batch * height_out * width_out}); + // grad_weight = at::mm(grad_output_m, columns.t()).view_as(weight); + // grad_bias = at::mv(grad_output_m, ones); + // auto grad_output_g = grad_output.view({batch, group, channels_out/group, height_out, width_out}); + // auto columns_g = columns.view({group, channels/group * kernel_h * kernel_w, batch * height_out * width_out}); + for (int g = 0; g < group; ++g) + { + auto grad_output_gm = grad_output_g.select(1, g).permute({1, 0, 2, 3}).contiguous().view({channels_out/group, batch_n * height_out * width_out}); + auto columns_gm = columns_g.select(0, g).t(); + auto grad_weight_gm = grad_weight_g.select(0, g).view({channels_out/group, channels_kernel * kernel_h * kernel_w}); + auto grad_bias_gm = grad_bias_g.select(0, g); + grad_weight_g.select(0, g) = at::addmm(grad_weight_gm, grad_output_gm, columns_gm).view_as(grad_weight_g.select(0, g)); + grad_bias_g.select(0, g) = at::addmv(grad_bias_gm, grad_output_gm, ones); + } + + } + + return { + grad_input, grad_offset, grad_mask, grad_weight, grad_bias + }; +} diff --git a/classification/mm_modules/DCN/src/cuda/modulated_deform_conv2d_cuda.h b/classification/mm_modules/DCN/src/cuda/modulated_deform_conv2d_cuda.h new file mode 100644 index 0000000..4ee2dce --- /dev/null +++ b/classification/mm_modules/DCN/src/cuda/modulated_deform_conv2d_cuda.h @@ -0,0 +1,40 @@ +#pragma once +#include + +at::Tensor +modulated_deform_conv2d_cuda_forward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &mask, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step); + +std::vector +modulated_deform_conv2d_cuda_backward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &mask, + const at::Tensor &grad_output, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step); + diff --git a/classification/mm_modules/DCN/src/deform_conv2d.h b/classification/mm_modules/DCN/src/deform_conv2d.h new file mode 100644 index 0000000..bf0af29 --- /dev/null +++ b/classification/mm_modules/DCN/src/deform_conv2d.h @@ -0,0 +1,84 @@ +#pragma once + +#include "cpu/deform_conv2d_cpu.h" + +#ifdef WITH_CUDA +#include "cuda/deform_conv2d_cuda.h" +#endif + + +at::Tensor +deform_conv2d_forward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + if (input.type().is_cuda()) + { +#ifdef WITH_CUDA + return deform_conv2d_cuda_forward(input, weight, bias, offset, + kernel_h, kernel_w, + stride_h, stride_w, + pad_h, pad_w, + dilation_h, dilation_w, + group, + deformable_group, + im2col_step); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} + +std::vector +deform_conv2d_backward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &grad_output, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + if (input.type().is_cuda()) + { +#ifdef WITH_CUDA + return deform_conv2d_cuda_backward(input, + weight, + bias, + offset, + grad_output, + kernel_h, kernel_w, + stride_h, stride_w, + pad_h, pad_w, + dilation_h, dilation_w, + group, + deformable_group, + im2col_step); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} + diff --git a/classification/mm_modules/DCN/src/modulated_deform_conv2d.h b/classification/mm_modules/DCN/src/modulated_deform_conv2d.h new file mode 100644 index 0000000..9c8043e --- /dev/null +++ b/classification/mm_modules/DCN/src/modulated_deform_conv2d.h @@ -0,0 +1,87 @@ +#pragma once + +#include "cpu/modulated_deform_conv2d_cpu.h" + +#ifdef WITH_CUDA +#include "cuda/modulated_deform_conv2d_cuda.h" +#endif + + +at::Tensor +modulated_deform_conv2d_forward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &mask, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + if (input.type().is_cuda()) + { +#ifdef WITH_CUDA + return modulated_deform_conv2d_cuda_forward(input, weight, bias, offset, mask, + kernel_h, kernel_w, + stride_h, stride_w, + pad_h, pad_w, + dilation_h, dilation_w, + group, + deformable_group, + im2col_step); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} + +std::vector +modulated_deform_conv2d_backward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &mask, + const at::Tensor &grad_output, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + if (input.type().is_cuda()) + { +#ifdef WITH_CUDA + return modulated_deform_conv2d_cuda_backward(input, + weight, + bias, + offset, + mask, + grad_output, + kernel_h, kernel_w, + stride_h, stride_w, + pad_h, pad_w, + dilation_h, dilation_w, + group, + deformable_group, + im2col_step); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} + diff --git a/classification/mm_modules/DCN/src/vision.cpp b/classification/mm_modules/DCN/src/vision.cpp new file mode 100644 index 0000000..5043fea --- /dev/null +++ b/classification/mm_modules/DCN/src/vision.cpp @@ -0,0 +1,10 @@ + +#include "deform_conv2d.h" +#include "modulated_deform_conv2d.h" + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("deform_conv2d_forward", &deform_conv2d_forward, "deform_conv2d_forward"); + m.def("deform_conv2d_backward", &deform_conv2d_backward, "deform_conv2d_backward"); + m.def("modulated_deform_conv2d_forward", &modulated_deform_conv2d_forward, "modulated_deform_conv2d_forward"); + m.def("modulated_deform_conv2d_backward", &modulated_deform_conv2d_backward, "modulated_deform_conv2d_backward"); +} diff --git a/classification/mm_modules/__init__.py b/classification/mm_modules/__init__.py new file mode 100644 index 0000000..faaaf79 --- /dev/null +++ b/classification/mm_modules/__init__.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + + diff --git a/classification/mm_modules/cocoapi_evaluator.py b/classification/mm_modules/cocoapi_evaluator.py new file mode 100644 index 0000000..0d47637 --- /dev/null +++ b/classification/mm_modules/cocoapi_evaluator.py @@ -0,0 +1,209 @@ +import json +import tempfile +import sys +from tqdm import tqdm + +from pycocotools.cocoeval import COCOeval +from torch.autograd import Variable + +from dataset.cocodataset import * +from dataset.data_augment import ValTransform +from utils.utils import * +from utils import distributed_util +from utils.vis_utils import make_vis, make_pred_vis +import time +import apex + +DEBUG =False + +def _accumulate_predictions_from_multiple_gpus(predictions_per_gpu): + all_predictions = distributed_util.scatter_gather(predictions_per_gpu) + if not distributed_util.is_main_process(): + return + # merge the list of dicts + predictions = [] + for p in all_predictions: + for a in p: + predictions.append(a) + + return predictions + +class COCOAPIEvaluator(): + """ + COCO AP Evaluation class. + All the data in the val2017 dataset are processed \ + and evaluated by COCO API. + """ + def __init__(self, data_dir, img_size, confthre, nmsthre, testset=False, voc=False, vis=False): + """ + Args: + data_dir (str): dataset root directory + img_size (int): image size after preprocess. images are resized \ + to squares whose shape is (img_size, img_size). + confthre (float): + confidence threshold ranging from 0 to 1, \ + which is defined in the config file. + nmsthre (float): + IoU threshold of non-max supression ranging from 0 to 1. + """ + json_f = 'instances_val2017.json' + name='val2017' + if testset: + json_f = 'image_info_test-dev2017.json' + name='test2017' + if voc: + json_f = 'pascal_test2007.json' + + self.testset= testset + self.dataset = COCODataset(data_dir=data_dir, + img_size=img_size, + json_file=json_f, + preproc = ValTransform(rgb_means=(0.485, 0.456, 0.406),std=(0.229, 0.224, 0.225)), + name=name, + voc = voc) + + self.num_images = len(self.dataset) + self.dataloader = torch.utils.data.DataLoader( + self.dataset, batch_size=1, shuffle=False, num_workers=0) + self.img_size = img_size + self.confthre = confthre + self.nmsthre = nmsthre + self.voc = voc + self.vis = vis + + def evaluate(self, model, half=False, distributed=False): + """ + COCO average precision (AP) Evaluation. Iterate inference on the test dataset + and the results are evaluated by COCO API. + Args: + model : model object + Returns: + ap50_95 (float) : calculated COCO AP for IoU=50:95 + ap50 (float) : calculated COCO AP for IoU=50 + """ + if isinstance(model, apex.parallel.DistributedDataParallel): + model = model.module + distributed=True + + model=model.eval() + cuda = torch.cuda.is_available() + if half: + Tensor = torch.cuda.HalfTensor if cuda else torch.HalfTensor + else: + Tensor = torch.cuda.FloatTensor if cuda else torch.FloatTensor + ids = [] + data_dict = [] + img_num = 0 + + indices = list(range(self.num_images)) + if distributed: + dis_indices = indices[distributed_util.get_rank()::distributed_util.get_world_size()] + else: + dis_indices = indices + progress_bar = tqdm if distributed_util.is_main_process() else iter + num_classes = 80 if not self.voc else 20 + + inference_time=0 + nms_time=0 + n_samples=len(dis_indices)-10 + + for k, i in enumerate(progress_bar(dis_indices)): + img, _, info_img, id_ = self.dataset[i] # load a batch + info_img = [float(info) for info in info_img] + id_ = int(id_) + ids.append(id_) + with torch.no_grad(): + img = Variable(img.type(Tensor).unsqueeze(0)) + if k > 9: + start=time.time() + + if self.vis: + outputs,fuse_weights,fused_f = model(img) + else: + outputs = model(img) + + if k > 9: + infer_end=time.time() + inference_time += (infer_end-start) + + outputs = postprocess( + outputs, num_classes, self.confthre, self.nmsthre) + + if k > 9: + nms_end=time.time() + nms_time +=(nms_end-infer_end) + + if outputs[0] is None: + continue + outputs = outputs[0].cpu().data + + bboxes = outputs[:, 0:4] + bboxes[:, 0::2] *= info_img[0] / self.img_size[0] + bboxes[:, 1::2] *= info_img[1] / self.img_size[1] + bboxes[:, 2] = bboxes[:,2] - bboxes[:,0] + bboxes[:, 3] = bboxes[:,3] - bboxes[:,1] + cls = outputs[:, 6] + scores = outputs[:, 4]* outputs[:,5] + for ind in range(bboxes.shape[0]): + label = self.dataset.class_ids[int(cls[ind])] + A = {"image_id": id_, "category_id": label, "bbox": bboxes[ind].numpy().tolist(), + "score": scores[ind].numpy().item(), "segmentation": []} # COCO json format + data_dict.append(A) + + if self.vis: + o_img,_,_,_ = self.dataset.pull_item(i) + make_vis('COCO', i, o_img, fuse_weights, fused_f) + class_names = self.dataset._classes + make_pred_vis('COCO', i, o_img, class_names, bboxes, cls, scores) + + if DEBUG and distributed_util.is_main_process(): + o_img,_ = self.dataset.pull_item(i) + class_names = self.dataset._classes + make_pred_vis('COCO', i, o_img, class_names, bboxes, cls, scores) + + if distributed: + distributed_util.synchronize() + data_dict = _accumulate_predictions_from_multiple_gpus(data_dict) + inference_time = torch.FloatTensor(1).type(Tensor).fill_(inference_time) + nms_time = torch.FloatTensor(1).type(Tensor).fill_(nms_time) + n_samples = torch.LongTensor(1).type(Tensor).fill_(n_samples) + distributed_util.synchronize() + torch.distributed.reduce(inference_time, dst=0) + torch.distributed.reduce(nms_time, dst=0) + torch.distributed.reduce(n_samples, dst=0) + inference_time = inference_time.item() + nms_time = nms_time.item() + n_samples = n_samples.item() + + if not distributed_util.is_main_process(): + return 0, 0 + + + print('Main process Evaluating...') + + annType = ['segm', 'bbox', 'keypoints'] + a_infer_time = 1000*inference_time / (n_samples) + a_nms_time= 1000*nms_time / (n_samples) + + print('Average forward time: %.2f ms, Average NMS time: %.2f ms, Average inference time: %.2f ms' %(a_infer_time, \ + a_nms_time, (a_infer_time+a_nms_time))) + + # Evaluate the Dt (detection) json comparing with the ground truth + if len(data_dict) > 0: + cocoGt = self.dataset.coco + # workaround: temporarily write data to json file because pycocotools can't process dict in py36. + if self.testset: + json.dump(data_dict, open('yolov3_2017.json', 'w')) + cocoDt = cocoGt.loadRes('yolov3_2017.json') + else: + _, tmp = tempfile.mkstemp() + json.dump(data_dict, open(tmp, 'w')) + cocoDt = cocoGt.loadRes(tmp) + cocoEval = COCOeval(self.dataset.coco, cocoDt, annType[1]) + cocoEval.evaluate() + cocoEval.accumulate() + cocoEval.summarize() + return cocoEval.stats[0], cocoEval.stats[1] + else: + return 0, 0 + diff --git a/classification/mm_modules/distributed_util.py b/classification/mm_modules/distributed_util.py new file mode 100644 index 0000000..dcd2479 --- /dev/null +++ b/classification/mm_modules/distributed_util.py @@ -0,0 +1,162 @@ +import os +import pickle +import tempfile +import time + +import torch + + +def get_world_size(): + if not torch.distributed.is_initialized(): + return 1 + return torch.distributed.get_world_size() + + +def get_rank(): + if not torch.distributed.is_initialized(): + return 0 + return torch.distributed.get_rank() + + +def is_main_process(): + if not torch.distributed.is_initialized(): + return True + return torch.distributed.get_rank() == 0 + + +def synchronize(): + """ + Helper function to synchronize between multiple processes when + using distributed training + """ + if not torch.distributed.is_initialized(): + return + world_size = torch.distributed.get_world_size() + rank = torch.distributed.get_rank() + if world_size == 1: + return + + def _send_and_wait(r): + if rank == r: + tensor = torch.tensor(0, device="cuda") + else: + tensor = torch.tensor(1, device="cuda") + torch.distributed.broadcast(tensor, r) + while tensor.item() == 1: + time.sleep(1) + + _send_and_wait(0) + # now sync on the main process + _send_and_wait(1) + + +def _encode(encoded_data, data): + # gets a byte representation for the data + encoded_bytes = pickle.dumps(data) + # convert this byte string into a byte tensor + storage = torch.ByteStorage.from_buffer(encoded_bytes) + tensor = torch.ByteTensor(storage).to("cuda") + # encoding: first byte is the size and then rest is the data + s = tensor.numel() + assert s <= 255, "Can't encode data greater than 255 bytes" + # put the encoded data in encoded_data + encoded_data[0] = s + encoded_data[1: (s + 1)] = tensor + + +def _decode(encoded_data): + size = encoded_data[0] + encoded_tensor = encoded_data[1: (size + 1)].to("cpu") + return pickle.loads(bytearray(encoded_tensor.tolist())) + + +# TODO try to use tensor in shared-memory instead of serializing to disk +# this involves getting the all_gather to work +def scatter_gather(data): + """ + This function gathers data from multiple processes, and returns them + in a list, as they were obtained from each process. + This function is useful for retrieving data from multiple processes, + when launching the code with torch.distributed.launch + Note: this function is slow and should not be used in tight loops, i.e., + do not use it in the training loop. + Arguments: + data: the object to be gathered from multiple processes. + It must be serializable + Returns: + result (list): a list with as many elements as there are processes, + where each element i in the list corresponds to the data that was + gathered from the process of rank i. + """ + # strategy: the main process creates a temporary directory, and communicates + # the location of the temporary directory to all other processes. + # each process will then serialize the data to the folder defined by + # the main process, and then the main process reads all of the serialized + # files and returns them in a list + if not torch.distributed.is_initialized(): + return [data] + synchronize() + # get rank of the current process + rank = torch.distributed.get_rank() + + # the data to communicate should be small + data_to_communicate = torch.empty(256, dtype=torch.uint8, device="cuda") + if rank == 0: + # manually creates a temporary directory, that needs to be cleaned + # afterwards + tmp_dir = tempfile.mkdtemp() + _encode(data_to_communicate, tmp_dir) + + synchronize() + # the main process (rank=0) communicates the data to all processes + torch.distributed.broadcast(data_to_communicate, 0) + + # get the data that was communicated + tmp_dir = _decode(data_to_communicate) + + # each process serializes to a different file + file_template = "file{}.pth" + tmp_file = os.path.join(tmp_dir, file_template.format(rank)) + torch.save(data, tmp_file) + + # synchronize before loading the data + synchronize() + + # only the master process returns the data + if rank == 0: + data_list = [] + world_size = torch.distributed.get_world_size() + for r in range(world_size): + file_path = os.path.join(tmp_dir, file_template.format(r)) + d = torch.load(file_path) + data_list.append(d) + # cleanup + os.remove(file_path) + # cleanup + os.rmdir(tmp_dir) + return data_list + + +def reduce_loss_dict(loss_dict): + """ + Reduce the loss dictionary from all processes so that process with rank + 0 has the averaged results. Returns a dict with the same fields as + loss_dict, after reduction. + """ + world_size = get_world_size() + if world_size < 2: + return loss_dict + with torch.no_grad(): + loss_names = [] + all_losses = [] + for k in sorted(loss_dict.keys()): + loss_names.append(k) + all_losses.append(loss_dict[k]) + all_losses = torch.stack(all_losses, dim=0) + torch.distributed.reduce(all_losses, dst=0) + if torch.distributed.get_rank() == 0: + # only main process gets accumulated, so only divide by + # world_size in this case + all_losses /= world_size + reduced_losses = {k: v for k, v in zip(loss_names, all_losses)} + return reduced_losses diff --git a/classification/mm_modules/fp16_utils/README.md b/classification/mm_modules/fp16_utils/README.md new file mode 100644 index 0000000..941de17 --- /dev/null +++ b/classification/mm_modules/fp16_utils/README.md @@ -0,0 +1,16 @@ +fp16_optimizer.py contains `FP16_Optimizer`, a Python class designed to wrap an existing Pytorch optimizer and automatically enable master parameters and loss scaling in a manner transparent to the user. To use `FP16_Optimizer`, only two lines of one's Python model need to change. + +#### [FP16_Optimizer API documentation](https://nvidia.github.io/apex/fp16_utils.html#automatic-management-of-master-params-loss-scaling) + +#### [Simple examples with FP16_Optimizer](https://github.com/NVIDIA/apex/tree/master/examples/FP16_Optimizer_simple) + +#### [Imagenet with FP16_Optimizer](https://github.com/NVIDIA/apex/tree/master/examples/imagenet) + +#### [word_language_model with FP16_Optimizer](https://github.com/NVIDIA/apex/tree/master/examples/word_language_model) + + +fp16_util.py contains a number of utilities to manually manage master parameters and loss scaling, if the user chooses. + +#### [Manual management documentation](https://nvidia.github.io/apex/fp16_utils.html#manual-master-parameter-management) + +The [Imagenet with FP16_Optimizer](https://github.com/NVIDIA/apex/tree/master/examples/imagenet) and [word_language_model with FP16_Optimizer](https://github.com/NVIDIA/apex/tree/master/examples/word_language_model) directories also contain `main.py` files that demonstrate manual management of master parameters and static loss scaling. These examples illustrate what sort of operations `FP16_Optimizer` is performing automatically. diff --git a/classification/mm_modules/fp16_utils/__init__.py b/classification/mm_modules/fp16_utils/__init__.py new file mode 100644 index 0000000..c7bb1f5 --- /dev/null +++ b/classification/mm_modules/fp16_utils/__init__.py @@ -0,0 +1,16 @@ +from .fp16util import ( + BN_convert_float, + network_to_half, + prep_param_lists, + model_grads_to_master_grads, + master_params_to_model_params, + tofp16, + to_python_float, + clip_grad_norm, + convert_module, + convert_network, + FP16Model, +) + +from .fp16_optimizer import FP16_Optimizer +from .loss_scaler import LossScaler, DynamicLossScaler diff --git a/classification/mm_modules/fp16_utils/fp16_optimizer.py b/classification/mm_modules/fp16_utils/fp16_optimizer.py new file mode 100644 index 0000000..fe999e0 --- /dev/null +++ b/classification/mm_modules/fp16_utils/fp16_optimizer.py @@ -0,0 +1,561 @@ +import torch +from torch import nn +from torch.autograd import Variable +from torch.nn.parameter import Parameter +from torch._utils import _flatten_dense_tensors, _unflatten_dense_tensors + +from .loss_scaler import DynamicLossScaler, LossScaler +from .fp16util import model_grads_to_master_grads, master_params_to_model_params, clip_grad_norm + +# TODO: Update overflow check + downscale to use Carl's fused kernel. +class FP16_Optimizer(object): + """ + :class:`FP16_Optimizer` is designed to wrap an existing PyTorch optimizer, + and manage static or dynamic loss scaling and master weights in a manner transparent to the user. + For standard use, only two lines must be changed: creating the :class:`FP16_Optimizer` instance, + and changing the call to ``backward``. + + Example:: + + model = torch.nn.Linear(D_in, D_out).cuda().half() + optimizer = torch.optim.SGD(model.parameters(), lr=1e-3) + # Name the FP16_Optimizer instance to replace the existing optimizer + # (recommended but not required): + optimizer = FP16_Optimizer(optimizer, static_loss_scale = 128.0) + ... + # loss.backward() becomes: + optimizer.backward(loss) + ... + + Example with dynamic loss scaling:: + + ... + optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True) + # optional arg to control dynamic loss scaling behavior + # dynamic_loss_args={'scale_window' : 500}) + # Usually, dynamic_loss_args is not necessary. + + Args: + init_optimizer (torch.optim.optimizer): Existing optimizer created with the parameters to optimize. Internally, :class:`FP16_Optimizer` replaces the passed optimizer's fp16 parameters, if any, with fp32 master parameters copied from the original ones. :class:`FP16_Optimizer` also stores references to the original fp16 parameters, and updates these fp16 parameters from the master fp32 copy at the end of each :attr:`step`. + static_loss_scale (float, optional, default=1.0): Loss scale used internally to scale gradients computed by the model. Any fp16 gradients will be copied to fp32, then downscaled before being applied to the fp32 master params, so ``static_loss_scale`` should not affect learning rate. + dynamic_loss_scale (bool, optional, default=False): Use dynamic loss scaling. If True, this will override any ``static_loss_scale`` option. + dynamic_loss_args (dict, optional, default=None): Dict of kwargs that will be forwarded to the internal :class:`DynamicLossScaler` instance's constructor. Keys of this dict must match kwargs accepted by :class:`DynamicLossScaler`'s constructor. If ``dynamic_loss_args`` is unspecified, :class:`DynamicLossScaler`'s defaults will be used. + verbose (bool, optional, default=True): By default, FP16_Optimizer's constructor prints out the parameters and parameter groups it is ingesting, as a sanity check. If this becomes annoying (e.g. for large models), it can be disabled by passing ``verbose=False``. ``verbose=False`` will not disable printing when the loss scale is readjusted during dynamic loss scaling. + + ``init_optimizer`` is expected to have been constructed in the ordinary way. + It is recommended (although not required) that the newly constructed :class:`FP16_Optimizer` instance be + named to replace ``init_optimizer``, for two reasons: + First, it means that references to the same name + later in the file will not have to change. + Second, :class:`FP16_Optimizer` reserves the right (as an implementation detail) to + modify ``init_optimizer``. If you do choose a unique name for the new + :class:`FP16_Optimizer` instance, you should only work with this new instance, + because the preexisting optimizer might no longer behave as expected. + + ``init_optimizer`` may be any Pytorch optimizer. + It may contain a mixture of fp16 and fp32 parameters organized into any number of + ``param_groups`` with different hyperparameters. The :class:`FP16_Optimizer` constructor will + ingest these ``param_groups`` and remember them. + + Calls to :: + + loss.backward() + + must be replaced with :: + + optimizer.backward(loss) + + because :class:`FP16_Optimizer` requires ownership of the backward pass to implement + loss scaling and copies to master gradients. + + .. note:: + Loss scaling, either static or dynamic, is orthogonal to learning rate, because gradients + are downscaled before being applied. This means that adjusting the loss scale, or using + dynamic loss scaling, should not require retuning the learning rate or any other + hyperparameters. + + + **Advanced options** + + **Closures**: :class:`FP16_Optimizer` can wrap a Pytorch optimizer that receives a closure. + See docstring for :attr:`step`. + + **Gradient clipping**: Use :attr:`clip_master_grads`. + + **Multiple losses**: If your model accumulates gradients from multiple losses, + this can be made more efficient by supplying ``update_master_grads=False`` + to :attr:`backward`. See docstring for :attr:`backward`. + + **Manually adjusting loss scale**: The current loss scale can be retrieved or set via :: + + print(optimizer.loss_scale) + optimizer.loss_scale = new_loss_scale + + For static loss scaling, manually adjusting the loss scale over time is a reasonable + thing to do. During later epochs, gradients may become smaller, and a + higher loss scale may be required, analogous to scheduling the learning rate. Dynamic loss + scaling is more subtle (see :class:`DynamicLossScaler`) and in this case, manually adjusting + the loss scale is not recommended. + + **Multi_GPU training**: If the wrapped ``init_optimizer`` was created from a model wrapped in + Pytorch DistributedDataParallel or Apex DistributedDataParallel, :class:`FP16_Optimizer` + should still work as intended. + """ + + def __init__(self, + init_optimizer, + static_loss_scale=1.0, + dynamic_loss_scale=False, + dynamic_loss_args=None, + verbose=True): + if not torch.cuda.is_available: + raise SystemError("Cannot use fp16 without CUDA.") + + self.verbose = verbose + + self.optimizer = init_optimizer + # init_state_dict sets up an alternative way to cast per-param state tensors. + # Stashing here in case https://github.com/pytorch/pytorch/issues/7733 makes it necessary. + # init_state_dict = init_optimizer.state_dict() + + self.fp16_groups = [] + self.fp32_from_fp16_groups = [] + self.fp32_from_fp32_groups = [] + for i, param_group in enumerate(self.optimizer.param_groups): + self.maybe_print("FP16_Optimizer processing param group {}:".format(i)) + fp16_params_this_group = [] + fp32_params_this_group = [] + fp32_from_fp16_params_this_group = [] + for i, param in enumerate(param_group['params']): + if param.requires_grad: + if param.type() == 'torch.cuda.HalfTensor': + self.maybe_print("FP16_Optimizer received torch.cuda.HalfTensor with {}" + .format(param.size())) + fp16_params_this_group.append(param) + master_param = param.detach().clone().float() + master_param.requires_grad = True + param_group['params'][i] = master_param + fp32_from_fp16_params_this_group.append(master_param) + # Reset existing state dict key to the new master param. + # We still need to recast per-param state tensors, if any, to FP32. + if param in self.optimizer.state: + self.optimizer.state[master_param] = self.optimizer.state.pop(param) + elif param.type() == 'torch.cuda.FloatTensor': + self.maybe_print("FP16_Optimizer received torch.cuda.FloatTensor with {}" + .format(param.size())) + fp32_params_this_group.append(param) + param_group['params'][i] = param + else: + raise TypeError("Wrapped parameters must be either " + "torch.cuda.FloatTensor or torch.cuda.HalfTensor. " + "Received {}".format(param.type())) + + self.fp16_groups.append(fp16_params_this_group) + self.fp32_from_fp16_groups.append(fp32_from_fp16_params_this_group) + self.fp32_from_fp32_groups.append(fp32_params_this_group) + + # Leverage state_dict() and load_state_dict() to recast preexisting per-param state tensors + self.optimizer.load_state_dict(self.optimizer.state_dict()) + # alternative way to cast per-param state tensors: + # self.optimizer.load_state_dict(init_state_dict) + + if dynamic_loss_scale: + self.dynamic_loss_scale = True + if dynamic_loss_args is not None: + self.loss_scaler = DynamicLossScaler(**dynamic_loss_args) + else: + self.loss_scaler = DynamicLossScaler() + else: + self.dynamic_loss_scale = False + self.loss_scaler = LossScaler(static_loss_scale) + + self.overflow = False + self.first_closure_call_this_step = True + + self.clip_grad_norm = clip_grad_norm + + def maybe_print(self, msg): + if self.verbose: + print(msg) + + def __getstate__(self): + raise RuntimeError("FP16_Optimizer should be serialized using state_dict().") + + def __setstate__(self, state): + raise RuntimeError("FP16_Optimizer should be deserialized using load_state_dict().") + + def zero_grad(self, set_grads_to_None=False): + """ + Zero fp32 and fp16 parameter grads. + """ + # In principle, only the .grad attributes of the model params need to be zeroed, + # because gradients are copied into the FP32 master params. However, we zero + # all gradients owned by the optimizer, just to be safe: + for group in self.optimizer.param_groups: + for p in group['params']: + if set_grads_to_None: + p.grad = None + else: + if p.grad is not None: + p.grad.detach_() + p.grad.zero_() + + # Zero fp16 gradients owned by the model: + for fp16_group in self.fp16_groups: + for param in fp16_group: + if set_grads_to_None: + param.grad = None + else: + if param.grad is not None: + param.grad.detach_() # as in torch.optim.optimizer.zero_grad() + param.grad.zero_() + + def _check_overflow(self): + params = [] + for group in self.fp16_groups: + for param in group: + params.append(param) + for group in self.fp32_from_fp32_groups: + for param in group: + params.append(param) + self.overflow = self.loss_scaler.has_overflow(params) + + def _update_scale(self, has_overflow=False): + self.loss_scaler.update_scale(has_overflow) + + def _master_params_to_model_params(self): + for fp16_group, fp32_from_fp16_group in zip(self.fp16_groups, self.fp32_from_fp16_groups): + master_params_to_model_params(fp16_group, fp32_from_fp16_group) + + # To consider: Integrate distributed with this wrapper by registering a hook on each variable + # that does the overflow check, gradient copy + downscale, and fp32 allreduce in a different stream. + def _model_grads_to_master_grads(self): + for fp16_group, fp32_from_fp16_group in zip(self.fp16_groups, self.fp32_from_fp16_groups): + model_grads_to_master_grads(fp16_group, fp32_from_fp16_group) + + def _downscale_master(self): + if self.loss_scale != 1.0: + for group in self.optimizer.param_groups: + for param in group['params']: + if param.grad is not None: + param.grad.data.mul_(1./self.loss_scale) + + def clip_master_grads(self, max_norm, norm_type=2): + """ + Clips fp32 master gradients via ``torch.nn.utils.clip_grad_norm``. + + Args: + max_norm (float or int): max norm of the gradients + norm_type (float or int): type of the used p-norm. Can be ``'inf'`` for + infinity norm. + + Returns: + Total norm of the current fp32 gradients (viewed as a single vector). + + .. warning:: + Returns -1 if the most recently computed fp16 gradients overflowed (that is, if ``self.overflow`` is ``True``). + """ + if not self.overflow: + fp32_params = [] + for param_group in self.optimizer.param_groups: + for param in param_group['params']: + fp32_params.append(param) + return self.clip_grad_norm(fp32_params, max_norm, norm_type) + else: + return -1 + + def state_dict(self): + """ + Returns a dict containing the current state of this :class:`FP16_Optimizer` instance. + This dict contains attributes of :class:`FP16_Optimizer`, as well as the state_dict + of the contained Pytorch optimizer. + Example:: + + checkpoint = {} + checkpoint['model'] = model.state_dict() + checkpoint['optimizer'] = optimizer.state_dict() + torch.save(checkpoint, "saved.pth") + """ + state_dict = {} + state_dict['loss_scaler'] = self.loss_scaler + state_dict['dynamic_loss_scale'] = self.dynamic_loss_scale + state_dict['overflow'] = self.overflow + state_dict['first_closure_call_this_step'] = self.first_closure_call_this_step + state_dict['optimizer_state_dict'] = self.optimizer.state_dict() + state_dict['fp32_from_fp16'] = self.fp32_from_fp16_groups + return state_dict + + def load_state_dict(self, state_dict): + """ + Loads a state_dict created by an earlier call to state_dict(). + If ``fp16_optimizer_instance`` was constructed from some ``init_optimizer``, + whose parameters in turn came from ``model``, it is expected that the user + will call ``model.load_state_dict()`` before + ``fp16_optimizer_instance.load_state_dict()`` is called. + + Example:: + + model = torch.nn.Linear(D_in, D_out).cuda().half() + optimizer = torch.optim.SGD(model.parameters(), lr=1e-3) + optimizer = FP16_Optimizer(optimizer, static_loss_scale = 128.0) + ... + checkpoint = torch.load("saved.pth") + model.load_state_dict(checkpoint['model']) + optimizer.load_state_dict(checkpoint['optimizer']) + """ + # I think it should actually be ok to reload the optimizer before the model. + self.loss_scaler = state_dict['loss_scaler'] + self.dynamic_loss_scale = state_dict['dynamic_loss_scale'] + self.overflow = state_dict['overflow'] + self.first_closure_call_this_step = state_dict['first_closure_call_this_step'] + self.optimizer.load_state_dict(state_dict['optimizer_state_dict']) + # At this point, the optimizer's references to the model's fp32 parameters are up to date. + # The optimizer's hyperparameters and internal buffers are also up to date. + # However, the fp32 master copies of the model's fp16 params stored by the optimizer are still + # out of date. There are two options. + # 1: Refresh the master params from the model's fp16 params. + # This requires less storage but incurs precision loss. + # 2: Save and restore the fp32 master copies separately. + # We choose option 2. + # + # Pytorch Optimizer.load_state_dict casts saved buffers (e.g. momentum) to the type and device + # of their associated parameters, because it's possible those buffers might not exist yet in + # the current optimizer instance. In our case, as long as the current FP16_Optimizer has been + # constructed in the same way as the one whose state_dict we are loading, the same master params + # are guaranteed to exist, so we can just copy_() from the saved master params. + for current_group, saved_group in zip(self.fp32_from_fp16_groups, state_dict['fp32_from_fp16']): + for current, saved in zip(current_group, saved_group): + current.data.copy_(saved.data) + + def step(self, closure=None): # could add clip option. + """ + If no closure is supplied, :attr:`step` should be called after + ``fp16_optimizer_obj.backward(loss)``. + :attr:`step` updates the fp32 master copy of parameters using the optimizer supplied to + :class:`FP16_Optimizer`'s constructor, then copies the updated fp32 params into the fp16 params + originally referenced by :class:`FP16_Optimizer`'s constructor, so the user may immediately run + another forward pass using their model. + + If a closure is supplied, :attr:`step` may be called without a prior call to + :attr:`backward(loss)`. + This control flow is identical to `ordinary Pytorch optimizer use`_ with closures. + However, the user should take care that any ``loss.backward()`` call within the closure + has been replaced by ``fp16_optimizer_obj.backward(loss)``. + + Args: + closure (optional): Closure that will be supplied to the underlying optimizer originally passed to :class:`FP16_Optimizer`'s constructor. closure should call :attr:`zero_grad()` on the :class:`FP16_Optimizer` object, compute the loss, call :attr:`backward(loss)`, and return the loss. + + Example with closure:: + + # optimizer is assumed to be an FP16_Optimizer object, previously constructed from an + # existing pytorch optimizer. + for input, target in dataset: + def closure(): + optimizer.zero_grad() + output = model(input) + loss = loss_fn(output, target) + # loss.backward() becomes: + optimizer.backward(loss) + return loss + optimizer.step(closure) + + .. warning:: + Currently, calling :attr:`step` with a closure is not compatible with dynamic loss scaling. + + .. _`ordinary Pytorch optimizer use`: + http://pytorch.org/docs/master/optim.html#optimizer-step-closure + """ + + scale = self.loss_scaler.loss_scale + self._update_scale(self.overflow) + + if self.overflow: + print("OVERFLOW! Skipping step. Attempted loss scale: {}, reducing to {}" + .format(scale, self.loss_scale)) + return + + if closure is not None: + retval = self._step_with_closure(closure) + else: + retval = self.optimizer.step() + + self._master_params_to_model_params() + + return retval + + def _step_with_closure(self, closure): + def wrapped_closure(): + # helpful for debugging + # print("Calling wrapped_closure, first_closure_call_this_step = {}" + # .format(self.first_closure_call_this_step)) + if self.first_closure_call_this_step: + # We expect that the fp16 params are initially fresh on entering self.step(), + # so _master_params_to_model_params() is unnecessary the first time wrapped_closure() + # is called within self.optimizer.step(). + self.first_closure_call_this_step = False + else: + # If self.optimizer.step() internally calls wrapped_closure more than once, + # it may update the fp32 params after each call. However, self.optimizer + # doesn't know about the fp16 params at all. If the fp32 params get updated, + # we can't rely on self.optimizer to refresh the fp16 params. We need + # to handle that manually: + self._master_params_to_model_params() + # Our API expects the user to give us ownership of the backward() call by + # replacing all calls to loss.backward() with optimizer.backward(loss). + # This requirement holds whether or not the call to backward() is made within a closure. + # If the user is properly calling optimizer.backward(loss) within "closure," + # calling closure() here will give the fp32 master params fresh gradients + # for the optimizer to play with, so all wrapped_closure needs to do is call + # closure() and return the loss. + temp_loss = closure() + while(self.overflow): + scale = self.loss_scaler.loss_scale + self._update_scale(self.overflow) + print("OVERFLOW within closure! Skipping step. Attempted loss scale: {}, " + "reducing to {}".format(scale, self.loss_scale)) + temp_loss = closure() + return temp_loss + + retval = self.optimizer.step(wrapped_closure) + + self.first_closure_call_this_step = True + + return retval + + def backward(self, loss, update_master_grads=True, retain_graph=False): + """ + :attr:`backward` performs the following conceptual steps: + + 1. fp32_loss = loss.float() (see first Note below) + 2. scaled_loss = fp32_loss*loss_scale + 3. scaled_loss.backward(), which accumulates scaled gradients into the ``.grad`` attributes of the model's leaves (which may be fp16, fp32, or a mixture, depending how your model was defined). + 4. fp16 grads are then copied to the master params' ``.grad`` attributes (see second Note), which are guaranteed to be fp32. + 5. Finally, master grads are divided by loss_scale. + + In this way, after :attr:`backward`, the master params have fresh gradients, + and :attr:`step` may be called. + + .. note:: + :attr:`backward` internally converts the loss to fp32 before applying the loss scale. + This provides some additional safety against overflow if the user has supplied an + fp16 loss value. + However, for maximum overflow safety, the user should + compute the loss criterion (MSE, cross entropy, etc) in fp32 before supplying it to + :attr:`backward`. + + .. warning:: + The gradients found in a model's leaves after the call to + :attr:`backward` should not be regarded as valid in general, + because it's possible + they have been scaled (and in the case of dynamic loss scaling, + the scale factor may change over time). + If the user wants to inspect gradients after a call to :attr:`backward`, + only the master gradients should be regarded as valid. These can be retrieved via + :attr:`inspect_master_grad_data()`. + + Args: + loss: The loss output by the user's model. loss may be either float or half (but see first Note above). + update_master_grads (bool, optional, default=True): Option to copy fp16 grads to fp32 grads on this call. By setting this to False, the user can delay the copy, which is useful to eliminate redundant fp16->fp32 grad copies if :attr:`backward` is being called on multiple losses in one iteration. If set to False, the user becomes responsible for calling :attr:`update_master_grads` before calling :attr:`step`. + retain_graph (bool, optional, default=False): Forwards the usual ``retain_graph=True`` option to the internal call to ``loss.backward``. If ``retain_graph`` is being used to accumulate gradient values from multiple backward passes before calling ``optimizer.step``, passing ``update_master_grads=False`` is also recommended (see Example below). + + Example:: + + # Ordinary operation: + optimizer.backward(loss) + + # Naive operation with multiple losses (technically valid, but less efficient): + # fp32 grads will be correct after the second call, but + # the first call incurs an unnecessary fp16->fp32 grad copy. + optimizer.backward(loss1) + optimizer.backward(loss2) + + # More efficient way to handle multiple losses: + # The fp16->fp32 grad copy is delayed until fp16 grads from all + # losses have been accumulated. + optimizer.backward(loss1, update_master_grads=False) + optimizer.backward(loss2, update_master_grads=False) + optimizer.update_master_grads() + """ + # To consider: try multiple backward passes using retain_grad=True to find + # a loss scale that works. After you find a loss scale that works, do a final dummy + # backward pass with retain_graph=False to tear down the graph. Doing this would avoid + # discarding the iteration, but probably wouldn't improve overall efficiency. + self.loss_scaler.backward(loss.float(), retain_graph=retain_graph) + if update_master_grads: + self.update_master_grads() + + def update_master_grads(self): + """ + Copy the ``.grad`` attribute from stored references to fp16 parameters to + the ``.grad`` attribute of the fp32 master parameters that are directly + updated by the optimizer. :attr:`update_master_grads` only needs to be called if + ``fp16_optimizer_obj.backward`` was called with ``update_master_grads=False``. + """ + if self.dynamic_loss_scale: + self._check_overflow() + if self.overflow: return + self._model_grads_to_master_grads() + self._downscale_master() + + def inspect_master_grad_data(self): + """ + When running with :class:`FP16_Optimizer`, + ``.grad`` attributes of a model's fp16 leaves should not be + regarded as truthful, because they might be scaled. + After a call to :attr:`fp16_optimizer_obj.backward(loss)`, if no overflow was encountered, + the fp32 master params' ``.grad`` + attributes will contain valid gradients properly divided by the loss scale. However, + because :class:`FP16_Optimizer` flattens some parameters, accessing them may be + nonintuitive. :attr:`inspect_master_grad_data` + allows those gradients to be viewed with shapes corresponding to their associated model leaves. + + Returns: + List of lists (one list for each parameter group). The list for each parameter group + is a list of the ``.grad.data`` attributes of the fp32 master params belonging to that group. + """ + if self.overflow: + print("Warning: calling FP16_Optimizer.inspect_master_grad_data while in an overflow state. " + "Gradients are currently invalid (may be inf, nan, or stale). Returning None.") + return None + else: + # The optimizer owns only references to master params. + master_grads_data = [] + for param_group in self.optimizer.param_groups: + master_grads_this_group = [] + for param in param_group['params']: + if param.grad is not None: + master_grads_this_group.append(param.grad.data) + else: + master_grads_this_group.append(None) + master_grads_data.append(master_grads_this_group) + return master_grads_data + + + # Promote loss scale so it can be retrieved or set via "fp16_optimizer_instance.loss_scale" + def _get_loss_scale(self): + return self.loss_scaler.loss_scale + + def _set_loss_scale(self, value): + self.loss_scaler.cur_scale = value + + loss_scale = property(_get_loss_scale, _set_loss_scale) + + # Promote state so it can be retrieved or set via "fp16_optimizer_instance.state" + def _get_state(self): + return self.optimizer.state + + def _set_state(self, value): + self.optimizer.state = value + + state = property(_get_state, _set_state) + + # Promote param_groups so it can be retrieved or set via "fp16_optimizer_instance.param_groups" + # (for example, to adjust the learning rate) + def _get_param_groups(self): + return self.optimizer.param_groups + + def _set_param_groups(self, value): + self.optimizer.param_groups = value + + param_groups = property(_get_param_groups, _set_param_groups) + diff --git a/classification/mm_modules/fp16_utils/fp16util.py b/classification/mm_modules/fp16_utils/fp16util.py new file mode 100644 index 0000000..66c13e4 --- /dev/null +++ b/classification/mm_modules/fp16_utils/fp16util.py @@ -0,0 +1,185 @@ +import torch +import torch.nn as nn +from torch.autograd import Variable +from torch._utils import _flatten_dense_tensors, _unflatten_dense_tensors + + +class tofp16(nn.Module): + """ + Utility module that implements:: + + def forward(self, input): + return input.half() + """ + + def __init__(self): + super(tofp16, self).__init__() + + def forward(self, input): + return input.half() + + +def BN_convert_float(module): + """ + Utility function for network_to_half(). + + Retained for legacy purposes. + """ + if isinstance(module, torch.nn.modules.batchnorm._BatchNorm) and module.affine is True: + module.float() + for child in module.children(): + BN_convert_float(child) + return module + + +def network_to_half(network): + """ + Convert model to half precision in a batchnorm-safe way. + + Retained for legacy purposes. It is recommended to use FP16Model. + """ + return nn.Sequential(tofp16(), BN_convert_float(network.half())) + + +def convert_module(module, dtype): + """ + Converts a module's immediate parameters and buffers to dtype. + """ + for param in module.parameters(recurse=False): + if param is not None: + if param.data.dtype.is_floating_point: + param.data = param.data.to(dtype=dtype) + if param._grad is not None and param._grad.data.dtype.is_floating_point: + param._grad.data = param._grad.data.to(dtype=dtype) + + for buf in module.buffers(recurse=False): + if buf is not None and buf.data.dtype.is_floating_point: + buf.data = buf.data.to(dtype=dtype) + + +def convert_network(network, dtype): + """ + Converts a network's parameters and buffers to dtype. + """ + for module in network.modules(): + if isinstance(module, torch.nn.modules.batchnorm._BatchNorm) and module.affine is True: + continue + convert_module(module, dtype) + return network + + +class FP16Model(nn.Module): + """ + Convert model to half precision in a batchnorm-safe way. + """ + + def __init__(self, network): + super(FP16Model, self).__init__() + self.network = convert_network(network, dtype=torch.half) + + def forward(self, *inputs): + inputs = tuple(t.half() for t in inputs) + return self.network(*inputs) + + +def backwards_debug_hook(grad): + raise RuntimeError("master_params recieved a gradient in the backward pass!") + +def prep_param_lists(model, flat_master=False): + """ + Creates a list of FP32 master parameters for a given model, as in + `Training Neural Networks with Mixed Precision: Real Examples`_. + + Args: + model (torch.nn.Module): Existing Pytorch model + flat_master (bool, optional, default=False): Flatten the master parameters into a single tensor, as a performance optimization. + Returns: + A tuple (``model_params``, ``master_params``). ``model_params`` is a list of the model's parameters for later use with :func:`model_grads_to_master_grads` and :func:`master_params_to_model_params`. ``master_params`` is a list of FP32 master gradients. If ``flat_master=True``, ``master_params`` will be a list with one element. + + Example:: + + model_params, master_params = prep_param_lists(model) + + .. warning:: + Currently, if ``flat_master=True``, all the model's parameters must be the same type. If the model has parameters of different types, use ``flat_master=False``, or use :class:`FP16_Optimizer`. + + .. _`Training Neural Networks with Mixed Precision: Real Examples`: + http://on-demand.gputechconf.com/gtc/2018/video/S81012/ + """ + model_params = [param for param in model.parameters() if param.requires_grad] + + if flat_master: + # Give the user some more useful error messages + try: + # flatten_dense_tensors returns a contiguous flat array. + # http://pytorch.org/docs/master/_modules/torch/_utils.html + master_params = _flatten_dense_tensors([param.data for param in model_params]).float() + except: + print("Error in prep_param_lists: model may contain a mixture of parameters " + "of different types. Use flat_master=False, or use F16_Optimizer.") + raise + master_params = torch.nn.Parameter(master_params) + master_params.requires_grad = True + # master_params.register_hook(backwards_debug_hook) + if master_params.grad is None: + master_params.grad = master_params.new(*master_params.size()) + return model_params, [master_params] + else: + master_params = [param.clone().float().detach() for param in model_params] + for param in master_params: + param.requires_grad = True + return model_params, master_params + + +def model_grads_to_master_grads(model_params, master_params, flat_master=False): + """ + Copy model gradients to master gradients. + + Args: + model_params: List of model parameters created by :func:`prep_param_lists`. + master_params: List of FP32 master parameters created by :func:`prep_param_lists`. If ``master_params`` was created with ``flat_master=True``, ``flat_master=True`` should also be supplied to :func:`model_grads_to_master_grads`. + """ + if flat_master: + # The flattening may incur one more deep copy than is necessary. + master_params[0].grad.data.copy_( + _flatten_dense_tensors([p.grad.data for p in model_params])) + else: + for model, master in zip(model_params, master_params): + if model.grad is not None: + if master.grad is None: + master.grad = Variable(master.data.new(*master.data.size())) + master.grad.data.copy_(model.grad.data) + else: + master.grad = None + + +def master_params_to_model_params(model_params, master_params, flat_master=False): + """ + Copy master parameters to model parameters. + + Args: + model_params: List of model parameters created by :func:`prep_param_lists`. + master_params: List of FP32 master parameters created by :func:`prep_param_lists`. If ``master_params`` was created with ``flat_master=True``, ``flat_master=True`` should also be supplied to :func:`master_params_to_model_params`. + """ + if flat_master: + for model, master in zip(model_params, + _unflatten_dense_tensors(master_params[0].data, model_params)): + model.data.copy_(master) + else: + for model, master in zip(model_params, master_params): + model.data.copy_(master.data) + +# Backward compatibility fixes + +def to_python_float(t): + if hasattr(t, 'item'): + return t.item() + else: + return t[0] + +TORCH_MAJOR = int(torch.__version__.split('.')[0]) +TORCH_MINOR = int(torch.__version__.split('.')[1]) +if TORCH_MAJOR == 0 and TORCH_MINOR <= 4: + clip_grad_norm = torch.nn.utils.clip_grad_norm +else: + clip_grad_norm = torch.nn.utils.clip_grad_norm_ diff --git a/classification/mm_modules/fp16_utils/loss_scaler.py b/classification/mm_modules/fp16_utils/loss_scaler.py new file mode 100644 index 0000000..b9f32fe --- /dev/null +++ b/classification/mm_modules/fp16_utils/loss_scaler.py @@ -0,0 +1,186 @@ +import torch + +# item() is a recent addition, so this helps with backward compatibility. +def to_python_float(t): + if hasattr(t, 'item'): + return t.item() + else: + return t[0] + +class LossScaler: + """ + Class that manages a static loss scale. This class is intended to interact with + :class:`FP16_Optimizer`, and should not be directly manipulated by the user. + + Use of :class:`LossScaler` is enabled via the ``static_loss_scale`` argument to + :class:`FP16_Optimizer`'s constructor. + + Args: + scale (float, optional, default=1.0): The loss scale. + """ + + def __init__(self, scale=1): + self.cur_scale = scale + + # `params` is a list / generator of torch.Variable + def has_overflow(self, params): + return False + + # `x` is a torch.Tensor + def _has_inf_or_nan(x): + return False + + def update_scale(self, overflow): + pass + + @property + def loss_scale(self): + return self.cur_scale + + def scale_gradient(self, module, grad_in, grad_out): + return tuple(self.loss_scale * g for g in grad_in) + + def backward(self, loss, retain_graph=False): + scaled_loss = loss*self.loss_scale + scaled_loss.backward(retain_graph=retain_graph) + +class DynamicLossScaler: + """ + Class that manages dynamic loss scaling. It is recommended to use :class:`DynamicLossScaler` + indirectly, by supplying ``dynamic_loss_scale=True`` to the constructor of + :class:`FP16_Optimizer`. However, it's important to understand how :class:`DynamicLossScaler` + operates, because the default options can be changed using the + the ``dynamic_loss_args`` argument to :class:`FP16_Optimizer`'s constructor. + + Loss scaling is designed to combat the problem of underflowing gradients encountered at long + times when training fp16 networks. Dynamic loss scaling begins by attempting a very high loss + scale. Ironically, this may result in OVERflowing gradients. If overflowing gradients are + encountered, :class:`DynamicLossScaler` informs :class:`FP16_Optimizer` that an overflow has + occurred. + :class:`FP16_Optimizer` then skips the update step for this particular iteration/minibatch, + and :class:`DynamicLossScaler` adjusts the loss scale to a lower value. + If a certain number of iterations occur without overflowing gradients detected, + :class:`DynamicLossScaler` increases the loss scale once more. + In this way :class:`DynamicLossScaler` attempts to "ride the edge" of + always using the highest loss scale possible without incurring overflow. + + Args: + init_scale (float, optional, default=2**32): Initial loss scale attempted by :class:`DynamicLossScaler.` + scale_factor (float, optional, default=2.0): Factor used when adjusting the loss scale. If an overflow is encountered, the loss scale is readjusted to loss scale/``scale_factor``. If ``scale_window`` consecutive iterations take place without an overflow, the loss scale is readjusted to loss_scale*``scale_factor``. + scale_window (int, optional, default=1000): Number of consecutive iterations without an overflow to wait before increasing the loss scale. + """ + + def __init__(self, + init_scale=2**32, + scale_factor=2., + scale_window=1000): + self.cur_scale = init_scale + self.cur_iter = 0 + self.last_overflow_iter = -1 + self.scale_factor = scale_factor + self.scale_window = scale_window + + # `params` is a list / generator of torch.Variable + def has_overflow(self, params): + for p in params: + if p.grad is not None and DynamicLossScaler._has_inf_or_nan(p.grad.data): + return True + + return False + + # `x` is a torch.Tensor + def _has_inf_or_nan(x): + try: + # if x is half, the .float() incurs an additional deep copy, but it's necessary if + # Pytorch's .sum() creates a one-element tensor of the same type as x + # (which is true for some recent version of pytorch). + cpu_sum = float(x.float().sum()) + # More efficient version that can be used if .sum() returns a Python scalar + # cpu_sum = float(x.sum()) + except RuntimeError as instance: + # We want to check if inst is actually an overflow exception. + # RuntimeError could come from a different error. + # If so, we still want the exception to propagate. + if "value cannot be converted" not in instance.args[0]: + raise + return True + else: + if cpu_sum == float('inf') or cpu_sum == -float('inf') or cpu_sum != cpu_sum: + return True + return False + + # `overflow` is boolean indicating whether the gradient overflowed + def update_scale(self, overflow): + if overflow: + # self.cur_scale /= self.scale_factor + self.cur_scale = max(self.cur_scale/self.scale_factor, 1) + self.last_overflow_iter = self.cur_iter + else: + if (self.cur_iter - self.last_overflow_iter) % self.scale_window == 0: + self.cur_scale *= self.scale_factor + self.cur_iter += 1 + + @property + def loss_scale(self): + return self.cur_scale + + def scale_gradient(self, module, grad_in, grad_out): + return tuple(self.loss_scale * g for g in grad_in) + + def backward(self, loss, retain_graph=False): + scaled_loss = loss*self.loss_scale + scaled_loss.backward(retain_graph=retain_graph) + +############################################################## +# Example usage below here -- assuming it's in a separate file +############################################################## +""" +TO-DO separate out into an example. +if __name__ == "__main__": + import torch + from torch.autograd import Variable + from dynamic_loss_scaler import DynamicLossScaler + + # N is batch size; D_in is input dimension; + # H is hidden dimension; D_out is output dimension. + N, D_in, H, D_out = 64, 1000, 100, 10 + + # Create random Tensors to hold inputs and outputs, and wrap them in Variables. + x = Variable(torch.randn(N, D_in), requires_grad=False) + y = Variable(torch.randn(N, D_out), requires_grad=False) + + w1 = Variable(torch.randn(D_in, H), requires_grad=True) + w2 = Variable(torch.randn(H, D_out), requires_grad=True) + parameters = [w1, w2] + + learning_rate = 1e-6 + optimizer = torch.optim.SGD(parameters, lr=learning_rate) + loss_scaler = DynamicLossScaler() + + for t in range(500): + y_pred = x.mm(w1).clamp(min=0).mm(w2) + loss = (y_pred - y).pow(2).sum() * loss_scaler.loss_scale + print('Iter {} loss scale: {}'.format(t, loss_scaler.loss_scale)) + print('Iter {} scaled loss: {}'.format(t, loss.data[0])) + print('Iter {} unscaled loss: {}'.format(t, loss.data[0] / loss_scaler.loss_scale)) + + # Run backprop + optimizer.zero_grad() + loss.backward() + + # Check for overflow + has_overflow = DynamicLossScaler.has_overflow(parameters) + + # If no overflow, unscale grad and update as usual + if not has_overflow: + for param in parameters: + param.grad.data.mul_(1. / loss_scaler.loss_scale) + optimizer.step() + # Otherwise, don't do anything -- ie, skip iteration + else: + print('OVERFLOW!') + + # Update loss scale for next iteration + loss_scaler.update_scale(has_overflow) + +""" diff --git a/classification/mm_modules/utils.py b/classification/mm_modules/utils.py new file mode 100644 index 0000000..c58b062 --- /dev/null +++ b/classification/mm_modules/utils.py @@ -0,0 +1,138 @@ +from __future__ import division +import torch +import torchvision +import numpy as np +import cv2 + +def postprocess(prediction, num_classes, conf_thre=0.7, nms_thre=0.45): + """ + Postprocess for the output of YOLO model + perform box transformation, specify the class for each detection, + and perform class-wise non-maximum suppression. + Args: + prediction (torch tensor): The shape is :math:`(N, B, 4)`. + :math:`N` is the number of predictions, + :math:`B` the number of boxes. The last axis consists of + :math:`xc, yc, w, h` where `xc` and `yc` represent a center + of a bounding box. + num_classes (int): + number of dataset classes. + conf_thre (float): + confidence threshold ranging from 0 to 1, + which is defined in the config file. + nms_thre (float): + IoU threshold of non-max suppression ranging from 0 to 1. + + Returns: + output (list of torch tensor): + + """ + box_corner = prediction.new(prediction.shape) + box_corner[:, :, 0] = prediction[:, :, 0] - prediction[:, :, 2] / 2 + box_corner[:, :, 1] = prediction[:, :, 1] - prediction[:, :, 3] / 2 + box_corner[:, :, 2] = prediction[:, :, 0] + prediction[:, :, 2] / 2 + box_corner[:, :, 3] = prediction[:, :, 1] + prediction[:, :, 3] / 2 + prediction[:, :, :4] = box_corner[:, :, :4] + + output = [None for _ in range(len(prediction))] + for i, image_pred in enumerate(prediction): + + # If none are remaining => process next image + if not image_pred.size(0): + continue + # Get score and class with highest confidence + class_conf, class_pred = torch.max( + image_pred[:, 5:5 + num_classes], 1, keepdim=True) + + conf_mask = (image_pred[:, 4] * class_conf.squeeze() >= conf_thre).squeeze() + # Detections ordered as (x1, y1, x2, y2, obj_conf, class_conf, class_pred) + detections = torch.cat( + (image_pred[:, :5], class_conf, class_pred.float()), 1) + detections = detections[conf_mask] + if not detections.size(0): + continue + + # Iterate through all predicted classes + unique_labels = detections[:, -1].unique() + + for c in unique_labels: + # Get the detections with the particular class + detections_class = detections[detections[:, -1] == c] + nms_out_index = torchvision.ops.nms( + detections_class[:, :4], detections_class[:, 4]*detections_class[:, 5], nms_thre) + detections_class = detections_class[nms_out_index] + if output[i] is None: + output[i] = detections_class + else: + output[i] = torch.cat((output[i], detections_class)) + + return output + + +def bboxes_iou(bboxes_a, bboxes_b, xyxy=True): + """Calculate the Intersection of Unions (IoUs) between bounding boxes. + IoU is calculated as a ratio of area of the intersection + and area of the union. + + Args: + bbox_a (array): An array whose shape is :math:`(N, 4)`. + :math:`N` is the number of bounding boxes. + The dtype should be :obj:`numpy.float32`. + bbox_b (array): An array similar to :obj:`bbox_a`, + whose shape is :math:`(K, 4)`. + The dtype should be :obj:`numpy.float32`. + Returns: + array: + An array whose shape is :math:`(N, K)`. \ + An element at index :math:`(n, k)` contains IoUs between \ + :math:`n` th bounding box in :obj:`bbox_a` and :math:`k` th bounding \ + box in :obj:`bbox_b`. + + from: https://github.com/chainer/chainercv + """ + if bboxes_a.shape[1] != 4 or bboxes_b.shape[1] != 4: + raise IndexError + + if xyxy: + tl = torch.max(bboxes_a[:, None, :2], bboxes_b[:, :2]) + br = torch.min(bboxes_a[:, None, 2:], bboxes_b[:, 2:]) + area_a = torch.prod(bboxes_a[:, 2:] - bboxes_a[:, :2], 1) + area_b = torch.prod(bboxes_b[:, 2:] - bboxes_b[:, :2], 1) + else: + tl = torch.max((bboxes_a[:, None, :2] - bboxes_a[:, None, 2:] / 2), + (bboxes_b[:, :2] - bboxes_b[:, 2:] / 2)) + br = torch.min((bboxes_a[:, None, :2] + bboxes_a[:, None, 2:] / 2), + (bboxes_b[:, :2] + bboxes_b[:, 2:] / 2)) + + area_a = torch.prod(bboxes_a[:, 2:], 1) + area_b = torch.prod(bboxes_b[:, 2:], 1) + en = (tl < br).type(tl.type()).prod(dim=2) + area_i = torch.prod(br - tl, 2) * en # * ((tl < br).all()) + return area_i / (area_a[:, None] + area_b - area_i) + + +def matrix_iou(a,b): + """ + return iou of a and b, numpy version for data augenmentation + """ + lt = np.maximum(a[:, np.newaxis, :2], b[:, :2]) + rb = np.minimum(a[:, np.newaxis, 2:], b[:, 2:]) + + area_i = np.prod(rb - lt, axis=2) * (lt < rb).all(axis=2) + area_a = np.prod(a[:, 2:] - a[:, :2], axis=1) + area_b = np.prod(b[:, 2:] - b[:, :2], axis=1) + return area_i / (area_a[:, np.newaxis] + area_b - area_i+1e-12) + +def visual(img, boxes, scores): + + COLORS = [(255, 0, 0), (0, 255, 0), (0, 0, 255)] + FONT = cv2.FONT_HERSHEY_SIMPLEX + for i in range(boxes.shape[0]): + + cv2.rectangle(img, (int(boxes[i][0]),int(boxes[i][1])),(int(boxes[i][2]),int(boxes[i][3])),COLORS[i%3],2) + cv2.putText(img, 'Object: %.2f'%scores[i],(int(boxes[i][0])-3,int(boxes[i][1])-5), FONT, + 0.4, (0,0,0),2) + + return img + + diff --git a/classification/mm_modules/vis_utils.py b/classification/mm_modules/vis_utils.py new file mode 100644 index 0000000..a322afc --- /dev/null +++ b/classification/mm_modules/vis_utils.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- + +import numpy as np +import os +import matplotlib + +matplotlib.use('AGG') + +import matplotlib.pyplot as plt +import torch +import cv2 +import math +from skimage import transform + +def make_vis(dataset, index, img, fuse_weights, fused_fs): + save_dir = 'vis_output/{}/{}'.format(dataset,index) + os.makedirs(save_dir, exist_ok=True) + + for i in range(len(fuse_weights)): + weights = fuse_weights[i].float().cpu().squeeze().numpy() + max_v = weights.max() + min_v = weights.min() + for j in range(3): + v = weights[j,:,:] + save_name = os.path.join(save_dir, 'level_{}_weight_{}.png'.format(i+1,j+1)) + add_heat(img, v, max_v, min_v, save=save_name) + + fused_f = fused_fs[i].float().cpu().squeeze().numpy() + max_f = fused_f.max() + min_f = fused_f.min() + save_f_name = os.path.join(save_dir, 'fused_feature_level_{}.png'.format(i+1)) + add_heat(img, fused_f, max_f, min_f, save=save_f_name) + +def make_pred_vis(dataset,index, img, class_names, bboxes, cls, scores): + save_preddir = 'vis_output/{}/pred/'.format(dataset) + os.makedirs(save_preddir, exist_ok=True) + + save_pred_name = os.path.join(save_preddir,'{}.png'.format(index)) + + bboxes = bboxes.numpy() + scores = scores.numpy() + cls_ids = cls.numpy() + + im = vis(img, bboxes, scores, cls_ids, class_names) + + cv2.imwrite(save_pred_name, im) + +def vis(img, boxes, scores, cls_ids, conf=0.5, class_names=None, color=None): + + colors = torch.FloatTensor([[1,0,1],[0,0,1],[0,1,1],[0,1,0],[1,1,0],[1,0,0]]); + def get_color(c, x, max_val): + ratio = float(x)/max_val * 5 + i = int(math.floor(ratio)) + j = int(math.ceil(ratio)) + ratio = ratio - i + r = (1-ratio) * colors[i][c] + ratio*colors[j][c] + return int(r*255) + + width = img.shape[1] + height = img.shape[0] + for i in range(len(boxes)): + box = boxes[i] + cls_conf = scores[i] + if cls_conf < conf: + continue + x1 = int(box[0]) + y1 = int(box[1]) + x2 = int(box[0]+box[2]) + y2 = int(box[1]+box[3]) + + + if color: + rgb = color + else: + rgb = (255, 0, 0) + if class_names is not None: + cls_conf = scores[i] + cls_id = int(cls_ids[i]) + class_name = class_names[cls_id] + classes = len(class_names) + offset = cls_id * 123456 % classes + red = get_color(2, offset, classes) + green = get_color(1, offset, classes) + blue = get_color(0, offset, classes) + if color is None: + rgb = (red, green, blue) + img = cv2.putText(img, '%s: %.2f'%(class_name,cls_conf), (x1,y1-5), cv2.FONT_HERSHEY_SIMPLEX, 0.3, rgb, 1) + img = cv2.rectangle(img, (x1,y1), (x2,y2), rgb, 1) + return img + +def add_heat(image, heat_map, max_v, min_v, alpha=0.4, save=None, cmap='jet', axis='off'): + height = image.shape[0] + width = image.shape[1] + + # resize heat map + heat_map_resized = transform.resize(heat_map, (height, width)) + + # normalize heat map + max_value = max_v + min_value = min_v + normalized_heat_map = (heat_map_resized - min_value) / (max_value - min_value) + + # display + plt.imshow(image) + plt.imshow(255 * normalized_heat_map, alpha=alpha, cmap=cmap) + plt.axis(axis) + + if save is not None: + plt.savefig(save, bbox_inches='tight', pad_inches=0) + + + + diff --git a/classification/mm_modules/voc_evaluator.py b/classification/mm_modules/voc_evaluator.py new file mode 100644 index 0000000..1fc5ae8 --- /dev/null +++ b/classification/mm_modules/voc_evaluator.py @@ -0,0 +1,204 @@ +import json +import tempfile +import sys +from tqdm import tqdm + +from pycocotools.cocoeval import COCOeval +from torch.autograd import Variable + +from dataset.vocdataset import * +from dataset.data_augment import ValTransform +from utils.utils import * +from utils import distributed_util +from utils.vis_utils import make_vis, make_pred_vis + +import time + +#DEBUG = True +DEBUG = False + +def _accumulate_predictions_from_multiple_gpus(predictions_per_gpu): + all_predictions = distributed_util.scatter_gather(predictions_per_gpu) + if not distributed_util.is_main_process(): + return + # merge the list of dicts + predictions = {} + for p in all_predictions: + predictions.update(p) + # convert a dict where the key is the index in a list + image_ids = list(sorted(predictions.keys())) + if len(image_ids) != image_ids[-1] + 1: + print('num_imgs: ',len(image_ids)) + print('last img_id: ',image_ids[-1]) + print( + "Number of images that were gathered from multiple processes is not " + "a contiguous set. Some images might be missing from the evaluation" + ) + + # convert to a list + predictions = [predictions[i] for i in image_ids] + return predictions + + +class VOCEvaluator(): + """ + COCO AP Evaluation class. + All the data in the val2017 dataset are processed \ + and evaluated by COCO API. + """ + def __init__(self, data_dir, img_size, confthre, nmsthre,vis=False): + """ + Args: + data_dir (str): dataset root directory + img_size (int): image size after preprocess. images are resized \ + to squares whose shape is (img_size, img_size). + confthre (float): + confidence threshold ranging from 0 to 1, \ + which is defined in the config file. + nmsthre (float): + IoU threshold of non-max supression ranging from 0 to 1. + """ + test_sets = [('2007', 'test'),] + self.dataset = VOCDetection( + root=data_dir, + image_sets = test_sets, + input_dim=img_size, + preproc = ValTransform(rgb_means=(0.485, 0.456, 0.406),std=(0.229, 0.224, 0.225)),) + self.num_images = len(self.dataset) + self.dataloader = torch.utils.data.DataLoader( + self.dataset, batch_size=1, shuffle=False, num_workers=0) + self.img_size = img_size + self.confthre = confthre + self.nmsthre = nmsthre + self.vis=vis + + def evaluate(self, model, half=False): + """ + COCO average precision (AP) Evaluation. Iterate inference on the test dataset + and the results are evaluated by COCO API. + Args: + model : model object + Returns: + ap50_95 (float) : calculated COCO AP for IoU=50:95 + ap50 (float) : calculated COCO AP for IoU=50 + """ + if isinstance(model, torch.nn.parallel.DistributedDataParallel): + model = model.module + model.eval() + cuda = torch.cuda.is_available() + if half: + Tensor = torch.cuda.HalfTensor if cuda else torch.HalfTensor + else: + Tensor = torch.cuda.FloatTensor if cuda else torch.FloatTensor + + ids = [] + data_dict = [] + dataiterator = iter(self.dataloader) + img_num = 0 + indices = list(range(self.num_images)) + dis_indices = indices[distributed_util.get_rank()::distributed_util.get_world_size()] + progress_bar = tqdm if distributed_util.is_main_process() else iter + num_classes = 20 + predictions = {} + + if distributed_util.is_main_process(): + inference_time=0 + nms_time=0 + n_samples=len(dis_indices) + + for i in progress_bar(dis_indices): + img, _, info_img, id_ = self.dataset[i] # load a batch + info_img = [float(info) for info in info_img] + ids.append(id_) + with torch.no_grad(): + img = Variable(img.type(Tensor).unsqueeze(0)) + + if distributed_util.is_main_process() and i > 9: + start=time.time() + + if self.vis: + outputs,fuse_weights,fused_f = model(img) + else: + outputs = model(img) + + if distributed_util.is_main_process() and i > 9: + infer_end=time.time() + inference_time += (infer_end-start) + + outputs = postprocess( + outputs, 20, self.confthre, self.nmsthre) + + + if distributed_util.is_main_process() and i > 9: + nms_end=time.time() + nms_time +=(nms_end-infer_end) + + if outputs[0] is None: + predictions[i] = (None, None, None) + continue + outputs = outputs[0].cpu().data + + bboxes = outputs[:, 0:4] + bboxes[:, 0::2] *= info_img[0] / self.img_size[0] + bboxes[:, 1::2] *= info_img[1] / self.img_size[1] + cls = outputs[:, 6] + scores = outputs[:, 4]* outputs[:,5] + predictions[i] = (bboxes, cls, scores) + + if self.vis: + o_img,_,_,_ = self.dataset.pull_item(i) + make_vis('VOC', i, o_img, fuse_weights, fused_f) + class_names = self.dataset._classes + + bbox = bboxes.clone() + bbox[:, 2] = bbox[:,2] - bbox[:,0] + bbox[:, 3] = bbox[:,3] - bbox[:,1] + + make_pred_vis('VOC', i, o_img, class_names, bbox, cls, scores) + + if DEBUG and distributed_util.is_main_process(): + o_img,_,_,_ = self.dataset.pull_item(i) + class_names = self.dataset._classes + bbox = bboxes.clone() + bbox[:, 2] = bbox[:,2] - bbox[:,0] + bbox[:, 3] = bbox[:,3] - bbox[:,1] + make_pred_vis('VOC', i, o_img, class_names, bbox, cls, scores) + + distributed_util.synchronize() + predictions = _accumulate_predictions_from_multiple_gpus(predictions) + if not distributed_util.is_main_process(): + return 0, 0 + + + print('Main process Evaluating...') + + a_infer_time = 1000*inference_time / (n_samples-10) + a_nms_time= 1000*nms_time / (n_samples-10) + + print('Average forward time: %.2f ms, Average NMS time: %.2f ms, Average inference time: %.2f ms' %(a_infer_time, \ + a_nms_time, (a_infer_time+a_nms_time))) + + all_boxes = [[[] for _ in range(self.num_images)] + for _ in range(num_classes)] + for img_num in range(self.num_images): + bboxes, cls, scores = predictions[img_num] + if bboxes is None: + for j in range(num_classes): + all_boxes[j][img_num] = np.empty([0,5],dtype=np.float32) + continue + for j in range(num_classes): + mask_c = (cls == j) + if sum(mask_c) ==0: + all_boxes[j][img_num] = np.empty([0,5],dtype=np.float32) + continue + + c_dets = torch.cat((bboxes, scores.unsqueeze(1)),dim=1) + all_boxes[j][img_num] = c_dets[mask_c].numpy() + + sys.stdout.write('im_eval: {:d}/{:d} \r'.format(img_num+1, self.num_images)) + sys.stdout.flush() + + with tempfile.TemporaryDirectory() as tempdir: + mAP50, mAP70 = self.dataset.evaluate_detections(all_boxes, tempdir) + return mAP50,mAP70 + diff --git a/classification/models/__init__.py b/classification/models/__init__.py new file mode 100644 index 0000000..2d9c65e --- /dev/null +++ b/classification/models/__init__.py @@ -0,0 +1 @@ +from .build import build_model \ No newline at end of file diff --git a/classification/models/attentions.py b/classification/models/attentions.py new file mode 100644 index 0000000..505ab87 --- /dev/null +++ b/classification/models/attentions.py @@ -0,0 +1,284 @@ +import os +import torch +import torch.nn as nn +from timm.models.layers import DropPath, to_2tuple, trunc_normal_ +import math +import torch.nn.functional as F + +def conv_flops(k, c_in, c_out, stride, padding, resolution, bias=True, dialation=1, groups=1): + batch_size = 1 + output_dims = math.floor((resolution + 2*padding - dialation*(k - 1) - 1) / stride + 1) + kernel_dims = k + in_channels = c_in + out_channels = c_out + # groups = 1 + + filters_per_channel = out_channels // groups + conv_per_position_flops = int(kernel_dims**2) * in_channels * filters_per_channel + + active_elements_count = batch_size * int(output_dims**2) + + overall_conv_flops = conv_per_position_flops * active_elements_count + bias_flops = 0 + + if bias is not None: + bias_flops = out_channels * active_elements_count + + overall_flops = overall_conv_flops + bias_flops + + return int(overall_flops) + +class Attention(nn.Module): + def __init__(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0.): + super().__init__() + self.num_heads = num_heads + self.dim = dim + head_dim = dim // num_heads + self.scale = qk_scale or head_dim ** -0.5 + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x): + B, N, C = x.shape + qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[2] # make torchscript happy (cannot use tensor as tuple) + + attn = (q @ k.transpose(-2, -1)) * self.scale + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + + def flops(self, N): + # calculate flops for 1 window with token length of N + flops = 0 + # qkv = self.qkv(x) + flops += N * self.dim * 3 * self.dim + # attn = (q @ k.transpose(-2, -1)) + flops += self.num_heads * N * (self.dim // self.num_heads) * N + # x = (attn @ v) + flops += self.num_heads * N * N * (self.dim // self.num_heads) + # x = self.proj(x) + flops += N * self.dim * self.dim + return flops + + +class RelPosAttention(nn.Module): + + def __init__(self, dim, input_resolution, num_heads, qkv_bias=True, qk_scale=None, attn_drop=0., proj_drop=0.): + + super().__init__() + self.dim = dim + self.input_resolution = input_resolution # Wh, Ww + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = qk_scale or head_dim ** -0.5 + + # define a parameter table of relative position bias + self.relative_position_bias_table = nn.Parameter( + torch.zeros((2 * input_resolution[0] - 1) * (2 * input_resolution[1] - 1), num_heads)) # 2*Wh-1 * 2*Ww-1, nH + + # get pair-wise relative position index for each token inside the window + coords_h = torch.arange(self.input_resolution[0]) + coords_w = torch.arange(self.input_resolution[1]) + coords = torch.stack(torch.meshgrid([coords_h, coords_w])) # 2, Wh, Ww + coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww + relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :] # 2, Wh*Ww, Wh*Ww + relative_coords = relative_coords.permute(1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2 + relative_coords[:, :, 0] += self.input_resolution[0] - 1 # shift to start from 0 + relative_coords[:, :, 1] += self.input_resolution[1] - 1 + relative_coords[:, :, 0] *= 2 * self.input_resolution[1] - 1 + relative_position_index = relative_coords.sum(-1) # Wh*Ww, Wh*Ww + self.register_buffer("relative_position_index", relative_position_index) + trunc_normal_(self.relative_position_bias_table, std=.02) + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + self.softmax = nn.Softmax(dim=-1) + + def forward(self, x): + B_, N, C = x.shape + qkv = self.qkv(x).reshape(B_, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[2] # make torchscript happy (cannot use tensor as tuple) + + q = q * self.scale + attn = (q @ k.transpose(-2, -1)) + + relative_position_bias = self.relative_position_bias_table[self.relative_position_index.view(-1)].view( + self.input_resolution[0] * self.input_resolution[1], self.input_resolution[0] * self.input_resolution[1], -1) # Wh*Ww,Wh*Ww,nH + relative_position_bias = relative_position_bias.permute(2, 0, 1).contiguous() # nH, Wh*Ww, Wh*Ww + attn = attn + relative_position_bias.unsqueeze(0) + + attn = self.softmax(attn) + + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B_, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + + def extra_repr(self) -> str: + return f'dim={self.dim}, input_resolution={self.input_resolution}, num_heads={self.num_heads}' + + def flops(self, N): + # calculate flops for 1 window with token length of N + flops = 0 + # qkv = self.qkv(x) + flops += N * self.dim * 3 * self.dim + # attn = (q @ k.transpose(-2, -1)) + flops += self.num_heads * N * (self.dim // self.num_heads) * N + # x = (attn @ v) + flops += self.num_heads * N * N * (self.dim // self.num_heads) + # x = self.proj(x) + flops += N * self.dim * self.dim + return flops + + +class HiLo(nn.Module): + """ + HiLo Attention + + Link: https://arxiv.org/abs/2205.13213 + """ + def __init__(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0., window_size=2, alpha=0.5): + super().__init__() + assert dim % num_heads == 0, f"dim {dim} should be divided by num_heads {num_heads}." + head_dim = int(dim/num_heads) + self.dim = dim + + # self-attention heads in Lo-Fi + self.l_heads = int(num_heads * alpha) + # token dimension in Lo-Fi + self.l_dim = self.l_heads * head_dim + + # self-attention heads in Hi-Fi + self.h_heads = num_heads - self.l_heads + # token dimension in Hi-Fi + self.h_dim = self.h_heads * head_dim + + # local window size. The `s` in our paper. + self.ws = window_size + + if self.ws == 1: + # ws == 1 is equal to a standard multi-head self-attention + self.h_heads = 0 + self.h_dim = 0 + self.l_heads = num_heads + self.l_dim = dim + + self.scale = qk_scale or head_dim ** -0.5 + + # Low frequence attention (Lo-Fi) + if self.l_heads > 0: + if self.ws != 1: + self.sr = nn.AvgPool2d(kernel_size=window_size, stride=window_size) + self.l_q = nn.Linear(self.dim, self.l_dim, bias=qkv_bias) + self.l_kv = nn.Linear(self.dim, self.l_dim * 2, bias=qkv_bias) + self.l_proj = nn.Linear(self.l_dim, self.l_dim) + + # High frequence attention (Hi-Fi) + if self.h_heads > 0: + self.h_qkv = nn.Linear(self.dim, self.h_dim * 3, bias=qkv_bias) + self.h_proj = nn.Linear(self.h_dim, self.h_dim) + + def hifi(self, x): + B, H, W, C = x.shape + h_group, w_group = H // self.ws, W // self.ws + + total_groups = h_group * w_group + + x = x.reshape(B, h_group, self.ws, w_group, self.ws, C).transpose(2, 3) + + qkv = self.h_qkv(x).reshape(B, total_groups, -1, 3, self.h_heads, self.h_dim // self.h_heads).permute(3, 0, 1, 4, 2, 5) + q, k, v = qkv[0], qkv[1], qkv[2] # B, hw, n_head, ws*ws, head_dim + + attn = (q @ k.transpose(-2, -1)) * self.scale # B, hw, n_head, ws*ws, ws*ws + attn = attn.softmax(dim=-1) + attn = (attn @ v).transpose(2, 3).reshape(B, h_group, w_group, self.ws, self.ws, self.h_dim) + x = attn.transpose(2, 3).reshape(B, h_group * self.ws, w_group * self.ws, self.h_dim) + + x = self.h_proj(x) + return x + + def lofi(self, x): + B, H, W, C = x.shape + + q = self.l_q(x).reshape(B, H * W, self.l_heads, self.l_dim // self.l_heads).permute(0, 2, 1, 3) + + if self.ws > 1: + x_ = x.permute(0, 3, 1, 2) + x_ = self.sr(x_).reshape(B, C, -1).permute(0, 2, 1) + kv = self.l_kv(x_).reshape(B, -1, 2, self.l_heads, self.l_dim // self.l_heads).permute(2, 0, 3, 1, 4) + else: + kv = self.l_kv(x).reshape(B, -1, 2, self.l_heads, self.l_dim // self.l_heads).permute(2, 0, 3, 1, 4) + k, v = kv[0], kv[1] + + attn = (q @ k.transpose(-2, -1)) * self.scale + attn = attn.softmax(dim=-1) + + x = (attn @ v).transpose(1, 2).reshape(B, H, W, self.l_dim) + x = self.l_proj(x) + return x + + def forward(self, x): + B, N, C = x.shape + H = W = int(N ** 0.5) + + x = x.reshape(B, H, W, C) + + if self.h_heads == 0: + x = self.lofi(x) + return x.reshape(B, N, C) + + if self.l_heads == 0: + x = self.hifi(x) + return x.reshape(B, N, C) + + hifi_out = self.hifi(x) + lofi_out = self.lofi(x) + + x = torch.cat((hifi_out, lofi_out), dim=-1) + x = x.reshape(B, N, C) + return x + + def flops(self, N): + H = int(N ** 0.5) + # when the height and width cannot be divided by ws, we pad the feature map in the same way as Swin Transformer for object detection/segmentation + Hp = Wp = self.ws * math.ceil(H / self.ws) + + Np = Hp * Wp + + # For Hi-Fi + # qkv + hifi_flops = Np * self.dim * self.h_dim * 3 + nW = Np / self.ws / self.ws + window_len = self.ws * self.ws + # q @ k and attn @ v + window_flops = window_len * window_len * self.h_dim * 2 + hifi_flops += nW * window_flops + # projection + hifi_flops += Np * self.h_dim * self.h_dim + + # for Lo-Fi + # q + lofi_flops = Np * self.dim * self.l_dim + # H = int(Np ** 0.5) + kv_len = (Hp // self.ws) ** 2 + # k, v + lofi_flops += kv_len * self.dim * self.l_dim * 2 + # q @ k and attn @ v + lofi_flops += Np * self.l_dim * kv_len * 2 + # projection + lofi_flops += Np * self.l_dim * self.l_dim + + return hifi_flops + lofi_flops + diff --git a/classification/models/blocks.py b/classification/models/blocks.py new file mode 100644 index 0000000..6b93ce7 --- /dev/null +++ b/classification/models/blocks.py @@ -0,0 +1,141 @@ +from models.attentions import * + + +class DWConv(nn.Module): + def __init__(self, dim=768): + super(DWConv, self).__init__() + self.dwconv = nn.Conv2d(dim, dim, 3, 1, 1, bias=True, groups=dim) + + def forward(self, x): + B, N, C = x.shape + H = W = int(math.sqrt(N)) + x = x.transpose(1, 2).view(B, C, H, W) + x = self.dwconv(x) + x = x.flatten(2).transpose(1, 2) + + return x + +class DWMlp(nn.Module): + def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0., linear=False): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.dwconv = DWConv(hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + self.linear = linear + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + elif isinstance(m, nn.Conv2d): + fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + fan_out //= m.groups + m.weight.data.normal_(0, math.sqrt(2.0 / fan_out)) + if m.bias is not None: + m.bias.data.zero_() + + def forward(self, x): + x = self.fc1(x) + x = self.dwconv(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + +class Mlp(nn.Module): + def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + +class ConvFFNBlock(nn.Module): + + def __init__(self, dim, input_resolution, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0., + drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm, local_ws=1, alpha=0.5): + super().__init__() + self.input_resolution = input_resolution + self.dim = dim + self.mlp_ratio = mlp_ratio + self.local_ws=local_ws + # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = DWMlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) + + def forward(self, x): + x = x + self.drop_path(self.mlp(self.norm2(x))) + return x + + def flops(self): + flops = 0 + H, W = self.input_resolution + # dw conv + mlp_hidden_dim = self.dim * self.mlp_ratio + flops += conv_flops(3, mlp_hidden_dim, mlp_hidden_dim, 1, 1, H, groups=mlp_hidden_dim) + # mlp + flops += 2 * H * W * self.dim * self.dim * self.mlp_ratio + # norm2 + flops += self.dim * H * W + return flops + +class Block(nn.Module): + + def __init__(self, dim, input_resolution, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0., + drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm, local_ws=1, alpha=0.5): + super().__init__() + self.input_resolution = input_resolution + self.dim = dim + self.mlp_ratio = mlp_ratio + self.norm1 = norm_layer(dim) + self.attn = HiLo(dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, + attn_drop=attn_drop, proj_drop=drop, window_size=local_ws, alpha=alpha) + # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = DWMlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) + + def forward(self, x): + x = x + self.drop_path(self.attn(self.norm1(x))) + x = x + self.drop_path(self.mlp(self.norm2(x))) + return x + + def flops(self): + flops = 0 + H, W = self.input_resolution + # norm1 + flops += self.dim * H * W + # attn + msa_flops = self.attn.flops(H * W) + # dw conv + mlp_hidden_dim = self.dim * self.mlp_ratio + flops += conv_flops(3, mlp_hidden_dim, mlp_hidden_dim, 1, 1, H, groups=mlp_hidden_dim) + # mlp + mlp_flops = 2 * H * W * self.dim * self.dim * self.mlp_ratio + # norm2 + flops += self.dim * H * W + msa_flops + mlp_flops + return flops + diff --git a/classification/models/build.py b/classification/models/build.py new file mode 100644 index 0000000..c76c06a --- /dev/null +++ b/classification/models/build.py @@ -0,0 +1,28 @@ +from .litv2 import LITv2 + + +def build_model(config): + model_type = config.MODEL.TYPE + if model_type == 'litv2': + model = LITv2(img_size=config.DATA.IMG_SIZE, + patch_size=config.MODEL.LIT.PATCH_SIZE, + in_chans=config.MODEL.LIT.IN_CHANS, + num_classes=config.MODEL.NUM_CLASSES, + embed_dim=config.MODEL.LIT.EMBED_DIM, + depths=config.MODEL.LIT.DEPTHS, + num_heads=config.MODEL.LIT.NUM_HEADS, + mlp_ratio=config.MODEL.LIT.MLP_RATIO, + qkv_bias=config.MODEL.LIT.QKV_BIAS, + qk_scale=config.MODEL.LIT.QK_SCALE, + drop_rate=config.MODEL.DROP_RATE, + drop_path_rate=config.MODEL.DROP_PATH_RATE, + ape=config.MODEL.LIT.APE, + patch_norm=config.MODEL.LIT.PATCH_NORM, + use_checkpoint=config.TRAIN.USE_CHECKPOINT, + alpha=config.MODEL.LIT.ALPHA, + local_ws=config.MODEL.LIT.LOCAL_WS + ) + else: + raise NotImplementedError(f"Unkown model: {model_type}") + + return model diff --git a/classification/models/layers.py b/classification/models/layers.py new file mode 100644 index 0000000..6d425fc --- /dev/null +++ b/classification/models/layers.py @@ -0,0 +1,70 @@ +from models.blocks import * +import torch.utils.checkpoint as checkpoint + +class LITLayer(nn.Module): + """ A basic LIT layer for one stage. + + Args: + dim (int): Number of input channels. + input_resolution (tuple[int]): Input resolution. + depth (int): Number of blocks. + num_heads (int): Number of attention heads. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None + use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. + """ + + def __init__(self, dim, input_resolution, depth, num_heads, + mlp_ratio=4., qkv_bias=True, qk_scale=None, drop=0., attn_drop=0., + drop_path=0., norm_layer=nn.LayerNorm, downsample=None, use_checkpoint=False, has_msa=True, local_ws=1, alpha=0.5): + + super().__init__() + self.dim = dim + self.input_resolution = input_resolution + self.depth = depth + self.use_checkpoint = use_checkpoint + + # build blocks + block = Block if has_msa else ConvFFNBlock + self.blocks = nn.ModuleList([ + block(dim=dim, input_resolution=input_resolution, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, qk_scale=qk_scale, + drop=drop, attn_drop=attn_drop, + drop_path=drop_path[i] if isinstance(drop_path, list) else drop_path, + norm_layer=norm_layer, local_ws=local_ws, alpha=alpha) + for i in range(depth)]) + + # patch merging layer + if downsample is not None: + self.downsample = downsample(input_resolution, dim=dim, norm_layer=norm_layer) + else: + self.downsample = None + + def forward(self, x): + for blk in self.blocks: + if self.use_checkpoint: + x = checkpoint.checkpoint(blk, x) + else: + x = blk(x) + if self.downsample is not None: + x = self.downsample(x) + return x + + def extra_repr(self) -> str: + return f"dim={self.dim}, input_resolution={self.input_resolution}, depth={self.depth}" + + def flops(self): + flops = 0 + for blk in self.blocks: + flops += blk.flops() + if self.downsample is not None: + flops += self.downsample.flops() + return flops diff --git a/classification/models/litv2.py b/classification/models/litv2.py new file mode 100644 index 0000000..02f2de3 --- /dev/null +++ b/classification/models/litv2.py @@ -0,0 +1,111 @@ +import torch.nn as nn +from models.patch_embed import * +from models.layers import * + +def flops_to_string(flops, units='G', precision=2): + if units.startswith('G'): + return str(round(flops / 10. ** 9, precision)) + ' ' + units + elif units.startswith('M'): + return str(round(flops / 10. ** 6, precision)) + ' ' + units + elif units.startswith('K'): + return str(round(flops / 10. ** 3, precision)) + ' ' + units + else: + return str(flops) + units + +class LITv2(nn.Module): + def __init__(self, img_size=224, patch_size=4, in_chans=3, num_classes=1000, + embed_dim=96, depths=[2, 2, 6, 2], num_heads=[3, 6, 12, 24], + mlp_ratio=4., qkv_bias=True, qk_scale=None, + drop_rate=0., attn_drop_rate=0., drop_path_rate=0.1, + norm_layer=nn.LayerNorm, ape=False, patch_norm=True, + use_checkpoint=False, has_msa=[0, 0, 1, 1], alpha=0.5, local_ws=None, **kwargs): + super().__init__() + + self.num_classes = num_classes + self.num_layers = len(depths) + self.embed_dim = embed_dim + self.ape = ape + self.patch_norm = patch_norm + self.num_features = int(embed_dim * 2 ** (self.num_layers - 1)) + self.mlp_ratio = mlp_ratio + self.has_msa = has_msa + self.local_ws = local_ws + + # split image into non-overlapping patches + self.patch_embed = PatchEmbed( + img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim, + norm_layer=norm_layer if self.patch_norm else None) + + patches_resolution = self.patch_embed.patches_resolution + self.patches_resolution = patches_resolution + + self.pos_drop = nn.Dropout(p=drop_rate) + + # stochastic depth + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))] # stochastic depth decay rule + + # build layers + self.layers = nn.ModuleList() + for i_layer in range(self.num_layers): + layer = LITLayer(dim=int(embed_dim * 2 ** i_layer), + input_resolution=(patches_resolution[0] // (2 ** i_layer), + patches_resolution[1] // (2 ** i_layer)), + depth=depths[i_layer], + num_heads=num_heads[i_layer], + mlp_ratio=self.mlp_ratio, + qkv_bias=qkv_bias, qk_scale=qk_scale, + drop=drop_rate, attn_drop=attn_drop_rate, + drop_path=dpr[sum(depths[:i_layer]):sum(depths[:i_layer + 1])], + norm_layer=norm_layer, + downsample=DTM if (i_layer < self.num_layers - 1) else None, + use_checkpoint=use_checkpoint, has_msa=self.has_msa[i_layer]==1, local_ws=self.local_ws[i_layer], alpha=alpha) + self.layers.append(layer) + + self.norm = norm_layer(self.num_features) + self.avgpool = nn.AdaptiveAvgPool1d(1) + self.head = nn.Linear(self.num_features, num_classes) if num_classes > 0 else nn.Identity() + + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + @torch.jit.ignore + def no_weight_decay(self): + return {'absolute_pos_embed'} + + @torch.jit.ignore + def no_weight_decay_keywords(self): + return {'relative_position_bias_table'} + + def forward_features(self, x): + x = self.patch_embed(x) + + for layer in self.layers: + x = layer(x) + + x = self.norm(x) # B L C + x = self.avgpool(x.transpose(1, 2)) # B C 1 + x = torch.flatten(x, 1) + return x + + def forward(self, x): + x = self.forward_features(x) + x = self.head(x) + return x + + def flops(self): + flops = 0 + flops += self.patch_embed.flops() + for i, layer in enumerate(self.layers): + temp = layer.flops() + flops += temp + flops += self.num_features * self.patches_resolution[0] * self.patches_resolution[1] // (2 ** self.num_layers) + flops += self.num_features * self.num_classes + return flops \ No newline at end of file diff --git a/classification/models/patch_embed.py b/classification/models/patch_embed.py new file mode 100644 index 0000000..514d552 --- /dev/null +++ b/classification/models/patch_embed.py @@ -0,0 +1,132 @@ +import torch +import torch.nn as nn +from mm_modules.DCN.modules.deform_conv2d import DeformConv2dPack +from timm.models.layers import DropPath, to_2tuple, trunc_normal_ +import math + +def conv_flops(k, c_in, c_out, stride, padding, resolution, bias=True, dialation=1): + batch_size = 1 + output_dims = math.floor((resolution + 2*padding - dialation*(k - 1) - 1) / stride + 1) + kernel_dims = k + in_channels = c_in + out_channels = c_out + groups = 1 + + filters_per_channel = out_channels // groups + conv_per_position_flops = int(kernel_dims**2) * in_channels * filters_per_channel + + active_elements_count = batch_size * int(output_dims**2) + + overall_conv_flops = conv_per_position_flops * active_elements_count + bias_flops = 0 + + if bias is not None: + bias_flops = out_channels * active_elements_count + + overall_flops = overall_conv_flops + bias_flops + + return int(overall_flops) + + +class PatchEmbed(nn.Module): + r""" Image to Patch Embedding + + Args: + img_size (int): Image size. Default: 224. + patch_size (int): Patch token size. Default: 4. + in_chans (int): Number of input image channels. Default: 3. + embed_dim (int): Number of linear projection output channels. Default: 96. + norm_layer (nn.Module, optional): Normalization layer. Default: None + """ + + def __init__(self, img_size=224, patch_size=4, in_chans=3, embed_dim=96, norm_layer=None): + super().__init__() + img_size = to_2tuple(img_size) + patch_size = to_2tuple(patch_size) + patches_resolution = [img_size[0] // patch_size[0], img_size[1] // patch_size[1]] + self.img_size = img_size + self.patch_size = patch_size + self.patches_resolution = patches_resolution + self.num_patches = patches_resolution[0] * patches_resolution[1] + + self.in_chans = in_chans + self.embed_dim = embed_dim + + self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size) + if norm_layer is not None: + self.norm = norm_layer(embed_dim) + else: + self.norm = None + + def forward(self, x): + B, C, H, W = x.shape + # FIXME look at relaxing size constraints + assert H == self.img_size[0] and W == self.img_size[1], \ + f"Input image size ({H}*{W}) doesn't match model ({self.img_size[0]}*{self.img_size[1]})." + x = self.proj(x).flatten(2).transpose(1, 2) # B Ph*Pw C + # _, _, H, W = x.shape + if self.norm is not None: + x = self.norm(x) + return x + + def flops(self): + Ho, Wo = self.patches_resolution + flops = Ho * Wo * self.embed_dim * self.in_chans * (self.patch_size[0] * self.patch_size[1]) + if self.norm is not None: + flops += Ho * Wo * self.embed_dim + return flops + + +class DTM(nn.Module): + r""" Deformable Token Merging. + + Link: https://arxiv.org/abs/2105.14217 + + Args: + input_resolution (tuple[int]): Resolution of input feature. + dim (int): Number of input channels. + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__(self, input_resolution, dim, norm_layer=nn.LayerNorm): + super().__init__() + self.input_resolution = input_resolution + self.dim = dim + self.kernel_size = 2 + self.stride = 2 + self.padding = 0 + self.c_in = dim + self.c_out = dim*2 + self.dconv = DeformConv2dPack(dim, dim*2, kernel_size=2, stride=2, padding=0) + self.norm_layer = nn.BatchNorm2d(dim*2) + self.act_layer = nn.GELU() + + def forward(self, x, return_offset=False): + """ + x: B, H*W, C + """ + H, W = self.input_resolution + B, L, C = x.shape + assert L == H * W, "input feature has wrong size" + assert H % 2 == 0 and W % 2 == 0, f"x size ({H}*{W}) are not even." + + x = x.reshape(B, H, W, C).permute(0, 3, 1, 2).contiguous() + x, offset = self.dconv(x, return_offset=False) + x = self.act_layer(self.norm_layer(x)).flatten(2).transpose(1, 2) + if return_offset: + return x, offset + else: + return x + + def extra_repr(self) -> str: + return f"input_resolution={self.input_resolution}, dim={self.dim}" + + def flops(self): + H, W = self.input_resolution + offset_flops = conv_flops(self.kernel_size, self.c_in, 12, self.stride, self.padding, H) + dconv_flops = conv_flops(self.kernel_size, self.c_in, self.c_out, self.stride, self.padding, H) + deformable_flops = offset_flops + dconv_flops + # norm layer + norm_flops = (H // 2) * (W // 2) * 2 * self.dim + + return deformable_flops + norm_flops diff --git a/classification/optimizer.py b/classification/optimizer.py new file mode 100644 index 0000000..34586d5 --- /dev/null +++ b/classification/optimizer.py @@ -0,0 +1,60 @@ +from torch import optim as optim + + +def build_optimizer(config, model): + """ + Build optimizer, set weight decay of normalization to 0 by default. + """ + skip = {} + skip_keywords = {} + if hasattr(model, 'no_weight_decay'): + skip = model.no_weight_decay() + if hasattr(model, 'no_weight_decay_keywords'): + skip_keywords = model.no_weight_decay_keywords() + parameters = set_weight_decay(model, config, skip, skip_keywords) + + opt_lower = config.TRAIN.OPTIMIZER.NAME.lower() + optimizer = None + if opt_lower == 'sgd': + optimizer = optim.SGD(parameters, momentum=config.TRAIN.OPTIMIZER.MOMENTUM, nesterov=True, + lr=config.TRAIN.BASE_LR, weight_decay=config.TRAIN.WEIGHT_DECAY) + elif opt_lower == 'adamw': + optimizer = optim.AdamW(parameters, eps=config.TRAIN.OPTIMIZER.EPS, betas=config.TRAIN.OPTIMIZER.BETAS, + lr=config.TRAIN.BASE_LR, weight_decay=config.TRAIN.WEIGHT_DECAY) + + return optimizer + + +def set_weight_decay(model, config, skip_list=(), skip_keywords=()): + has_decay = [] + no_decay = [] + dconv = [] + for name, param in model.named_parameters(): + if not param.requires_grad: + continue # frozen weights + if len(param.shape) == 1 or name.endswith(".bias") or (name in skip_list) or \ + check_keywords_in_name(name, skip_keywords): + no_decay.append(param) + # print(f"{name} has no weight decay") + else: + if 'conv_offset' in name and config.MODEL.OFFSET_LR_MULTI != 1.0: + dconv.append(param) + else: + has_decay.append(param) + + if len(dconv) > 0: + return [{'params': has_decay}, + {'params': no_decay, 'weight_decay': 0.}, + {'params': dconv, 'lr': config.TRAIN.BASE_LR * config.MODEL.OFFSET_LR_MULTI}] + else: + return [{'params': has_decay}, + {'params': no_decay, 'weight_decay': 0.}] + + + +def check_keywords_in_name(name, keywords=()): + isin = False + for keyword in keywords: + if keyword in name: + isin = True + return isin diff --git a/classification/utils.py b/classification/utils.py new file mode 100644 index 0000000..8b7116a --- /dev/null +++ b/classification/utils.py @@ -0,0 +1,136 @@ +import os +import torch +import torch.distributed as dist + +try: + from apex import amp +except ImportError: + amp = None + + +def load_checkpoint(config, model, optimizer, lr_scheduler, logger): + logger.info(f"==============> Resuming from {config.MODEL.RESUME}....................") + if config.MODEL.RESUME.startswith('https'): + checkpoint = torch.hub.load_state_dict_from_url( + config.MODEL.RESUME, map_location='cpu', check_hash=True) + else: + checkpoint = torch.load(config.MODEL.RESUME, map_location='cpu') + if 'model' in checkpoint: + msg = model.load_state_dict(checkpoint['model'], strict=False) + else: + msg = model.load_state_dict(checkpoint, strict=False) + logger.info(msg) + max_accuracy = 0.0 + if not config.EVAL_MODE and 'optimizer' in checkpoint and 'lr_scheduler' in checkpoint and 'epoch' in checkpoint: + optimizer.load_state_dict(checkpoint['optimizer']) + lr_scheduler.load_state_dict(checkpoint['lr_scheduler']) + config.defrost() + config.TRAIN.START_EPOCH = checkpoint['epoch'] + 1 + config.freeze() + if 'amp' in checkpoint and config.AMP_OPT_LEVEL != "O0" and checkpoint['config'].AMP_OPT_LEVEL != "O0": + amp.load_state_dict(checkpoint['amp']) + logger.info(f"=> loaded successfully '{config.MODEL.RESUME}' (epoch {checkpoint['epoch']})") + if 'max_accuracy' in checkpoint: + max_accuracy = checkpoint['max_accuracy'] + + del checkpoint + torch.cuda.empty_cache() + return max_accuracy + + +def load_pretrained(config, model, optimizer, lr_scheduler, logger): + logger.info(f"==============> Load Pretrained Model from {config.MODEL.PRETRAINED}....................") + checkpoint = torch.load(config.MODEL.PRETRAINED, map_location='cpu') + if 'model' in checkpoint: + msg = model.load_state_dict(checkpoint['model'], strict=False) + else: + msg = model.load_state_dict(checkpoint, strict=False) + logger.info(msg) + max_accuracy = 0.0 + if not config.EVAL_MODE and 'optimizer' in checkpoint and 'lr_scheduler' in checkpoint and 'epoch' in checkpoint: + optimizer.load_state_dict(checkpoint['optimizer']) + lr_scheduler.load_state_dict(checkpoint['lr_scheduler']) + config.defrost() + config.TRAIN.START_EPOCH = checkpoint['epoch'] + 1 + config.freeze() + if 'amp' in checkpoint and config.AMP_OPT_LEVEL != "O0" and checkpoint['config'].AMP_OPT_LEVEL != "O0": + amp.load_state_dict(checkpoint['amp']) + logger.info(f"=> loaded successfully '{config.MODEL.RESUME}' (epoch {checkpoint['epoch']})") + if 'max_accuracy' in checkpoint: + max_accuracy = checkpoint['max_accuracy'] + + del checkpoint + torch.cuda.empty_cache() + return max_accuracy + + + +def save_checkpoint(config, epoch, model, max_accuracy, optimizer, lr_scheduler, logger): + save_state = {'model': model.state_dict(), + 'optimizer': optimizer.state_dict(), + 'lr_scheduler': lr_scheduler.state_dict(), + 'max_accuracy': max_accuracy, + 'epoch': epoch, + 'config': config} + if config.AMP_OPT_LEVEL != "O0": + save_state['amp'] = amp.state_dict() + + save_path = os.path.join(config.OUTPUT, f'ckpt_epoch_{epoch}.pth') + logger.info(f"{save_path} saving......") + torch.save(save_state, save_path) + logger.info(f"{save_path} saved !!!") + +def save_last_checkpoint(config, epoch, model, max_accuracy, optimizer, lr_scheduler, logger): + save_state = {'model': model.state_dict(), + 'optimizer': optimizer.state_dict(), + 'lr_scheduler': lr_scheduler.state_dict(), + 'max_accuracy': max_accuracy, + 'epoch': epoch, + 'config': config} + if config.AMP_OPT_LEVEL != "O0": + save_state['amp'] = amp.state_dict() + + save_path = os.path.join(config.OUTPUT, f'ckpt_epoch_latest.pth') + logger.info(f"{save_path} saving......") + torch.save(save_state, save_path) + logger.info(f"{save_path} saved !!!") + +def save_best_checkpoint(config, epoch, model, max_accuracy, optimizer, lr_scheduler, logger): + save_state = {'model': model.state_dict(), + 'optimizer': optimizer.state_dict(), + 'lr_scheduler': lr_scheduler.state_dict(), + 'max_accuracy': max_accuracy, + 'epoch': epoch, + 'config': config} + if config.AMP_OPT_LEVEL != "O0": + save_state['amp'] = amp.state_dict() + + save_path = os.path.join(config.OUTPUT, f'ckpt_epoch_best.pth') + logger.info(f"{save_path} saving......") + torch.save(save_state, save_path) + logger.info(f"{save_path} saved !!!") + +def get_grad_norm(parameters, norm_type=2): + if isinstance(parameters, torch.Tensor): + parameters = [parameters] + parameters = list(filter(lambda p: p.grad is not None, parameters)) + norm_type = float(norm_type) + total_norm = 0 + for p in parameters: + param_norm = p.grad.data.norm(norm_type) + total_norm += param_norm.item() ** norm_type + total_norm = total_norm ** (1. / norm_type) + return total_norm + +def auto_resume_helper(output_dir): + checkpoints = os.path.join(output_dir, f'ckpt_epoch_latest.pth') + if os.path.exists(checkpoints): + return checkpoints + else: + return None + +def reduce_tensor(tensor): + rt = tensor.clone() + dist.all_reduce(rt, op=dist.ReduceOp.SUM) + rt /= dist.get_world_size() + return rt diff --git a/detection/.github/CODE_OF_CONDUCT.md b/detection/.github/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..efd4305 --- /dev/null +++ b/detection/.github/CODE_OF_CONDUCT.md @@ -0,0 +1,76 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at chenkaidev@gmail.com. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq diff --git a/detection/.github/CONTRIBUTING.md b/detection/.github/CONTRIBUTING.md new file mode 100644 index 0000000..c669626 --- /dev/null +++ b/detection/.github/CONTRIBUTING.md @@ -0,0 +1 @@ +We appreciate all contributions to improve MMDetection. Please refer to [CONTRIBUTING.md](https://github.com/open-mmlab/mmcv/blob/master/CONTRIBUTING.md) in MMCV for more details about the contributing guideline. diff --git a/detection/.github/ISSUE_TEMPLATE/config.yml b/detection/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..56bbd88 --- /dev/null +++ b/detection/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,9 @@ +blank_issues_enabled: false + +contact_links: + - name: Common Issues + url: https://mmdetection.readthedocs.io/en/latest/faq.html + about: Check if your issue already has solutions + - name: MMDetection Documentation + url: https://mmdetection.readthedocs.io/en/latest/ + about: Check if your question is answered in docs diff --git a/detection/.github/ISSUE_TEMPLATE/error-report.md b/detection/.github/ISSUE_TEMPLATE/error-report.md new file mode 100644 index 0000000..23cb9c1 --- /dev/null +++ b/detection/.github/ISSUE_TEMPLATE/error-report.md @@ -0,0 +1,47 @@ +--- +name: Error report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +Thanks for your error report and we appreciate it a lot. + +**Checklist** + +1. I have searched related issues but cannot get the expected help. +2. I have read the [FAQ documentation](https://mmdetection.readthedocs.io/en/latest/faq.html) but cannot get the expected help. +3. The bug has not been fixed in the latest version. + +**Describe the bug** +A clear and concise description of what the bug is. + +**Reproduction** + +1. What command or script did you run? + +```none +A placeholder for the command. +``` + +2. Did you make any modifications on the code or config? Did you understand what you have modified? +3. What dataset did you use? + +**Environment** + +1. Please run `python mmdet/utils/collect_env.py` to collect necessary environment information and paste it here. +2. You may add addition that may be helpful for locating the problem, such as + - How you installed PyTorch [e.g., pip, conda, source] + - Other environment variables that may be related (such as `$PATH`, `$LD_LIBRARY_PATH`, `$PYTHONPATH`, etc.) + +**Error traceback** +If applicable, paste the error trackback here. + +```none +A placeholder for trackback. +``` + +**Bug fix** +If you have already identified the reason, you can provide the information here. If you are willing to create a PR to fix it, please also leave a comment here and that would be much appreciated! diff --git a/detection/.github/ISSUE_TEMPLATE/feature_request.md b/detection/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..33f9d5f --- /dev/null +++ b/detection/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,22 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Describe the feature** + +**Motivation** +A clear and concise description of the motivation of the feature. +Ex1. It is inconvenient when [....]. +Ex2. There is a recent paper [....], which is very helpful for [....]. + +**Related resources** +If there is an official code release or third-party implementations, please also provide the information here, which would be very helpful. + +**Additional context** +Add any other context or screenshots about the feature request here. +If you would like to implement the feature and create a PR, please leave a comment here and that would be much appreciated. diff --git a/detection/.github/ISSUE_TEMPLATE/general_questions.md b/detection/.github/ISSUE_TEMPLATE/general_questions.md new file mode 100644 index 0000000..b5a6451 --- /dev/null +++ b/detection/.github/ISSUE_TEMPLATE/general_questions.md @@ -0,0 +1,8 @@ +--- +name: General questions +about: Ask general questions to get help +title: '' +labels: '' +assignees: '' + +--- diff --git a/detection/.github/ISSUE_TEMPLATE/reimplementation_questions.md b/detection/.github/ISSUE_TEMPLATE/reimplementation_questions.md new file mode 100644 index 0000000..6b35838 --- /dev/null +++ b/detection/.github/ISSUE_TEMPLATE/reimplementation_questions.md @@ -0,0 +1,68 @@ +--- +name: Reimplementation Questions +about: Ask about questions during model reimplementation +title: '' +labels: 'reimplementation' +assignees: '' + +--- + +**Notice** + +There are several common situations in the reimplementation issues as below + +1. Reimplement a model in the model zoo using the provided configs +2. Reimplement a model in the model zoo on other dataset (e.g., custom datasets) +3. Reimplement a custom model but all the components are implemented in MMDetection +4. Reimplement a custom model with new modules implemented by yourself + +There are several things to do for different cases as below. + +- For case 1 & 3, please follow the steps in the following sections thus we could help to quick identify the issue. +- For case 2 & 4, please understand that we are not able to do much help here because we usually do not know the full code and the users should be responsible to the code they write. +- One suggestion for case 2 & 4 is that the users should first check whether the bug lies in the self-implemented code or the original code. For example, users can first make sure that the same model runs well on supported datasets. If you still need help, please describe what you have done and what you obtain in the issue, and follow the steps in the following sections and try as clear as possible so that we can better help you. + +**Checklist** + +1. I have searched related issues but cannot get the expected help. +2. The issue has not been fixed in the latest version. + +**Describe the issue** + +A clear and concise description of what the problem you meet and what have you done. + +**Reproduction** + +1. What command or script did you run? + +```none +A placeholder for the command. +``` + +2. What config dir you run? + +```none +A placeholder for the config. +``` + +3. Did you make any modifications on the code or config? Did you understand what you have modified? +4. What dataset did you use? + +**Environment** + +1. Please run `python mmdet/utils/collect_env.py` to collect necessary environment information and paste it here. +2. You may add addition that may be helpful for locating the problem, such as + 1. How you installed PyTorch [e.g., pip, conda, source] + 2. Other environment variables that may be related (such as `$PATH`, `$LD_LIBRARY_PATH`, `$PYTHONPATH`, etc.) + +**Results** + +If applicable, paste the related results here, e.g., what you expect and what you get. + +```none +A placeholder for results comparison +``` + +**Issue fix** + +If you have already identified the reason, you can provide the information here. If you are willing to create a PR to fix it, please also leave a comment here and that would be much appreciated! diff --git a/detection/.gitignore b/detection/.gitignore new file mode 100644 index 0000000..b492b88 --- /dev/null +++ b/detection/.gitignore @@ -0,0 +1,121 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +data/ +data +.vscode +.idea +.DS_Store + +# custom +*.pkl +*.pkl.json +*.log.json +work_dirs/ +pretrained/ +# Pytorch +*.pth +*.py~ +*.sh~ diff --git a/detection/LICENSE b/detection/LICENSE new file mode 100644 index 0000000..1217763 --- /dev/null +++ b/detection/LICENSE @@ -0,0 +1,220 @@ +Copyright 2018-2019 Open-MMLab. All rights reserved. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2018-2019 Open-MMLab. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + ====================================================================================== + Swin-Transformer-Object-Detection Subcomponents: + + The Swin-Transformer-Object-Detection project contains subcomponents with separate + copyright notices and license terms. Your use of the source code for the these + subcomponents is subject to the terms and conditions of the following licenses. + + ======================================================================================= + MIT license + ======================================================================================= + + The following components are provided under an MIT license. + + 1. swin_transformer.py - For details, mmdet/models/backbones/swin_transformer.py + Copyright (c) 2021 Microsoft diff --git a/detection/README.md b/detection/README.md new file mode 100644 index 0000000..bb01a4d --- /dev/null +++ b/detection/README.md @@ -0,0 +1,81 @@ +# Object Detection Code for LITv2 + + +## Usage + +1. Install [mmdetection](https://github.com/open-mmlab/mmdetection/blob/master/docs/get_started.md) + +2. Download COCO 2017 from the [official website](https://cocodataset.org/#download) and prepare the dataset. The directory structure should look like + + ``` + coco + ├── annotations + ├── train2017 + └── val2017 + ``` + + Next, create a symbolic link to this repo by + + ```bash + cd detection/ + mkdir data + ln -s [path/to/coco] data/ + ``` + +3. Download LITv2 pretrained weights on ImageNet. + + + +## Training + +```bash +bash tools/dist_train.sh --cfg-options model.pretrained= [model.backbone.use_checkpoint=True] [other optional arguments] +``` + +For example, you can train LITv2-S with 8 GPUs by + +``` +bash tools/dist_train.sh configs/litv2/retinanet_litv2_s_fpn_1x_coco.py 8 --cfg-options model.pretrained=litv2_s.pth +``` + +## Inference + +```bash +# multi-gpu testing +tools/dist_test.sh --eval bbox segm + +# single-gpu testing +python tools/test.py --eval bbox segm +``` + +For example, you can test LITv2-S with RetinaNet on 8 GPUs by + +```bash +bash tools/dist_test.sh configs/litv2/retinanet_litv2_s_fpn_1x_coco.py retinanet_litv2_s_fpn_1x_coco.pth 8 --eval bbox +``` + + +## Benchmark + +To get the FLOPs, run + +```bash +python tools/get_flops.py configs/litv2/retinanet_litv2_s_fpn_1x_coco.py +``` + +This should give + +```bash +Input shape: (3, 1280, 800) +Flops: 242.39 GFLOPs +Params: 38.01 M +``` + +To test the FPS, run + +```bash +python -m torch.distributed.launch --nproc_per_node=1 --master_port=29500 tools/benchmark.py \ + configs/litv2/retinanet_litv2_s_fpn_1x_coco.py \ + --checkpoint retinanet_litv2_s_fpn_1x_coco.pth \ + --launcher pytorch +``` diff --git a/detection/configs/_base_/datasets/coco_detection.py b/detection/configs/_base_/datasets/coco_detection.py new file mode 100644 index 0000000..09a75c4 --- /dev/null +++ b/detection/configs/_base_/datasets/coco_detection.py @@ -0,0 +1,48 @@ +dataset_type = 'CocoDataset' +data_root = 'data/coco/' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations', with_bbox=True), + dict(type='Resize', img_scale=(1333, 800), keep_ratio=True), + dict(type='RandomFlip', flip_ratio=0.5), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size_divisor=32), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_bboxes', 'gt_labels']), +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(1333, 800), + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size_divisor=32), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +data = dict( + samples_per_gpu=2, + workers_per_gpu=2, + train=dict( + type=dataset_type, + ann_file=data_root + 'annotations/instances_train2017.json', + img_prefix=data_root + 'train2017/', + pipeline=train_pipeline), + val=dict( + type=dataset_type, + ann_file=data_root + 'annotations/instances_val2017.json', + img_prefix=data_root + 'val2017/', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + ann_file=data_root + 'annotations/instances_val2017.json', + img_prefix=data_root + 'val2017/', + pipeline=test_pipeline)) +evaluation = dict(interval=1, metric='bbox') diff --git a/detection/configs/_base_/datasets/coco_instance.py b/detection/configs/_base_/datasets/coco_instance.py new file mode 100644 index 0000000..f6ea4f4 --- /dev/null +++ b/detection/configs/_base_/datasets/coco_instance.py @@ -0,0 +1,48 @@ +dataset_type = 'CocoDataset' +data_root = 'data/coco/' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations', with_bbox=True, with_mask=True), + dict(type='Resize', img_scale=(1333, 800), keep_ratio=True), + dict(type='RandomFlip', flip_ratio=0.5), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size_divisor=32), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_bboxes', 'gt_labels', 'gt_masks']), +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(1333, 800), + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size_divisor=32), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +data = dict( + samples_per_gpu=2, + workers_per_gpu=2, + train=dict( + type=dataset_type, + ann_file=data_root + 'annotations/instances_train2017.json', + img_prefix=data_root + 'train2017/', + pipeline=train_pipeline), + val=dict( + type=dataset_type, + ann_file=data_root + 'annotations/instances_val2017.json', + img_prefix=data_root + 'val2017/', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + ann_file=data_root + 'annotations/instances_val2017.json', + img_prefix=data_root + 'val2017/', + pipeline=test_pipeline)) +evaluation = dict(metric=['bbox', 'segm']) diff --git a/detection/configs/_base_/datasets/coco_instance_semantic.py b/detection/configs/_base_/datasets/coco_instance_semantic.py new file mode 100644 index 0000000..f7c072e --- /dev/null +++ b/detection/configs/_base_/datasets/coco_instance_semantic.py @@ -0,0 +1,53 @@ +dataset_type = 'CocoDataset' +data_root = 'data/coco/' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='LoadAnnotations', with_bbox=True, with_mask=True, with_seg=True), + dict(type='Resize', img_scale=(1333, 800), keep_ratio=True), + dict(type='RandomFlip', flip_ratio=0.5), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size_divisor=32), + dict(type='SegRescale', scale_factor=1 / 8), + dict(type='DefaultFormatBundle'), + dict( + type='Collect', + keys=['img', 'gt_bboxes', 'gt_labels', 'gt_masks', 'gt_semantic_seg']), +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(1333, 800), + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip', flip_ratio=0.5), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size_divisor=32), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +data = dict( + samples_per_gpu=2, + workers_per_gpu=2, + train=dict( + type=dataset_type, + ann_file=data_root + 'annotations/instances_train2017.json', + img_prefix=data_root + 'train2017/', + seg_prefix=data_root + 'stuffthingmaps/train2017/', + pipeline=train_pipeline), + val=dict( + type=dataset_type, + ann_file=data_root + 'annotations/instances_val2017.json', + img_prefix=data_root + 'val2017/', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + ann_file=data_root + 'annotations/instances_val2017.json', + img_prefix=data_root + 'val2017/', + pipeline=test_pipeline)) +evaluation = dict(metric=['bbox', 'segm']) diff --git a/detection/configs/_base_/default_runtime.py b/detection/configs/_base_/default_runtime.py new file mode 100644 index 0000000..55097c5 --- /dev/null +++ b/detection/configs/_base_/default_runtime.py @@ -0,0 +1,16 @@ +checkpoint_config = dict(interval=1) +# yapf:disable +log_config = dict( + interval=50, + hooks=[ + dict(type='TextLoggerHook'), + # dict(type='TensorboardLoggerHook') + ]) +# yapf:enable +custom_hooks = [dict(type='NumClassCheckHook')] + +dist_params = dict(backend='nccl') +log_level = 'INFO' +load_from = None +resume_from = None +workflow = [('train', 1)] diff --git a/detection/configs/_base_/models/mask_rcnn_fpn.py b/detection/configs/_base_/models/mask_rcnn_fpn.py new file mode 100644 index 0000000..aeabe6c --- /dev/null +++ b/detection/configs/_base_/models/mask_rcnn_fpn.py @@ -0,0 +1,127 @@ +# model settings +model = dict( + type='MaskRCNN', + pretrained=None, + backbone=dict( + type='LITv2', + embed_dim=96, + depths=[2, 2, 6, 2], + num_heads=[3, 6, 12, 24], + window_size=7, + mlp_ratio=4., + qkv_bias=True, + qk_scale=None, + drop_rate=0., + attn_drop_rate=0., + drop_path_rate=0.2, + ape=False, + patch_norm=True, + out_indices=(0, 1, 2, 3), + use_checkpoint=False), + neck=dict( + type='FPN', + in_channels=[96, 192, 384, 768], + out_channels=256, + num_outs=5), + rpn_head=dict( + type='RPNHead', + in_channels=256, + feat_channels=256, + anchor_generator=dict( + type='AnchorGenerator', + scales=[8], + ratios=[0.5, 1.0, 2.0], + strides=[4, 8, 16, 32, 64]), + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[.0, .0, .0, .0], + target_stds=[1.0, 1.0, 1.0, 1.0]), + loss_cls=dict( + type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), + loss_bbox=dict(type='L1Loss', loss_weight=1.0)), + roi_head=dict( + type='StandardRoIHead', + bbox_roi_extractor=dict( + type='SingleRoIExtractor', + roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0), + out_channels=256, + featmap_strides=[4, 8, 16, 32]), + bbox_head=dict( + type='Shared2FCBBoxHead', + in_channels=256, + fc_out_channels=1024, + roi_feat_size=7, + num_classes=80, + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[0., 0., 0., 0.], + target_stds=[0.1, 0.1, 0.2, 0.2]), + reg_class_agnostic=False, + loss_cls=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), + loss_bbox=dict(type='L1Loss', loss_weight=1.0)), + mask_roi_extractor=dict( + type='SingleRoIExtractor', + roi_layer=dict(type='RoIAlign', output_size=14, sampling_ratio=0), + out_channels=256, + featmap_strides=[4, 8, 16, 32]), + mask_head=dict( + type='FCNMaskHead', + num_convs=4, + in_channels=256, + conv_out_channels=256, + num_classes=80, + loss_mask=dict( + type='CrossEntropyLoss', use_mask=True, loss_weight=1.0))), + # model training and testing settings + train_cfg=dict( + rpn=dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.7, + neg_iou_thr=0.3, + min_pos_iou=0.3, + match_low_quality=True, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=256, + pos_fraction=0.5, + neg_pos_ub=-1, + add_gt_as_proposals=False), + allowed_border=-1, + pos_weight=-1, + debug=False), + rpn_proposal=dict( + nms_pre=2000, + max_per_img=1000, + nms=dict(type='nms', iou_threshold=0.7), + min_bbox_size=0), + rcnn=dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.5, + neg_iou_thr=0.5, + min_pos_iou=0.5, + match_low_quality=True, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=512, + pos_fraction=0.25, + neg_pos_ub=-1, + add_gt_as_proposals=True), + mask_size=28, + pos_weight=-1, + debug=False)), + test_cfg=dict( + rpn=dict( + nms_pre=1000, + max_per_img=1000, + nms=dict(type='nms', iou_threshold=0.7), + min_bbox_size=0), + rcnn=dict( + score_thr=0.05, + nms=dict(type='nms', iou_threshold=0.5), + max_per_img=100, + mask_thr_binary=0.5))) diff --git a/detection/configs/_base_/models/retinanet_fpn.py b/detection/configs/_base_/models/retinanet_fpn.py new file mode 100644 index 0000000..9dca124 --- /dev/null +++ b/detection/configs/_base_/models/retinanet_fpn.py @@ -0,0 +1,67 @@ +# model settings +model = dict( + type='RetinaNet', + pretrained=None, + backbone=dict( + type='LITv2', + embed_dim=96, + depths=[2, 2, 6, 2], + num_heads=[3, 6, 12, 24], + window_size=7, + mlp_ratio=4., + qkv_bias=True, + qk_scale=None, + drop_rate=0., + attn_drop_rate=0., + drop_path_rate=0.2, + ape=False, + patch_norm=True, + out_indices=(0, 1, 2, 3), + use_checkpoint=False), + neck=dict( + type='FPN', + in_channels=[256, 512, 1024, 2048], + out_channels=256, + start_level=1, + add_extra_convs='on_input', + num_outs=5), + bbox_head=dict( + type='RetinaHead', + num_classes=80, + in_channels=256, + stacked_convs=4, + feat_channels=256, + anchor_generator=dict( + type='AnchorGenerator', + octave_base_scale=4, + scales_per_octave=3, + ratios=[0.5, 1.0, 2.0], + strides=[8, 16, 32, 64, 128]), + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[.0, .0, .0, .0], + target_stds=[1.0, 1.0, 1.0, 1.0]), + loss_cls=dict( + type='FocalLoss', + use_sigmoid=True, + gamma=2.0, + alpha=0.25, + loss_weight=1.0), + loss_bbox=dict(type='L1Loss', loss_weight=1.0)), + # training and testing settings + train_cfg=dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.5, + neg_iou_thr=0.4, + min_pos_iou=0, + ignore_iof_thr=-1), + allowed_border=-1, + pos_weight=-1, + debug=False), + test_cfg=dict( + nms_pre=1000, + min_bbox_size=0, + score_thr=0.05, + nms=dict(type='nms', iou_threshold=0.5), + max_per_img=100)) diff --git a/detection/configs/_base_/schedules/schedule_1x.py b/detection/configs/_base_/schedules/schedule_1x.py new file mode 100644 index 0000000..f0ae6ee --- /dev/null +++ b/detection/configs/_base_/schedules/schedule_1x.py @@ -0,0 +1,11 @@ +# optimizer +optimizer = dict(type='SGD', lr=0.02, weight_decay=0.0001) +optimizer_config = dict(grad_clip=None) +# learning policy +lr_config = dict( + policy='step', + warmup='linear', + warmup_iters=500, + warmup_ratio=0.001, + step=[8, 11]) +runner = dict(type='EpochBasedRunner', max_epochs=12) diff --git a/detection/configs/_base_/schedules/schedule_20e.py b/detection/configs/_base_/schedules/schedule_20e.py new file mode 100644 index 0000000..00e8590 --- /dev/null +++ b/detection/configs/_base_/schedules/schedule_20e.py @@ -0,0 +1,11 @@ +# optimizer +optimizer = dict(type='SGD', lr=0.02, momentum=0.9, weight_decay=0.0001) +optimizer_config = dict(grad_clip=None) +# learning policy +lr_config = dict( + policy='step', + warmup='linear', + warmup_iters=500, + warmup_ratio=0.001, + step=[16, 19]) +runner = dict(type='EpochBasedRunner', max_epochs=20) diff --git a/detection/configs/_base_/schedules/schedule_2x.py b/detection/configs/_base_/schedules/schedule_2x.py new file mode 100644 index 0000000..69dc9ee --- /dev/null +++ b/detection/configs/_base_/schedules/schedule_2x.py @@ -0,0 +1,11 @@ +# optimizer +optimizer = dict(type='SGD', lr=0.02, momentum=0.9, weight_decay=0.0001) +optimizer_config = dict(grad_clip=None) +# learning policy +lr_config = dict( + policy='step', + warmup='linear', + warmup_iters=500, + warmup_ratio=0.001, + step=[16, 22]) +runner = dict(type='EpochBasedRunner', max_epochs=24) diff --git a/detection/configs/litv2/mask_rcnn_litv2_b_fpn_1x_coco.py b/detection/configs/litv2/mask_rcnn_litv2_b_fpn_1x_coco.py new file mode 100644 index 0000000..cc3e8f1 --- /dev/null +++ b/detection/configs/litv2/mask_rcnn_litv2_b_fpn_1x_coco.py @@ -0,0 +1,34 @@ +_base_ = [ + '../_base_/models/mask_rcnn_fpn.py', + '../_base_/datasets/coco_instance.py', + '../_base_/schedules/schedule_1x.py', '../_base_/default_runtime.py' +] + +model = dict( + backbone=dict( + embed_dim=128, + depths=[2, 2, 18, 2], + num_heads=[ 4, 8, 16, 32 ], + drop_path_rate=0.3, + patch_norm=True, + use_checkpoint=True, + alpha=0.9, + local_ws=[0, 0, 2, 1] + ), + neck=dict(in_channels=[128, 256, 512, 1024])) + +optimizer = dict(type='AdamW', lr=0.0002, weight_decay=0.0001, + paramwise_cfg=dict(custom_keys={'norm': dict(decay_mult=0.)}) + ) + +lr_config = dict(step=[8, 11]) +runner = dict(type='EpochBasedRunnerAmp', max_epochs=12) + +optimizer_config = dict( + type="DistOptimizerHook", + update_interval=1, + grad_clip=None, + coalesce=True, + bucket_size_mb=-1, + use_fp16=True, +) diff --git a/detection/configs/litv2/mask_rcnn_litv2_b_fpn_1x_coco_ws_4.py b/detection/configs/litv2/mask_rcnn_litv2_b_fpn_1x_coco_ws_4.py new file mode 100644 index 0000000..a2a957c --- /dev/null +++ b/detection/configs/litv2/mask_rcnn_litv2_b_fpn_1x_coco_ws_4.py @@ -0,0 +1,34 @@ +_base_ = [ + '../_base_/models/mask_rcnn_fpn.py', + '../_base_/datasets/coco_instance.py', + '../_base_/schedules/schedule_1x.py', '../_base_/default_runtime.py' +] + +model = dict( + backbone=dict( + embed_dim=128, + depths=[2, 2, 18, 2], + num_heads=[ 4, 8, 16, 32 ], + drop_path_rate=0.3, + patch_norm=True, + use_checkpoint=True, + alpha=0.9, + local_ws=[0, 0, 4, 1] + ), + neck=dict(in_channels=[128, 256, 512, 1024])) + +optimizer = dict(type='AdamW', lr=0.0002, weight_decay=0.0001, + paramwise_cfg=dict(custom_keys={'norm': dict(decay_mult=0.)}) + ) + +lr_config = dict(step=[8, 11]) +runner = dict(type='EpochBasedRunnerAmp', max_epochs=12) + +optimizer_config = dict( + type="DistOptimizerHook", + update_interval=1, + grad_clip=None, + coalesce=True, + bucket_size_mb=-1, + use_fp16=True, +) diff --git a/detection/configs/litv2/mask_rcnn_litv2_m_fpn_1x_coco.py b/detection/configs/litv2/mask_rcnn_litv2_m_fpn_1x_coco.py new file mode 100644 index 0000000..e7575aa --- /dev/null +++ b/detection/configs/litv2/mask_rcnn_litv2_m_fpn_1x_coco.py @@ -0,0 +1,34 @@ +_base_ = [ + '../_base_/models/mask_rcnn_fpn.py', + '../_base_/datasets/coco_instance.py', + '../_base_/schedules/schedule_1x.py', '../_base_/default_runtime.py' +] + +model = dict( + backbone=dict( + embed_dim=96, + depths=[2, 2, 18, 2], + num_heads=[3, 6, 12, 24], + drop_path_rate=0.2, + patch_norm=True, + use_checkpoint=False, + alpha=0.9, + local_ws=[0, 0, 2, 1] + ), + neck=dict(in_channels=[96, 192, 384, 768])) + +optimizer = dict(type='AdamW', lr=0.0002, weight_decay=0.0001, + paramwise_cfg=dict(custom_keys={'norm': dict(decay_mult=0.)}) + ) + +lr_config = dict(step=[8, 11]) +runner = dict(type='EpochBasedRunnerAmp', max_epochs=12) + +optimizer_config = dict( + type="DistOptimizerHook", + update_interval=1, + grad_clip=None, + coalesce=True, + bucket_size_mb=-1, + use_fp16=True, +) diff --git a/detection/configs/litv2/mask_rcnn_litv2_m_fpn_1x_coco_ws_4.py b/detection/configs/litv2/mask_rcnn_litv2_m_fpn_1x_coco_ws_4.py new file mode 100644 index 0000000..d804b98 --- /dev/null +++ b/detection/configs/litv2/mask_rcnn_litv2_m_fpn_1x_coco_ws_4.py @@ -0,0 +1,34 @@ +_base_ = [ + '../_base_/models/mask_rcnn_fpn.py', + '../_base_/datasets/coco_instance.py', + '../_base_/schedules/schedule_1x.py', '../_base_/default_runtime.py' +] + +model = dict( + backbone=dict( + embed_dim=96, + depths=[2, 2, 18, 2], + num_heads=[3, 6, 12, 24], + drop_path_rate=0.2, + patch_norm=True, + use_checkpoint=False, + alpha=0.9, + local_ws=[0, 0, 4, 1] + ), + neck=dict(in_channels=[96, 192, 384, 768])) + +optimizer = dict(type='AdamW', lr=0.0002, weight_decay=0.0001, + paramwise_cfg=dict(custom_keys={'norm': dict(decay_mult=0.)}) + ) + +lr_config = dict(step=[8, 11]) +runner = dict(type='EpochBasedRunnerAmp', max_epochs=12) + +optimizer_config = dict( + type="DistOptimizerHook", + update_interval=1, + grad_clip=None, + coalesce=True, + bucket_size_mb=-1, + use_fp16=True, +) diff --git a/detection/configs/litv2/mask_rcnn_litv2_s_fpn_1x_coco.py b/detection/configs/litv2/mask_rcnn_litv2_s_fpn_1x_coco.py new file mode 100644 index 0000000..1f10ddf --- /dev/null +++ b/detection/configs/litv2/mask_rcnn_litv2_s_fpn_1x_coco.py @@ -0,0 +1,34 @@ +_base_ = [ + '../_base_/models/mask_rcnn_fpn.py', + '../_base_/datasets/coco_instance.py', + '../_base_/schedules/schedule_1x.py', '../_base_/default_runtime.py' +] + +model = dict( + backbone=dict( + embed_dim=96, + depths=[2, 2, 6, 2], + num_heads=[3, 6, 12, 24], + drop_path_rate=0.2, + patch_norm=True, + use_checkpoint=False, + alpha=0.9, + local_ws=[0, 0, 2, 1] + ), + neck=dict(in_channels=[96, 192, 384, 768])) + +optimizer = dict(type='AdamW', lr=0.0002, weight_decay=0.0001, + paramwise_cfg=dict(custom_keys={'norm': dict(decay_mult=0.)}) + ) + +lr_config = dict(step=[8, 11]) +runner = dict(type='EpochBasedRunnerAmp', max_epochs=12) + +optimizer_config = dict( + type="DistOptimizerHook", + update_interval=1, + grad_clip=None, + coalesce=True, + bucket_size_mb=-1, + use_fp16=True, +) diff --git a/detection/configs/litv2/mask_rcnn_litv2_s_fpn_1x_coco_ws_4.py b/detection/configs/litv2/mask_rcnn_litv2_s_fpn_1x_coco_ws_4.py new file mode 100644 index 0000000..8aa6e25 --- /dev/null +++ b/detection/configs/litv2/mask_rcnn_litv2_s_fpn_1x_coco_ws_4.py @@ -0,0 +1,34 @@ +_base_ = [ + '../_base_/models/mask_rcnn_fpn.py', + '../_base_/datasets/coco_instance.py', + '../_base_/schedules/schedule_1x.py', '../_base_/default_runtime.py' +] + +model = dict( + backbone=dict( + embed_dim=96, + depths=[2, 2, 6, 2], + num_heads=[3, 6, 12, 24], + drop_path_rate=0.2, + patch_norm=True, + use_checkpoint=False, + alpha=0.9, + local_ws=[0, 0, 4, 1] + ), + neck=dict(in_channels=[96, 192, 384, 768])) + +optimizer = dict(type='AdamW', lr=0.0002, weight_decay=0.0001, + paramwise_cfg=dict(custom_keys={'norm': dict(decay_mult=0.)}) + ) + +lr_config = dict(step=[8, 11]) +runner = dict(type='EpochBasedRunnerAmp', max_epochs=12) + +optimizer_config = dict( + type="DistOptimizerHook", + update_interval=1, + grad_clip=None, + coalesce=True, + bucket_size_mb=-1, + use_fp16=True, +) diff --git a/detection/configs/litv2/retinanet_litv2_b_fpn_1x_coco.py b/detection/configs/litv2/retinanet_litv2_b_fpn_1x_coco.py new file mode 100644 index 0000000..0a0fe86 --- /dev/null +++ b/detection/configs/litv2/retinanet_litv2_b_fpn_1x_coco.py @@ -0,0 +1,42 @@ +_base_ = [ + '../_base_/models/retinanet_fpn.py', + '../_base_/datasets/coco_detection.py', + '../_base_/schedules/schedule_1x.py', '../_base_/default_runtime.py' +] +model = dict( + backbone=dict( + embed_dim=128, + depths=[2, 2, 18, 2], + num_heads=[ 4, 8, 16, 32 ], + drop_path_rate=0.3, + patch_norm=True, + use_checkpoint=True, + alpha=0.9, + local_ws=[0, 0, 2, 1] + ), + neck=dict( + type='FPN', + in_channels=[128, 256, 512, 1024], + out_channels=256, + start_level=1, + add_extra_convs='on_input', + num_outs=5) +) + +# optimizer +optimizer = dict(type='AdamW', lr=0.0001, weight_decay=0.0001, + paramwise_cfg=dict({'norm': dict(decay_mult=0.)})) + + +lr_config = dict(step=[8, 11]) +runner = dict(type='EpochBasedRunnerAmp', max_epochs=12) + +optimizer_config = dict( + type="DistOptimizerHook", + update_interval=1, + grad_clip=None, + coalesce=True, + bucket_size_mb=-1, + use_fp16=True, +) + diff --git a/detection/configs/litv2/retinanet_litv2_b_fpn_1x_coco_ws_4.py b/detection/configs/litv2/retinanet_litv2_b_fpn_1x_coco_ws_4.py new file mode 100644 index 0000000..52c71c6 --- /dev/null +++ b/detection/configs/litv2/retinanet_litv2_b_fpn_1x_coco_ws_4.py @@ -0,0 +1,42 @@ +_base_ = [ + '../_base_/models/retinanet_fpn.py', + '../_base_/datasets/coco_detection.py', + '../_base_/schedules/schedule_1x.py', '../_base_/default_runtime.py' +] +model = dict( + backbone=dict( + embed_dim=128, + depths=[2, 2, 18, 2], + num_heads=[ 4, 8, 16, 32 ], + drop_path_rate=0.3, + patch_norm=True, + use_checkpoint=True, + alpha=0.9, + local_ws=[0, 0, 4, 1] + ), + neck=dict( + type='FPN', + in_channels=[128, 256, 512, 1024], + out_channels=256, + start_level=1, + add_extra_convs='on_input', + num_outs=5) +) + +# optimizer +optimizer = dict(type='AdamW', lr=0.0001, weight_decay=0.0001, + paramwise_cfg=dict({'norm': dict(decay_mult=0.)})) + + +lr_config = dict(step=[8, 11]) +runner = dict(type='EpochBasedRunnerAmp', max_epochs=12) + +optimizer_config = dict( + type="DistOptimizerHook", + update_interval=1, + grad_clip=None, + coalesce=True, + bucket_size_mb=-1, + use_fp16=True, +) + diff --git a/detection/configs/litv2/retinanet_litv2_m_fpn_1x_coco.py b/detection/configs/litv2/retinanet_litv2_m_fpn_1x_coco.py new file mode 100644 index 0000000..92dc2b2 --- /dev/null +++ b/detection/configs/litv2/retinanet_litv2_m_fpn_1x_coco.py @@ -0,0 +1,41 @@ +_base_ = [ + '../_base_/models/retinanet_fpn.py', + '../_base_/datasets/coco_detection.py', + '../_base_/schedules/schedule_1x.py', '../_base_/default_runtime.py' +] +model = dict( + backbone=dict( + embed_dim=96, + depths=[2, 2, 18, 2], + num_heads=[3, 6, 12, 24], + drop_path_rate=0.2, + patch_norm=True, + use_checkpoint=False, + alpha=0.9, + local_ws=[0, 0, 2, 1] + ), + neck=dict( + type='FPN', + in_channels=[96, 192, 384, 768], + out_channels=256, + start_level=1, + add_extra_convs='on_input', + num_outs=5) +) + +# optimizer +optimizer = dict(type='AdamW', lr=0.0001, weight_decay=0.0001, + paramwise_cfg=dict({'norm': dict(decay_mult=0.)})) + + +lr_config = dict(step=[8, 11]) +runner = dict(type='EpochBasedRunnerAmp', max_epochs=12) + +optimizer_config = dict( + type="DistOptimizerHook", + update_interval=1, + grad_clip=None, + coalesce=True, + bucket_size_mb=-1, + use_fp16=True, +) \ No newline at end of file diff --git a/detection/configs/litv2/retinanet_litv2_m_fpn_1x_coco_ws_4.py b/detection/configs/litv2/retinanet_litv2_m_fpn_1x_coco_ws_4.py new file mode 100644 index 0000000..9c2726c --- /dev/null +++ b/detection/configs/litv2/retinanet_litv2_m_fpn_1x_coco_ws_4.py @@ -0,0 +1,41 @@ +_base_ = [ + '../_base_/models/retinanet_fpn.py', + '../_base_/datasets/coco_detection.py', + '../_base_/schedules/schedule_1x.py', '../_base_/default_runtime.py' +] +model = dict( + backbone=dict( + embed_dim=96, + depths=[2, 2, 18, 2], + num_heads=[3, 6, 12, 24], + drop_path_rate=0.2, + patch_norm=True, + use_checkpoint=False, + alpha=0.9, + local_ws=[0, 0, 4, 1] + ), + neck=dict( + type='FPN', + in_channels=[96, 192, 384, 768], + out_channels=256, + start_level=1, + add_extra_convs='on_input', + num_outs=5) +) + +# optimizer +optimizer = dict(type='AdamW', lr=0.0001, weight_decay=0.0001, + paramwise_cfg=dict({'norm': dict(decay_mult=0.)})) + + +lr_config = dict(step=[8, 11]) +runner = dict(type='EpochBasedRunnerAmp', max_epochs=12) + +optimizer_config = dict( + type="DistOptimizerHook", + update_interval=1, + grad_clip=None, + coalesce=True, + bucket_size_mb=-1, + use_fp16=True, +) \ No newline at end of file diff --git a/detection/configs/litv2/retinanet_litv2_s_fpn_1x_coco.py b/detection/configs/litv2/retinanet_litv2_s_fpn_1x_coco.py new file mode 100644 index 0000000..0cc67cd --- /dev/null +++ b/detection/configs/litv2/retinanet_litv2_s_fpn_1x_coco.py @@ -0,0 +1,43 @@ +_base_ = [ + '../_base_/models/retinanet_fpn.py', + '../_base_/datasets/coco_detection.py', + '../_base_/schedules/schedule_1x.py', '../_base_/default_runtime.py' +] +model = dict( + backbone=dict( + embed_dim=96, + depths=[2, 2, 6, 2], + num_heads=[3, 6, 12, 24], + drop_path_rate=0.2, + patch_norm=True, + use_checkpoint=False, + alpha=0.9, + local_ws=[0, 0, 2, 1] + ), + neck=dict( + type='FPN', + in_channels=[96, 192, 384, 768], + out_channels=256, + start_level=1, + add_extra_convs='on_input', + num_outs=5) +) + +# optimizer +optimizer = dict(type='AdamW', lr=0.0001, weight_decay=0.0001, + paramwise_cfg=dict({'norm': dict(decay_mult=0.)})) + + +lr_config = dict(step=[8, 11]) +runner = dict(type='EpochBasedRunnerAmp', max_epochs=12) + +optimizer_config = dict( + type="DistOptimizerHook", + update_interval=1, + grad_clip=None, + coalesce=True, + bucket_size_mb=-1, + use_fp16=True, +) + +# resume_from = 'path/to/latest.pth' \ No newline at end of file diff --git a/detection/configs/litv2/retinanet_litv2_s_fpn_1x_coco_ws_4.py b/detection/configs/litv2/retinanet_litv2_s_fpn_1x_coco_ws_4.py new file mode 100644 index 0000000..e288b38 --- /dev/null +++ b/detection/configs/litv2/retinanet_litv2_s_fpn_1x_coco_ws_4.py @@ -0,0 +1,43 @@ +_base_ = [ + '../_base_/models/retinanet_fpn.py', + '../_base_/datasets/coco_detection.py', + '../_base_/schedules/schedule_1x.py', '../_base_/default_runtime.py' +] +model = dict( + backbone=dict( + embed_dim=96, + depths=[2, 2, 6, 2], + num_heads=[3, 6, 12, 24], + drop_path_rate=0.2, + patch_norm=True, + use_checkpoint=False, + alpha=0.9, + local_ws=[0, 0, 4, 1] + ), + neck=dict( + type='FPN', + in_channels=[96, 192, 384, 768], + out_channels=256, + start_level=1, + add_extra_convs='on_input', + num_outs=5) +) + +# optimizer +optimizer = dict(type='AdamW', lr=0.0001, weight_decay=0.0001, + paramwise_cfg=dict({'norm': dict(decay_mult=0.)})) + + +lr_config = dict(step=[8, 11]) +runner = dict(type='EpochBasedRunnerAmp', max_epochs=12) + +optimizer_config = dict( + type="DistOptimizerHook", + update_interval=1, + grad_clip=None, + coalesce=True, + bucket_size_mb=-1, + use_fp16=True, +) + +# resume_from = 'path/to/latest.pth' \ No newline at end of file diff --git a/detection/mm_modules/DCN/deform_conv2d_naive.py b/detection/mm_modules/DCN/deform_conv2d_naive.py new file mode 100644 index 0000000..100ce74 --- /dev/null +++ b/detection/mm_modules/DCN/deform_conv2d_naive.py @@ -0,0 +1,93 @@ +import torch +import torch.nn as nn +from torch.nn import init +import math +import numpy as np +from torch.nn.modules.module import Module +import torch.nn.functional as F +from torch.nn.modules.utils import _pair + +class deform_conv2d_naive(Module): + def __init__(self, in_channels, out_channels, + kernel_size, stride, padding, dilation=1, groups=1, deformable_groups=1, bias=True): + super(deform_conv2d_naive, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = _pair(kernel_size) + self.stride = _pair(stride) + self.padding = _pair(padding) + self.dilation = _pair(dilation) + self.groups = groups + self.deformable_groups = deformable_groups + self.use_bias = bias + + self.weight = nn.Parameter(torch.Tensor( + out_channels, in_channels//groups, *self.kernel_size)) + self.bias = nn.Parameter(torch.Tensor(out_channels)) + self.reset_parameters() + if not self.use_bias: + self.bias.requires_grad = False + self.bias.data.zero_() + + def reset_parameters(self): + n = self.in_channels + init.kaiming_uniform_(self.weight, a=math.sqrt(5)) + if self.bias is not None: + fan_in, _ = init._calculate_fan_in_and_fan_out(self.weight) + bound = 1 / math.sqrt(fan_in) + init.uniform_(self.bias, -bound, bound) + + def forward(self, input, offset): + N = input.size(0) + in_channels = self.in_channels + out_channels = self.out_channels + in_h = input.size(2) + in_w = input.size(3) + out_h = offset.size(2) + out_w = offset.size(3) + kernel_h = self.kernel_size[0] + kernel_w = self.kernel_size[1] + # [1, kernel_h * kernel_w, out_h, out_w, 2] + mesh = self.compute_mesh_grid(in_h, in_w).cuda(input.get_device()) + offset = offset.view(N, self.deformable_groups, kernel_h, kernel_w, 2, out_h, out_w) + # [N * dg * kernel_h * kernel_w, out_h, out_w, 2] + offset = offset.permute(0, 1, 2, 3, 5, 6, 4).contiguous().view(N * self.deformable_groups * kernel_h * kernel_w, out_h, out_w, 2) + offset_x_normalize = (offset[:, :, :, 1]) / ((in_w - 1) * 1.0 / 2) + offset_y_normalize = (offset[:, :, :, 0]) / ((in_h - 1) * 1.0 / 2) + # [N * dg * kernel_h * kernel_w, out_h, out_w, 2] + offset = torch.cat([offset_x_normalize[..., None], offset_y_normalize[..., None]], dim=3) + # [N * dg * kernel_h * kernel_w, out_h, out_w, 2] + grid = mesh.expand(N * self.deformable_groups, -1, -1, -1, -1).contiguous().view(-1, out_h, out_w, 2) + offset + # [N * kernel_h * kernel_w * dg, in_channels/dg, in_h, in_w] + input = input[:, None, ...].expand(-1, kernel_h * kernel_w, -1, -1, -1).contiguous().view( + N * kernel_h * kernel_w * self.deformable_groups, in_channels // self.deformable_groups, in_h, in_w) + sampled_feat = F.grid_sample(input, grid).view(N, kernel_h * kernel_w, in_channels, out_h, out_w).permute(2, 1, 0, 3, 4).contiguous().view(in_channels * kernel_h * kernel_w, -1) + output_feat = torch.matmul(self.weight.view(self.weight.size(0), -1), sampled_feat).view(out_channels, N, out_h, out_w).permute(1,0,2,3) + return output_feat + + def compute_mesh_grid(self, in_h, in_w): + kernel_h, kernel_w = self.kernel_size + stride_h, stride_w = self.stride + dilation_h, dilation_w = self.dilation + padding_h, padding_w = self.padding + out_h = (in_h + 2 * padding_h - (dilation_h * (kernel_h - 1) + 1)) // stride_h + 1 + out_w = (in_w + 2 * padding_w - (dilation_w * (kernel_w - 1) + 1)) // stride_w + 1 + # [out_h, out_w] + mesh_y, mesh_x = torch.meshgrid(torch.arange(out_h), torch.arange(out_w)) + mesh_y = mesh_y * stride_h - padding_h + mesh_x = mesh_x * stride_w - padding_w + # [1, out_h, out_w] + mesh_y = mesh_y.unsqueeze(0).float() + mesh_x = mesh_x.unsqueeze(0).float() + # [kernel_h, kernel_w] + kernel_offset_y, kernel_offset_x = torch.meshgrid(torch.arange(kernel_h), torch.arange(kernel_w)) + # [kernel_h * kernel_w, 1, 1] + kernel_offset_y = kernel_offset_y.float().view(kernel_h * kernel_w, 1, 1) * dilation_h + kernel_offset_x = kernel_offset_x.float().view(kernel_h * kernel_w, 1, 1) * dilation_w + # [kernel_h * kernel_w, out_h, out_w] + mesh_y = mesh_y + kernel_offset_y + mesh_x = mesh_x + kernel_offset_x + mesh_y = (mesh_y - (in_h - 1) / 2.) / ((in_h - 1) / 2.) + mesh_x = (mesh_x - (in_w - 1) / 2.) / ((in_w - 1) / 2.) + mesh = torch.cat([mesh_x[None, ..., None], mesh_y[None, ..., None]], dim=4) + return mesh diff --git a/detection/mm_modules/DCN/functions/__init__.py b/detection/mm_modules/DCN/functions/__init__.py new file mode 100644 index 0000000..a80bf4d --- /dev/null +++ b/detection/mm_modules/DCN/functions/__init__.py @@ -0,0 +1,2 @@ +from .deform_conv2d_func import DeformConv2dFunction +from .modulated_deform_conv2d_func import ModulatedDeformConv2dFunction diff --git a/detection/mm_modules/DCN/functions/deform_conv2d_func.py b/detection/mm_modules/DCN/functions/deform_conv2d_func.py new file mode 100644 index 0000000..3352d37 --- /dev/null +++ b/detection/mm_modules/DCN/functions/deform_conv2d_func.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import math +import torch +from torch import nn +from torch.autograd import Function +from torch.nn.modules.utils import _pair +from torch.autograd.function import once_differentiable +try: + from apex import amp +except ImportError: + raise ImportError("Please install apex from https://www.github.com/nvidia/apex to run this example.") +# from torch.cuda import amp +import DCN + +class DeformConv2dFunction(Function): + @staticmethod + # @amp.custom_fwd(cast_inputs=torch.float32) + @amp.float_function + def forward(ctx, input, offset, weight, bias, + stride, padding, dilation, group, deformable_groups, im2col_step): + ctx.stride = _pair(stride) + ctx.padding = _pair(padding) + ctx.dilation = _pair(dilation) + ctx.kernel_size = _pair(weight.shape[2:4]) + ctx.group = group + ctx.deformable_groups = deformable_groups + ctx.im2col_step = im2col_step + output = DCN.deform_conv2d_forward(input, weight, bias, + offset, + ctx.kernel_size[0], ctx.kernel_size[1], + ctx.stride[0], ctx.stride[1], + ctx.padding[0], ctx.padding[1], + ctx.dilation[0], ctx.dilation[1], + ctx.group, + ctx.deformable_groups, + ctx.im2col_step) + ctx.save_for_backward(input, offset, weight, bias) + return output + + @staticmethod + @once_differentiable + @amp.float_function + # @amp.custom_bwd + def backward(ctx, grad_output): + input, offset, weight, bias = ctx.saved_tensors + grad_input, grad_offset, grad_weight, grad_bias = \ + DCN.deform_conv2d_backward(input, weight, + bias, + offset, + grad_output, + ctx.kernel_size[0], ctx.kernel_size[1], + ctx.stride[0], ctx.stride[1], + ctx.padding[0], ctx.padding[1], + ctx.dilation[0], ctx.dilation[1], + ctx.group, + ctx.deformable_groups, + ctx.im2col_step) + + return grad_input, grad_offset, grad_weight, grad_bias,\ + None, None, None, None, None, None diff --git a/detection/mm_modules/DCN/functions/modulated_deform_conv2d_func.py b/detection/mm_modules/DCN/functions/modulated_deform_conv2d_func.py new file mode 100644 index 0000000..be0dfbc --- /dev/null +++ b/detection/mm_modules/DCN/functions/modulated_deform_conv2d_func.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import math +import torch +from torch import nn +from torch.autograd import Function +from torch.nn.modules.utils import _pair +from torch.autograd.function import once_differentiable +# from torch.cuda import amp +import DCN +# try: +# from apex import amp +# except ImportError: +# raise ImportError("Please install apex from https://www.github.com/nvidia/apex to run this example.") +from torch.cuda import amp + +class ModulatedDeformConv2dFunction(Function): + @staticmethod + @amp.custom_fwd(cast_inputs=torch.float32) + # @amp.float_function + def forward(ctx, input, offset, mask, weight, bias, + stride, padding, dilation, groups, deformable_groups, im2col_step): + ctx.stride = _pair(stride) + ctx.padding = _pair(padding) + ctx.dilation = _pair(dilation) + ctx.kernel_size = _pair(weight.shape[2:4]) + ctx.groups = groups + ctx.deformable_groups = deformable_groups + ctx.im2col_step = im2col_step + output = DCN.modulated_deform_conv2d_forward(input, weight, bias, + offset, mask, + ctx.kernel_size[0], ctx.kernel_size[1], + ctx.stride[0], ctx.stride[1], + ctx.padding[0], ctx.padding[1], + ctx.dilation[0], ctx.dilation[1], + ctx.groups, + ctx.deformable_groups, + ctx.im2col_step) + ctx.save_for_backward(input, offset, mask, weight, bias) + return output + + @staticmethod + @once_differentiable + @amp.custom_bwd + # @amp.float_function + def backward(ctx, grad_output): + input, offset, mask, weight, bias = ctx.saved_tensors + grad_input, grad_offset, grad_mask, grad_weight, grad_bias = \ + DCN.modulated_deform_conv2d_backward(input, weight, + bias, + offset, mask, + grad_output, + ctx.kernel_size[0], ctx.kernel_size[1], + ctx.stride[0], ctx.stride[1], + ctx.padding[0], ctx.padding[1], + ctx.dilation[0], ctx.dilation[1], + ctx.groups, + ctx.deformable_groups, + ctx.im2col_step) + + return grad_input, grad_offset, grad_mask, grad_weight, grad_bias,\ + None, None, None, None, None, None diff --git a/detection/mm_modules/DCN/make.sh b/detection/mm_modules/DCN/make.sh new file mode 100644 index 0000000..b1cf55a --- /dev/null +++ b/detection/mm_modules/DCN/make.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# SBATCH --job-name=tiny-1-pool-in-pre + +#SBATCH --account=dl65 +#SBATCH --partition=m3g + +#SBATCH -n 1 +#SBATCH -c 8 +#SBATCH --gres=gpu:V100:1 +#SBATCH --mem=16GB +#SBATCH --time=1:00:00 + +#SBATCH --mail-user=zizhengpan98@gmail.com +#SBATCH --mail-type=END +#SBATCH --mail-type=FAIL + + +# Command to run a gpu job +# For example: +module load anaconda/2019.03-Python3.7-gcc5 +module load gcc/5.4.0 +module load cuda/10.1 +module load cudnn/7.6.5-cuda10.1 +export PROJECT=dl65 +export CONDA_ENVS_PATH=/projects/$PROJECT/$USER/conda_envs +export CONDA_PKGS_DIRS=/projects/$PROJECT/$USER/conda_pkgs +source activate /projects/$PROJECT/$USER/conda_envs/defconv +which python + + +nvidia-smi +nvcc -V +python setup.py build install diff --git a/detection/mm_modules/DCN/make_dsai.sh b/detection/mm_modules/DCN/make_dsai.sh new file mode 100644 index 0000000..db22bbd --- /dev/null +++ b/detection/mm_modules/DCN/make_dsai.sh @@ -0,0 +1,22 @@ +#!/bin/bash +#SBATCH --account=hhe +#SBATCH --time=48:00:00 +#SBATCH --ntasks=1 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=20 +#SBATCH --gres=gpu:1 +#SBATCH --mem=64GB +#SBATCH --exclude=node04 + +#SBATCH --mail-user=zizhengpan98@gmail.com +#SBATCH --mail-type=END +#SBATCH --mail-type=FAIL + +# Command to run a gpu job +# For example: +module load cuda-11.2.0-gcc-10.2.0-gsjevs3 +source activate torch171 + +nvidia-smi +nvcc -V +python setup.py build install diff --git a/detection/mm_modules/DCN/modules/__init__.py b/detection/mm_modules/DCN/modules/__init__.py new file mode 100644 index 0000000..552cca0 --- /dev/null +++ b/detection/mm_modules/DCN/modules/__init__.py @@ -0,0 +1,2 @@ +from .deform_conv2d import DeformConv2d, _DeformConv2d, DeformConv2dPack, DeformConv2dPackMore +from .modulated_deform_conv2d import ModulatedDeformConv2d, _ModulatedDeformConv2d, ModulatedDeformConv2dPack diff --git a/detection/mm_modules/DCN/modules/deform_conv2d.py b/detection/mm_modules/DCN/modules/deform_conv2d.py new file mode 100644 index 0000000..5d9698b --- /dev/null +++ b/detection/mm_modules/DCN/modules/deform_conv2d.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import torch +import math +from torch import nn +from torch.nn import init +from torch.nn.modules.utils import _pair + +from ..functions.deform_conv2d_func import DeformConv2dFunction + +class DeformConv2d(nn.Module): + + def __init__(self, in_channels, out_channels, + kernel_size, stride, padding, dilation=1, groups=1, deformable_groups=1, im2col_step=32, bias=True): + super(DeformConv2d, self).__init__() + + if in_channels % groups != 0: + raise ValueError('in_channels {} must be divisible by groups {}'.format(in_channels, groups)) + if out_channels % groups != 0: + raise ValueError('out_channels {} must be divisible by groups {}'.format(out_channels, groups)) + + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = _pair(kernel_size) + self.stride = _pair(stride) + self.padding = _pair(padding) + self.dilation = _pair(dilation) + self.groups = groups + self.deformable_groups = deformable_groups + self.im2col_step = im2col_step + self.use_bias = bias + + self.weight = nn.Parameter(torch.Tensor( + out_channels, in_channels//groups, *self.kernel_size)) + self.bias = nn.Parameter(torch.Tensor(out_channels)) + self.reset_parameters() + if not self.use_bias: + self.bias.requires_grad = False + self.bias.data.zero_() + + def reset_parameters(self): + n = self.in_channels + init.kaiming_uniform_(self.weight, a=math.sqrt(5)) + if self.bias is not None: + fan_in, _ = init._calculate_fan_in_and_fan_out(self.weight) + bound = 1 / math.sqrt(fan_in) + init.uniform_(self.bias, -bound, bound) + + def forward(self, input, offset): + assert 2 * self.deformable_groups * self.kernel_size[0] * self.kernel_size[1] == \ + offset.shape[1] + return DeformConv2dFunction.apply(input, offset, + self.weight, + self.bias, + self.stride, + self.padding, + self.dilation, + self.groups, + self.deformable_groups, + self.im2col_step) + +_DeformConv2d = DeformConv2dFunction.apply + +class DeformConv2dPack(DeformConv2d): + + def __init__(self, in_channels, out_channels, + kernel_size, stride, padding, + dilation=1, groups=1, deformable_groups=1, im2col_step=32, bias=True, lr_mult=0.1): + super(DeformConv2dPack, self).__init__(in_channels, out_channels, + kernel_size, stride, padding, dilation, groups, deformable_groups, im2col_step, bias) + + out_channels = self.deformable_groups * 2 * self.kernel_size[0] * self.kernel_size[1] + self.conv_offset = nn.Conv2d(self.in_channels, + out_channels, + kernel_size=self.kernel_size, + stride=self.stride, + padding=self.padding, + bias=True) + self.conv_offset.lr_mult = lr_mult + self.conv_offset.inited = True + self.init_offset() + + def init_offset(self): + self.conv_offset.weight.data.zero_() + self.conv_offset.bias.data.zero_() + + def forward(self, input, return_offset=False): + offset = self.conv_offset(input) + bs = input.size()[0] + im2col_step = bs // 2 if bs > 1 else 1 + # return DeformConv2dFunction.apply(input, offset, + # self.weight, + # self.bias, + # self.stride, + # self.padding, + # self.dilation, + # self.groups, + # self.deformable_groups, + # im2col_step) + out = DeformConv2dFunction.apply(input, offset, + self.weight, + self.bias, + self.stride, + self.padding, + self.dilation, + self.groups, + self.deformable_groups, + im2col_step) + if return_offset: + return out, offset + return out, None + + +class DeformConv2dPackMore(DeformConv2d): + + def __init__(self, in_channels, out_channels, + kernel_size, stride, padding, + dilation=1, groups=1, deformable_groups=1, im2col_step=64, bias=True, lr_mult=0.1): + super(DeformConv2dPackMore, self).__init__(in_channels, out_channels, + kernel_size, stride, padding, dilation, groups, deformable_groups, im2col_step, bias) + + out_channels = self.deformable_groups * 2 * self.kernel_size[0] * self.kernel_size[1] + self.conv_offset = nn.Sequential( + nn.Conv2d(self.in_channels, self.in_channels//4, kernel_size=1, bias=False), + nn.BatchNorm2d(self.in_channels//4), + nn.ReLU(inplace=True), + nn.Conv2d(self.in_channels//4, out_channels, kernel_size=self.kernel_size, stride=self.stride, padding=self.padding, bias=True) + ) + self.conv_offset[-1].lr_mult = lr_mult + self.conv_offset[-1].inited = True + self.init_offset() + + def init_offset(self): + self.conv_offset[-1].weight.data.zero_() + self.conv_offset[-1].bias.data.zero_() + + def forward(self, input): + offset = self.conv_offset(input) + bs = input.size()[0] + im2col_step = bs // 2 + return DeformConv2dFunction.apply(input, offset, + self.weight, + self.bias, + self.stride, + self.padding, + self.dilation, + self.groups, + self.deformable_groups, + im2col_step) diff --git a/detection/mm_modules/DCN/modules/modulated_deform_conv2d.py b/detection/mm_modules/DCN/modules/modulated_deform_conv2d.py new file mode 100644 index 0000000..2052051 --- /dev/null +++ b/detection/mm_modules/DCN/modules/modulated_deform_conv2d.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import torch +import math +from torch import nn +from torch.nn import init +from torch.nn.modules.utils import _pair + +from ..functions.modulated_deform_conv2d_func import ModulatedDeformConv2dFunction + +class ModulatedDeformConv2d(nn.Module): + + def __init__(self, in_channels, out_channels, + kernel_size, stride, padding, dilation=1, groups=1, deformable_groups=1, im2col_step=64, bias=True): + super(ModulatedDeformConv2d, self).__init__() + + if in_channels % groups != 0: + raise ValueError('in_channels {} must be divisible by groups {}'.format(in_channels, groups)) + if out_channels % groups != 0: + raise ValueError('out_channels {} must be divisible by groups {}'.format(out_channels, groups)) + + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = _pair(kernel_size) + self.stride = _pair(stride) + self.padding = _pair(padding) + self.dilation = _pair(dilation) + self.groups = groups + self.deformable_groups = deformable_groups + self.im2col_step = im2col_step + self.use_bias = bias + + self.weight = nn.Parameter(torch.Tensor( + out_channels, in_channels//groups, *self.kernel_size)) + self.bias = nn.Parameter(torch.Tensor(out_channels)) + self.reset_parameters() + if not self.use_bias: + self.bias.requires_grad = False + + def reset_parameters(self): + n = self.in_channels + init.kaiming_uniform_(self.weight, a=math.sqrt(5)) + if self.bias is not None: + fan_in, _ = init._calculate_fan_in_and_fan_out(self.weight) + bound = 1 / math.sqrt(fan_in) + init.uniform_(self.bias, -bound, bound) + + def forward(self, input, offset, mask): + assert 2 * self.deformable_groups * self.kernel_size[0] * self.kernel_size[1] == \ + offset.shape[1] + assert self.deformable_groups * self.kernel_size[0] * self.kernel_size[1] == \ + mask.shape[1] + return ModulatedDeformConv2dFunction.apply(input, offset, mask, + self.weight, + self.bias, + self.stride, + self.padding, + self.dilation, + self.groups, + self.deformable_groups, + self.im2col_step) + +_ModulatedDeformConv2d = ModulatedDeformConv2dFunction.apply + +class ModulatedDeformConv2dPack(ModulatedDeformConv2d): + + def __init__(self, in_channels, out_channels, + kernel_size, stride, padding, + dilation=1, groups=1, deformable_groups=1, im2col_step=64, bias=True, lr_mult=0.1): + super(ModulatedDeformConv2dPack, self).__init__(in_channels, out_channels, + kernel_size, stride, padding, dilation, groups, deformable_groups, im2col_step, bias) + + out_channels = self.deformable_groups * 3 * self.kernel_size[0] * self.kernel_size[1] + self.conv_offset_mask = nn.Conv2d(self.in_channels, + out_channels, + kernel_size=self.kernel_size, + stride=self.stride, + padding=self.padding, + bias=True) + self.conv_offset_mask.lr_mult = lr_mult + self.conv_offset_mask.inited = True + self.init_offset() + + def init_offset(self): + self.conv_offset_mask.weight.data.zero_() + self.conv_offset_mask.bias.data.zero_() + + def forward(self, input, return_offset=False): + out = self.conv_offset_mask(input) + o1, o2, mask = torch.chunk(out, 3, dim=1) + offset = torch.cat((o1, o2), dim=1) + mask = torch.sigmoid(mask) + + bs = input.size()[0] + im2col_step = bs // 2 + + out = ModulatedDeformConv2dFunction.apply(input, offset, mask, + self.weight, + self.bias, + self.stride, + self.padding, + self.dilation, + self.groups, + self.deformable_groups, + im2col_step) + if return_offset: + return out, offset + return out, None diff --git a/detection/mm_modules/DCN/setup.py b/detection/mm_modules/DCN/setup.py new file mode 100644 index 0000000..55cf53d --- /dev/null +++ b/detection/mm_modules/DCN/setup.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python + +import os +import glob + +import torch + +from torch.utils.cpp_extension import CUDA_HOME +from torch.utils.cpp_extension import CppExtension +from torch.utils.cpp_extension import CUDAExtension + +from setuptools import find_packages +from setuptools import setup + +requirements = ["torch", "torchvision"] + +def get_extensions(): + this_dir = os.path.dirname(os.path.abspath(__file__)) + extensions_dir = os.path.join(this_dir, "src") + + main_file = glob.glob(os.path.join(extensions_dir, "*.cpp")) + source_cpu = glob.glob(os.path.join(extensions_dir, "cpu", "*.cpp")) + source_cuda = glob.glob(os.path.join(extensions_dir, "cuda", "*.cu")) + + sources = main_file + source_cpu + extension = CppExtension + extra_compile_args = {"cxx": []} + define_macros = [] + + if torch.cuda.is_available() and CUDA_HOME is not None: + extension = CUDAExtension + sources += source_cuda + define_macros += [("WITH_CUDA", None)] + extra_compile_args["nvcc"] = [ + "-DCUDA_HAS_FP16=1", + "-D__CUDA_NO_HALF_OPERATORS__", + "-D__CUDA_NO_HALF_CONVERSIONS__", + "-D__CUDA_NO_HALF2_OPERATORS__", + ] + else: + raise NotImplementedError('Cuda is not availabel') + + sources = [os.path.join(extensions_dir, s) for s in sources] + include_dirs = [extensions_dir] + ext_modules = [ + extension( + "DCN", + sources, + include_dirs=include_dirs, + define_macros=define_macros, + extra_compile_args=extra_compile_args, + ) + ] + return ext_modules + +setup( + name="DCN", + version="1.0", + description="deformable convolutional networks", + packages=find_packages(exclude=("configs", "tests",)), + ext_modules=get_extensions(), + cmdclass={"build_ext": torch.utils.cpp_extension.BuildExtension}, +) diff --git a/detection/mm_modules/DCN/src/cpu/deform_conv2d_cpu.cpp b/detection/mm_modules/DCN/src/cpu/deform_conv2d_cpu.cpp new file mode 100644 index 0000000..64a67bb --- /dev/null +++ b/detection/mm_modules/DCN/src/cpu/deform_conv2d_cpu.cpp @@ -0,0 +1,47 @@ +#include + +#include +#include + + +at::Tensor +deform_conv2d_cpu_forward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + AT_ERROR("Not implement on cpu"); +} + +std::vector +deform_conv2d_cpu_backward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &grad_output, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + AT_ERROR("Not implement on cpu"); +} + diff --git a/detection/mm_modules/DCN/src/cpu/deform_conv2d_cpu.h b/detection/mm_modules/DCN/src/cpu/deform_conv2d_cpu.h new file mode 100644 index 0000000..585d3d8 --- /dev/null +++ b/detection/mm_modules/DCN/src/cpu/deform_conv2d_cpu.h @@ -0,0 +1,39 @@ +#pragma once +#include + +at::Tensor +deform_conv2d_cpu_forward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step); + +std::vector +deform_conv2d_cpu_backward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &grad_output, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step); + + diff --git a/detection/mm_modules/DCN/src/cpu/modulated_deform_conv2d_cpu.cpp b/detection/mm_modules/DCN/src/cpu/modulated_deform_conv2d_cpu.cpp new file mode 100644 index 0000000..b712a19 --- /dev/null +++ b/detection/mm_modules/DCN/src/cpu/modulated_deform_conv2d_cpu.cpp @@ -0,0 +1,49 @@ +#include + +#include +#include + + +at::Tensor +modulated_deform_conv2d_cpu_forward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &mask, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + AT_ERROR("Not implement on cpu"); +} + +std::vector +modulated_deform_conv2d_cpu_backward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &mask, + const at::Tensor &grad_output, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + AT_ERROR("Not implement on cpu"); +} + diff --git a/detection/mm_modules/DCN/src/cpu/modulated_deform_conv2d_cpu.h b/detection/mm_modules/DCN/src/cpu/modulated_deform_conv2d_cpu.h new file mode 100644 index 0000000..8f54d0e --- /dev/null +++ b/detection/mm_modules/DCN/src/cpu/modulated_deform_conv2d_cpu.h @@ -0,0 +1,41 @@ +#pragma once +#include + +at::Tensor +modulated_deform_conv2d_cpu_forward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &mask, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step); + +std::vector +modulated_deform_conv2d_cpu_backward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &mask, + const at::Tensor &grad_output, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step); + + diff --git a/detection/mm_modules/DCN/src/cuda/deform_2d_im2col_cuda.cuh b/detection/mm_modules/DCN/src/cuda/deform_2d_im2col_cuda.cuh new file mode 100644 index 0000000..266f3a2 --- /dev/null +++ b/detection/mm_modules/DCN/src/cuda/deform_2d_im2col_cuda.cuh @@ -0,0 +1,391 @@ +#include +#include +#include + +#include +#include + +// #include +#include +// #include + +#define CUDA_KERNEL_LOOP(i, n) \ + for (int i = blockIdx.x * blockDim.x + threadIdx.x; \ + i < (n); \ + i += blockDim.x * gridDim.x) + +const int CUDA_NUM_THREADS = 1024; +inline int GET_BLOCKS(const int N) +{ + return (N + CUDA_NUM_THREADS - 1) / CUDA_NUM_THREADS; +} + +template +__device__ scalar_t dmcn_2d_im2col_bilinear(const scalar_t *bottom_data, const int data_width, + const int height, const int width, scalar_t h, scalar_t w) +{ + int h_low = floor(h); + int w_low = floor(w); + int h_high = h_low + 1; + int w_high = w_low + 1; + + scalar_t lh = h - h_low; + scalar_t lw = w - w_low; + scalar_t hh = 1 - lh, hw = 1 - lw; + + scalar_t v1 = 0; + if (h_low >= 0 && w_low >= 0) + v1 = bottom_data[h_low * data_width + w_low]; + scalar_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) + v2 = bottom_data[h_low * data_width + w_high]; + scalar_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) + v3 = bottom_data[h_high * data_width + w_low]; + scalar_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) + v4 = bottom_data[h_high * data_width + w_high]; + + scalar_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + + scalar_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + return val; +} + +template +__device__ scalar_t dmcn_2d_get_gradient_weight(scalar_t argmax_h, scalar_t argmax_w, + const int h, const int w, const int height, const int width) +{ + if (argmax_h <= -1 || argmax_h >= height || argmax_w <= -1 || argmax_w >= width) + { + //empty + return 0; + } + + int argmax_h_low = floor(argmax_h); + int argmax_w_low = floor(argmax_w); + int argmax_h_high = argmax_h_low + 1; + int argmax_w_high = argmax_w_low + 1; + + scalar_t weight = 0; + if (h == argmax_h_low && w == argmax_w_low) + weight = (h + 1 - argmax_h) * (w + 1 - argmax_w); + if (h == argmax_h_low && w == argmax_w_high) + weight = (h + 1 - argmax_h) * (argmax_w + 1 - w); + if (h == argmax_h_high && w == argmax_w_low) + weight = (argmax_h + 1 - h) * (w + 1 - argmax_w); + if (h == argmax_h_high && w == argmax_w_high) + weight = (argmax_h + 1 - h) * (argmax_w + 1 - w); + return weight; +} + +template +__device__ scalar_t dmcn_2d_get_coordinate_weight(scalar_t argmax_h, scalar_t argmax_w, + const int height, const int width, const scalar_t *im_data, + const int data_width, const int bp_dir) +{ + if (argmax_h <= -1 || argmax_h >= height || argmax_w <= -1 || argmax_w >= width) + { + //empty + return 0; + } + + int argmax_h_low = floor(argmax_h); + int argmax_w_low = floor(argmax_w); + int argmax_h_high = argmax_h_low + 1; + int argmax_w_high = argmax_w_low + 1; + + scalar_t weight = 0; + + if (bp_dir == 0) + { + if (argmax_h_low >= 0 && argmax_w_low >= 0) + weight += -1 * (argmax_w_low + 1 - argmax_w) * im_data[argmax_h_low * data_width + argmax_w_low]; + if (argmax_h_low >= 0 && argmax_w_high <= width - 1) + weight += -1 * (argmax_w - argmax_w_low) * im_data[argmax_h_low * data_width + argmax_w_high]; + if (argmax_h_high <= height - 1 && argmax_w_low >= 0) + weight += (argmax_w_low + 1 - argmax_w) * im_data[argmax_h_high * data_width + argmax_w_low]; + if (argmax_h_high <= height - 1 && argmax_w_high <= width - 1) + weight += (argmax_w - argmax_w_low) * im_data[argmax_h_high * data_width + argmax_w_high]; + } + else if (bp_dir == 1) + { + if (argmax_h_low >= 0 && argmax_w_low >= 0) + weight += -1 * (argmax_h_low + 1 - argmax_h) * im_data[argmax_h_low * data_width + argmax_w_low]; + if (argmax_h_low >= 0 && argmax_w_high <= width - 1) + weight += (argmax_h_low + 1 - argmax_h) * im_data[argmax_h_low * data_width + argmax_w_high]; + if (argmax_h_high <= height - 1 && argmax_w_low >= 0) + weight += -1 * (argmax_h - argmax_h_low) * im_data[argmax_h_high * data_width + argmax_w_low]; + if (argmax_h_high <= height - 1 && argmax_w_high <= width - 1) + weight += (argmax_h - argmax_h_low) * im_data[argmax_h_high * data_width + argmax_w_high]; + } + + return weight; +} + +template +__global__ void deformable_2d_im2col_gpu_kernel(const int n, + const scalar_t *data_im, const scalar_t *data_offset, + const int height, const int width, const int kernel_h, + const int kernel_w, + const int pad_h, const int pad_w, + const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, const int num_channels, + const int deformable_group, + const int height_col, const int width_col, + scalar_t *data_col) +{ + // launch channels * batch_size * height_col * width_col cores + CUDA_KERNEL_LOOP(index, n) + { + // NOTE(CharlesShang): different from Dai Jifeng's MXNet implementation, col_buffer is of shape (c*kw*kh, N, oh, ow) + // here columns is of shape (N, c*kw*kh, oh * ow), need to adapt axis + // NOTE(Jiarui XU): different from CharlesShang's implementation, col_buffer is of shape (N, c*kw*kh, oh * ow) + // here columns is of shape (c*kw*kh, N, oh, ow), need to adapt axis + + // index index of output matrix + const int w_col = index % width_col; + const int h_col = (index / width_col) % height_col; + const int b_col = (index / width_col / height_col) % batch_size; + const int c_im = (index / width_col / height_col) / batch_size; + const int c_col = c_im * kernel_h * kernel_w; + + // compute deformable group index + const int deformable_group_index = c_im / channel_per_deformable_group; + + const int h_in = h_col * stride_h - pad_h; + const int w_in = w_col * stride_w - pad_w; + + scalar_t *data_col_ptr = data_col + ((c_col * batch_size + b_col) * height_col + h_col) * width_col + w_col; + // const scalar_t* data_im_ptr = data_im + ((b_col * num_channels + c_im) * height + h_in) * width + w_in; + const scalar_t *data_im_ptr = data_im + (b_col * num_channels + c_im) * height * width; + const scalar_t *data_offset_ptr = data_offset + (b_col * deformable_group + deformable_group_index) * 2 * kernel_h * kernel_w * height_col * width_col; + + for (int i = 0; i < kernel_h; ++i) + { + for (int j = 0; j < kernel_w; ++j) + { + const int data_offset_h_ptr = ((2 * (i * kernel_w + j)) * height_col + h_col) * width_col + w_col; + const int data_offset_w_ptr = ((2 * (i * kernel_w + j) + 1) * height_col + h_col) * width_col + w_col; + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + scalar_t val = static_cast(0); + const scalar_t h_im = h_in + i * dilation_h + offset_h; + const scalar_t w_im = w_in + j * dilation_w + offset_w; + if (h_im > -1 && w_im > -1 && h_im < height && w_im < width) + { + //const scalar_t map_h = i * dilation_h + offset_h; + //const scalar_t map_w = j * dilation_w + offset_w; + //const int cur_height = height - h_in; + //const int cur_width = width - w_in; + //val = dmcn_im2col_bilinear(data_im_ptr, width, cur_height, cur_width, map_h, map_w); + val = dmcn_2d_im2col_bilinear(data_im_ptr, width, height, width, h_im, w_im); + } + *data_col_ptr = val; + data_col_ptr += batch_size * height_col * width_col; + } + } + } +} + +template +__global__ void deformable_2d_col2im_gpu_kernel(const int n, + const scalar_t *data_col, const scalar_t *data_offset, + const int channels, const int height, const int width, + const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, + const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, const int deformable_group, + const int height_col, const int width_col, + scalar_t *grad_im) +{ + CUDA_KERNEL_LOOP(index, n) + { + const int j = (index / width_col / height_col / batch_size) % kernel_w; + const int i = (index / width_col / height_col / batch_size / kernel_w) % kernel_h; + const int c = index / width_col / height_col / batch_size / kernel_w / kernel_h; + // compute the start and end of the output + + const int deformable_group_index = c / channel_per_deformable_group; + + int w_out = index % width_col; + int h_out = (index / width_col) % height_col; + int b = (index / width_col / height_col) % batch_size; + int w_in = w_out * stride_w - pad_w; + int h_in = h_out * stride_h - pad_h; + + const scalar_t *data_offset_ptr = data_offset + (b * deformable_group + deformable_group_index) * 2 * kernel_h * kernel_w * height_col * width_col; + const int data_offset_h_ptr = ((2 * (i * kernel_w + j)) * height_col + h_out) * width_col + w_out; + const int data_offset_w_ptr = ((2 * (i * kernel_w + j) + 1) * height_col + h_out) * width_col + w_out; + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + const scalar_t cur_inv_h_data = h_in + i * dilation_h + offset_h; + const scalar_t cur_inv_w_data = w_in + j * dilation_w + offset_w; + + const scalar_t cur_top_grad = data_col[index]; + const int cur_h = (int)cur_inv_h_data; + const int cur_w = (int)cur_inv_w_data; + for (int dy = -2; dy <= 2; dy++) + { + for (int dx = -2; dx <= 2; dx++) + { + if (cur_h + dy >= 0 && cur_h + dy < height && + cur_w + dx >= 0 && cur_w + dx < width && + abs(cur_inv_h_data - (cur_h + dy)) < 1 && + abs(cur_inv_w_data - (cur_w + dx)) < 1) + { + int cur_bottom_grad_pos = ((b * channels + c) * height + cur_h + dy) * width + cur_w + dx; + scalar_t weight = dmcn_2d_get_gradient_weight(cur_inv_h_data, cur_inv_w_data, cur_h + dy, cur_w + dx, height, width); + atomicAdd(grad_im + cur_bottom_grad_pos, weight * cur_top_grad); + } + } + } + } +} + +template +__global__ void deformable_2d_col2im_coord_gpu_kernel(const int n, + const scalar_t *data_col, const scalar_t *data_im, + const scalar_t *data_offset, + const int channels, const int height, const int width, + const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, + const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, const int offset_channels, + const int deformable_group, + const int height_col, const int width_col, + scalar_t *grad_offset) +{ + CUDA_KERNEL_LOOP(index, n) + { + scalar_t val = 0; + int w = index % width_col; + int h = (index / width_col) % height_col; + int c = (index / width_col / height_col) % offset_channels; + int b = (index / width_col / height_col) / offset_channels; + // compute the start and end of the output + + const int deformable_group_index = c / (2 * kernel_h * kernel_w); + const int col_step = kernel_h * kernel_w; + int cnt = 0; + const scalar_t *data_col_ptr = data_col + deformable_group_index * channel_per_deformable_group * batch_size * width_col * height_col; + const scalar_t *data_im_ptr = data_im + (b * deformable_group + deformable_group_index) * channel_per_deformable_group / kernel_h / kernel_w * height * width; + const scalar_t *data_offset_ptr = data_offset + (b * deformable_group + deformable_group_index) * 2 * kernel_h * kernel_w * height_col * width_col; + + const int offset_c = c - deformable_group_index * 2 * kernel_h * kernel_w; + + for (int col_c = (offset_c / 2); col_c < channel_per_deformable_group; col_c += col_step) + { + const int col_pos = (((col_c * batch_size + b) * height_col) + h) * width_col + w; + const int bp_dir = offset_c % 2; + + int j = (col_pos / width_col / height_col / batch_size) % kernel_w; + int i = (col_pos / width_col / height_col / batch_size / kernel_w) % kernel_h; + int w_out = col_pos % width_col; + int h_out = (col_pos / width_col) % height_col; + int w_in = w_out * stride_w - pad_w; + int h_in = h_out * stride_h - pad_h; + const int data_offset_h_ptr = (((2 * (i * kernel_w + j)) * height_col + h_out) * width_col + w_out); + const int data_offset_w_ptr = (((2 * (i * kernel_w + j) + 1) * height_col + h_out) * width_col + w_out); + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + scalar_t inv_h = h_in + i * dilation_h + offset_h; + scalar_t inv_w = w_in + j * dilation_w + offset_w; + if (inv_h <= -1 || inv_w <= -1 || inv_h >= height || inv_w >= width) + { + inv_h = inv_w = -2; + } + const scalar_t weight = dmcn_2d_get_coordinate_weight( + inv_h, inv_w, + height, width, data_im_ptr + cnt * height * width, width, bp_dir); + val += weight * data_col_ptr[col_pos]; + cnt += 1; + } + // KERNEL_ASSIGN(grad_offset[index], offset_req, val); + grad_offset[index] = val; + } +} + +template +void deformable_2d_im2col_cuda(cudaStream_t stream, + const scalar_t *data_im, const scalar_t *data_offset, + const int batch_size, const int channels, const int height_im, const int width_im, + const int height_col, const int width_col, const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int deformable_group, scalar_t *data_col) { + // num_axes should be smaller than block size + const int channel_per_deformable_group = channels / deformable_group; + const int num_kernels = channels * batch_size * height_col * width_col; + deformable_2d_im2col_gpu_kernel + <<>>( + num_kernels, data_im, data_offset, height_im, width_im, kernel_h, kernel_w, + pad_h, pad_w, stride_h, stride_w, dilation_h, dilation_w, channel_per_deformable_group, + batch_size, channels, deformable_group, height_col, width_col, data_col); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in deformable_im2col_cuda: %s\n", cudaGetErrorString(err)); + } + +} + +template +void deformable_2d_col2im_cuda(cudaStream_t stream, + const scalar_t *data_col, const scalar_t *data_offset, + const int batch_size, const int channels, const int height_im, const int width_im, + const int height_col, const int width_col, const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int deformable_group, scalar_t *grad_im){ + + const int channel_per_deformable_group = channels / deformable_group; + const int num_kernels = channels * kernel_h * kernel_w * batch_size * height_col * width_col; + deformable_2d_col2im_gpu_kernel + <<>>( + num_kernels, data_col, data_offset, channels, height_im, width_im, + kernel_h, kernel_w, pad_h, pad_h, stride_h, stride_w, + dilation_h, dilation_w, channel_per_deformable_group, + batch_size, deformable_group, height_col, width_col, grad_im); + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in deformable_col2im_cuda: %s\n", cudaGetErrorString(err)); + } + +} + +template +void deformable_2d_col2im_coord_cuda(cudaStream_t stream, + const scalar_t *data_col, const scalar_t *data_im, const scalar_t *data_offset, + const int batch_size, const int channels, const int height_im, const int width_im, + const int height_col, const int width_col, const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int deformable_group, + scalar_t *grad_offset) { + const int num_kernels = batch_size * height_col * width_col * 2 * kernel_h * kernel_w * deformable_group; + const int channel_per_deformable_group = channels * kernel_h * kernel_w / deformable_group; + deformable_2d_col2im_coord_gpu_kernel + <<>>( + num_kernels, data_col, data_im, data_offset, channels, height_im, width_im, + kernel_h, kernel_w, pad_h, pad_w, stride_h, stride_w, + dilation_h, dilation_w, channel_per_deformable_group, + batch_size, 2 * kernel_h * kernel_w * deformable_group, deformable_group, height_col, width_col, + grad_offset); + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in deformable_col2im_coord_cuda: %s\n", cudaGetErrorString(err)); + } +} \ No newline at end of file diff --git a/detection/mm_modules/DCN/src/cuda/deform_conv2d_cuda.cu b/detection/mm_modules/DCN/src/cuda/deform_conv2d_cuda.cu new file mode 100644 index 0000000..0493211 --- /dev/null +++ b/detection/mm_modules/DCN/src/cuda/deform_conv2d_cuda.cu @@ -0,0 +1,273 @@ +#include +#include "cuda/deform_2d_im2col_cuda.cuh" + +#include +#include +#include +#include + +// #include +// #include +// #include + +// extern THCState *state; + +// author: Charles Shang +// https://github.com/torch/cunn/blob/master/lib/THCUNN/generic/SpatialConvolutionMM.cu + + +at::Tensor +deform_conv2d_cuda_forward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + // THCAssertSameGPU(THCudaTensor_checkGPU(state, 5, input, weight, bias, offset, mask)); + + AT_ASSERTM(input.is_contiguous(), "input tensor has to be contiguous"); + AT_ASSERTM(weight.is_contiguous(), "weight tensor has to be contiguous"); + + AT_ASSERTM(input.type().is_cuda(), "input must be a CUDA tensor"); + AT_ASSERTM(weight.type().is_cuda(), "weight must be a CUDA tensor"); + AT_ASSERTM(bias.type().is_cuda(), "bias must be a CUDA tensor"); + AT_ASSERTM(offset.type().is_cuda(), "offset must be a CUDA tensor"); + + const int batch = input.size(0); + const int channels = input.size(1); + const int height = input.size(2); + const int width = input.size(3); + + const int channels_out = weight.size(0); + const int channels_kernel = weight.size(1); + const int kernel_h_ = weight.size(2); + const int kernel_w_ = weight.size(3); + + const int im2col_step_ = std::min(batch, im2col_step); + // if (batch % im2col_step_ != 0) { + // printf("batch: %d im2col_step_: %d\n", batch, im2col_step_); + // } + + AT_ASSERTM(batch % im2col_step_ == 0, "batch(%d) must divide im2col_step(%d)", batch, im2col_step_); + AT_ASSERTM((channels % group == 0) && (channels_out % group == 0), + "channels(%d) and channels_out(%d) must divide group(%d)", channels, channels_out, group); + + // printf("Kernels: %d %d %d %d\n", kernel_h_, kernel_w_, kernel_w, kernel_h); + // printf("Channels: %d %d\n", channels, channels_kernel); + // printf("Channels: %d %d\n", channels_out, channels_kernel); + + AT_ASSERTM(kernel_h_ == kernel_h && kernel_w_ == kernel_w, + "Input shape and kernel shape wont match: (%d x %d vs %d x %d).", kernel_h_, kernel_w, kernel_h_, kernel_w_); + + AT_ASSERTM(channels == (channels_kernel * group), + "Input shape and kernel channels wont match: (%d vs %d).", channels, channels_kernel * group); + + const int height_out = (height + 2 * pad_h - (dilation_h * (kernel_h - 1) + 1)) / stride_h + 1; + const int width_out = (width + 2 * pad_w - (dilation_w * (kernel_w - 1) + 1)) / stride_w + 1; + + auto output = at::empty({batch * height_out * width_out, channels_out}, input.options()); + + // prepare group weight and bias + auto weight_g = weight.view({group, channels_out/group, channels_kernel, kernel_h, kernel_w}); + auto bias_g = bias.view({group, channels_out/group}); + + // define alias for easy use + const int batch_n = im2col_step_; + const int per_input_size = channels * height * width; + const int per_offset_size = offset.size(1) * offset.size(2) * offset.size(3); + auto output_n = output.view({batch/im2col_step_, batch_n * height_out * width_out, channels_out}); + for (int n = 0; n < batch/im2col_step_; ++n) + { + auto columns = at::empty({channels * kernel_h * kernel_w, batch_n * height_out * width_out}, input.options()); + AT_DISPATCH_FLOATING_TYPES(input.type(), "deform_conv_forward_cuda", ([&] { + deformable_2d_im2col_cuda(at::cuda::getCurrentCUDAStream(), + input.data() + n * im2col_step_ * per_input_size, + offset.data() + n * im2col_step_ * per_offset_size, + batch_n, channels, height, width, + height_out, width_out, kernel_h, kernel_w, + pad_h, pad_w, stride_h, stride_w, dilation_h, dilation_w, + deformable_group, + columns.data()); + + })); + + // auto columns_m = columns.t(); + // auto weight_m = weight.view({channels_out, channels_kernel * kernel_h * kernel_w}).t(); + // output = at::addmm(bias, columns_m, weight_m); + auto columns_g = columns.view({group, channels/group * kernel_h * kernel_w, batch_n * height_out * width_out}); + auto output_g = output_n.select(0, n).view({batch_n * height_out * width_out, group, channels_out/group}); + for (int g = 0; g < group; ++g) + { + auto columns_gm = columns_g.select(0, g).t(); + auto weight_gm = weight_g.select(0, g).view({channels_out/group, channels_kernel * kernel_h * kernel_w}).t(); + auto output_m = at::addmm(bias_g.select(0, g), columns_gm, weight_gm); + output_g.select(1, g) = output_m.view({batch_n * height_out * width_out, channels_out/group}); + } + + } + + output = output.view({batch, height_out, width_out, channels_out}).permute({0, 3, 1, 2}).contiguous(); + + return output; +} + +std::vector deform_conv2d_cuda_backward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &grad_output, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + + AT_ASSERTM(input.is_contiguous(), "input tensor has to be contiguous"); + AT_ASSERTM(weight.is_contiguous(), "weight tensor has to be contiguous"); + + AT_ASSERTM(input.type().is_cuda(), "input must be a CUDA tensor"); + AT_ASSERTM(weight.type().is_cuda(), "weight must be a CUDA tensor"); + AT_ASSERTM(bias.type().is_cuda(), "bias must be a CUDA tensor"); + AT_ASSERTM(offset.type().is_cuda(), "offset must be a CUDA tensor"); + + const int batch = input.size(0); + const int channels = input.size(1); + const int height = input.size(2); + const int width = input.size(3); + + const int channels_out = weight.size(0); + const int channels_kernel = weight.size(1); + const int kernel_h_ = weight.size(2); + const int kernel_w_ = weight.size(3); + + const int batch_ = grad_output.size(0); + const int channels_out_ = grad_output.size(1); + const int height_out_ = grad_output.size(2); + const int width_out_ = grad_output.size(3); + + const int im2col_step_ = std::min(im2col_step, batch); + + AT_ASSERTM(batch % im2col_step_ == 0, "batch(%d) must divide im2col_step(%d)", batch, im2col_step_); + + AT_ASSERTM((channels % group == 0) && (channels_out % group == 0), + "channels(%d) and channels_out(%d) must divide group(%d)", channels, channels_out, group); + + AT_ASSERTM(kernel_h_ == kernel_h && kernel_w_ == kernel_w, + "Input shape and kernel shape wont match: (%d x %d vs %d x %d).", kernel_h_, kernel_w, kernel_h_, kernel_w_); + + AT_ASSERTM(channels == (channels_kernel * group), + "Input shape and kernel channels wont match: (%d vs %d).", channels, channels_kernel * group); + + const int height_out = (height + 2 * pad_h - (dilation_h * (kernel_h - 1) + 1)) / stride_h + 1; + const int width_out = (width + 2 * pad_w - (dilation_w * (kernel_w - 1) + 1)) / stride_w + 1; + + AT_ASSERTM(batch == batch_, + "Input shape and grad_out batch wont match: (%d vs %d).", batch, batch_); + + AT_ASSERTM(channels_out == channels_out_, + "Input shape and grad_out channels_out wont match: (%d vs %d).", channels_out, channels_out_); + + AT_ASSERTM(height_out == height_out_ && width_out == width_out_, + "Input shape and grad_out shape wont match: (%d x %d vs %d x %d).", height_out, height_out_, width_out, width_out_); + + auto grad_input = at::zeros_like(input); + auto grad_offset = at::zeros_like(offset); + auto grad_weight = at::zeros_like(weight); + auto grad_bias = at::zeros_like(bias); + + // auto grad_output_m = grad_output.permute({1, 0, 2, 3}).contiguous().view({channels_out, batch * height_out * width_out}); + // auto weight_m = weight.view({channels_out, channels_kernel * kernel_h * kernel_w}).t(); + // columns = at::mm(weight_m, grad_output_m); + + // prepare group weight and bias + auto weight_g = weight.view({group, channels_out/group, channels_kernel, kernel_h, kernel_w}); + auto grad_weight_g = grad_weight.view({group, channels_out/group, channels_kernel, kernel_h, kernel_w}); + auto grad_bias_g = grad_bias.view({group, channels_out/group}); + + const int batch_n = im2col_step_; + const int per_input_size = channels * height * width; + const int per_offset_size = offset.size(1) * offset.size(2) * offset.size(3); + auto grad_output_n = grad_output.view({batch/im2col_step_, batch_n, channels_out, height_out, width_out}); + for (int n = 0; n < batch/im2col_step_; ++n) + { + auto grad_output_g = grad_output_n.select(0, n).view({batch_n, group, channels_out/group, height_out, width_out}); + auto ones = at::ones({batch_n * height_out * width_out}, input.options()); + auto columns = at::empty({channels * kernel_h * kernel_w, batch_n * 1 * height_out * width_out}, input.options()); + auto columns_g = columns.view({group, channels/group * kernel_h * kernel_w, batch_n * height_out * width_out}); + for (int g = 0; g < group; ++g) + { + auto grad_output_gm = grad_output_g.select(1, g).permute({1, 0, 2, 3}).contiguous().view({channels_out/group, batch_n * height_out * width_out}); + auto weight_gm = weight_g.select(0, g).view({channels_out/group, channels_kernel * kernel_h * kernel_w}).t(); + columns_g.select(0, g) = at::mm(weight_gm, grad_output_gm); + } + + AT_DISPATCH_FLOATING_TYPES(input.type(), "deform_conv_backward_cuda", ([&] { + deformable_2d_col2im_coord_cuda(at::cuda::getCurrentCUDAStream(), + columns.data(), + input.data() + n * im2col_step_ * per_input_size, + offset.data() + n * im2col_step_ * per_offset_size, + batch_n, channels, height, width, + height_out, width_out, kernel_h, kernel_w, + pad_h, pad_w, stride_h, stride_w, + dilation_h, dilation_w, deformable_group, + grad_offset.data() + n * im2col_step_ * per_offset_size); + // gradient w.r.t. input data + deformable_2d_col2im_cuda(at::cuda::getCurrentCUDAStream(), + columns.data(), + offset.data() + n * im2col_step_ * per_offset_size, + batch_n, channels, height, width, + height_out, width_out, kernel_h, kernel_w, + pad_h, pad_w, stride_h, stride_w, + dilation_h, dilation_w, deformable_group, + grad_input.data() + n * im2col_step_ * per_input_size); + + // gradient w.r.t. weight, dWeight should accumulate across the batch and group + deformable_2d_im2col_cuda(at::cuda::getCurrentCUDAStream(), + input.data() + n * im2col_step_ * per_input_size, + offset.data() + n * im2col_step_ * per_offset_size, + batch_n, channels, height, width, + height_out, width_out, kernel_h, kernel_w, + pad_h, pad_w, stride_h, stride_w, + dilation_h, dilation_w, deformable_group, + columns.data()); + + })); + + // auto grad_output_m = grad_output.permute({1, 0, 2, 3}).contiguous().view({channels_out, batch * height_out * width_out}); + // grad_weight = at::mm(grad_output_m, columns.t()).view_as(weight); + // grad_bias = at::mv(grad_output_m, ones); + // auto grad_output_g = grad_output.view({batch, group, channels_out/group, height_out, width_out}); + // auto columns_g = columns.view({group, channels/group * kernel_h * kernel_w, batch * height_out * width_out}); + for (int g = 0; g < group; ++g) + { + auto grad_output_gm = grad_output_g.select(1, g).permute({1, 0, 2, 3}).contiguous().view({channels_out/group, batch_n * height_out * width_out}); + auto columns_gm = columns_g.select(0, g).t(); + auto grad_weight_gm = grad_weight_g.select(0, g).view({channels_out/group, channels_kernel * kernel_h * kernel_w}); + auto grad_bias_gm = grad_bias_g.select(0, g); + grad_weight_g.select(0, g) = at::addmm(grad_weight_gm, grad_output_gm, columns_gm).view_as(grad_weight_g.select(0, g)); + grad_bias_g.select(0, g) = at::addmv(grad_bias_gm, grad_output_gm, ones); + } + + } + + return { + grad_input, grad_offset, grad_weight, grad_bias + }; +} diff --git a/detection/mm_modules/DCN/src/cuda/deform_conv2d_cuda.h b/detection/mm_modules/DCN/src/cuda/deform_conv2d_cuda.h new file mode 100644 index 0000000..0958453 --- /dev/null +++ b/detection/mm_modules/DCN/src/cuda/deform_conv2d_cuda.h @@ -0,0 +1,38 @@ +#pragma once +#include + +at::Tensor +deform_conv2d_cuda_forward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step); + +std::vector +deform_conv2d_cuda_backward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &grad_output, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step); + diff --git a/detection/mm_modules/DCN/src/cuda/modulated_deform_2d_im2col_cuda.cuh b/detection/mm_modules/DCN/src/cuda/modulated_deform_2d_im2col_cuda.cuh new file mode 100644 index 0000000..0ff18cb --- /dev/null +++ b/detection/mm_modules/DCN/src/cuda/modulated_deform_2d_im2col_cuda.cuh @@ -0,0 +1,420 @@ +#include +#include +#include + +#include +#include + +// #include +#include +// #include + +#define CUDA_KERNEL_LOOP(i, n) \ + for (int i = blockIdx.x * blockDim.x + threadIdx.x; \ + i < (n); \ + i += blockDim.x * gridDim.x) + +const int CUDA_NUM_THREADS = 1024; +inline int GET_BLOCKS(const int N) +{ + return (N + CUDA_NUM_THREADS - 1) / CUDA_NUM_THREADS; +} + + +template +__device__ scalar_t mdmcn_2d_im2col_bilinear(const scalar_t *bottom_data, const int data_width, + const int height, const int width, scalar_t h, scalar_t w) +{ + int h_low = floor(h); + int w_low = floor(w); + int h_high = h_low + 1; + int w_high = w_low + 1; + + scalar_t lh = h - h_low; + scalar_t lw = w - w_low; + scalar_t hh = 1 - lh, hw = 1 - lw; + + scalar_t v1 = 0; + if (h_low >= 0 && w_low >= 0) + v1 = bottom_data[h_low * data_width + w_low]; + scalar_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) + v2 = bottom_data[h_low * data_width + w_high]; + scalar_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) + v3 = bottom_data[h_high * data_width + w_low]; + scalar_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) + v4 = bottom_data[h_high * data_width + w_high]; + + scalar_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + + scalar_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + return val; +} + +template +__device__ scalar_t mdmcn_2d_get_gradient_weight(scalar_t argmax_h, scalar_t argmax_w, + const int h, const int w, const int height, const int width) +{ + if (argmax_h <= -1 || argmax_h >= height || argmax_w <= -1 || argmax_w >= width) + { + //empty + return 0; + } + + int argmax_h_low = floor(argmax_h); + int argmax_w_low = floor(argmax_w); + int argmax_h_high = argmax_h_low + 1; + int argmax_w_high = argmax_w_low + 1; + + scalar_t weight = 0; + if (h == argmax_h_low && w == argmax_w_low) + weight = (h + 1 - argmax_h) * (w + 1 - argmax_w); + if (h == argmax_h_low && w == argmax_w_high) + weight = (h + 1 - argmax_h) * (argmax_w + 1 - w); + if (h == argmax_h_high && w == argmax_w_low) + weight = (argmax_h + 1 - h) * (w + 1 - argmax_w); + if (h == argmax_h_high && w == argmax_w_high) + weight = (argmax_h + 1 - h) * (argmax_w + 1 - w); + return weight; +} + +template +__device__ scalar_t mdmcn_2d_get_coordinate_weight(scalar_t argmax_h, scalar_t argmax_w, + const int height, const int width, const scalar_t *im_data, + const int data_width, const int bp_dir) +{ + if (argmax_h <= -1 || argmax_h >= height || argmax_w <= -1 || argmax_w >= width) + { + //empty + return 0; + } + + int argmax_h_low = floor(argmax_h); + int argmax_w_low = floor(argmax_w); + int argmax_h_high = argmax_h_low + 1; + int argmax_w_high = argmax_w_low + 1; + + scalar_t weight = 0; + + if (bp_dir == 0) + { + if (argmax_h_low >= 0 && argmax_w_low >= 0) + weight += -1 * (argmax_w_low + 1 - argmax_w) * im_data[argmax_h_low * data_width + argmax_w_low]; + if (argmax_h_low >= 0 && argmax_w_high <= width - 1) + weight += -1 * (argmax_w - argmax_w_low) * im_data[argmax_h_low * data_width + argmax_w_high]; + if (argmax_h_high <= height - 1 && argmax_w_low >= 0) + weight += (argmax_w_low + 1 - argmax_w) * im_data[argmax_h_high * data_width + argmax_w_low]; + if (argmax_h_high <= height - 1 && argmax_w_high <= width - 1) + weight += (argmax_w - argmax_w_low) * im_data[argmax_h_high * data_width + argmax_w_high]; + } + else if (bp_dir == 1) + { + if (argmax_h_low >= 0 && argmax_w_low >= 0) + weight += -1 * (argmax_h_low + 1 - argmax_h) * im_data[argmax_h_low * data_width + argmax_w_low]; + if (argmax_h_low >= 0 && argmax_w_high <= width - 1) + weight += (argmax_h_low + 1 - argmax_h) * im_data[argmax_h_low * data_width + argmax_w_high]; + if (argmax_h_high <= height - 1 && argmax_w_low >= 0) + weight += -1 * (argmax_h - argmax_h_low) * im_data[argmax_h_high * data_width + argmax_w_low]; + if (argmax_h_high <= height - 1 && argmax_w_high <= width - 1) + weight += (argmax_h - argmax_h_low) * im_data[argmax_h_high * data_width + argmax_w_high]; + } + + return weight; +} + +template +__global__ void modulated_deformable_2d_im2col_gpu_kernel(const int n, + const scalar_t *data_im, const scalar_t *data_offset, + const scalar_t *data_mask, + const int height, const int width, const int kernel_h, + const int kernel_w, + const int pad_h, const int pad_w, + const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, const int num_channels, + const int deformable_group, + const int height_col, const int width_col, + scalar_t *data_col) +{ + // launch channels * batch_size * height_col * width_col cores + CUDA_KERNEL_LOOP(index, n) + { + // NOTE(CharlesShang): different from Dai Jifeng's MXNet implementation, col_buffer is of shape (c*kw*kh, N, oh, ow) + // here columns is of shape (N, c*kw*kh, oh * ow), need to adapt axis + // NOTE(Jiarui XU): different from CharlesShang's implementation, col_buffer is of shape (N, c*kw*kh, oh * ow) + // here columns is of shape (c*kw*kh, N, oh, ow), need to adapt axis + + // index index of output matrix + const int w_col = index % width_col; + const int h_col = (index / width_col) % height_col; + const int b_col = (index / width_col / height_col) % batch_size; + const int c_im = (index / width_col / height_col) / batch_size; + const int c_col = c_im * kernel_h * kernel_w; + + // compute deformable group index + const int deformable_group_index = c_im / channel_per_deformable_group; + + const int h_in = h_col * stride_h - pad_h; + const int w_in = w_col * stride_w - pad_w; + + scalar_t *data_col_ptr = data_col + ((c_col * batch_size + b_col) * height_col + h_col) * width_col + w_col; + //const scalar_t* data_im_ptr = data_im + ((b_col * num_channels + c_im) * height + h_in) * width + w_in; + const scalar_t *data_im_ptr = data_im + (b_col * num_channels + c_im) * height * width; + const scalar_t *data_offset_ptr = data_offset + (b_col * deformable_group + deformable_group_index) * 2 * kernel_h * kernel_w * height_col * width_col; + + const scalar_t *data_mask_ptr = data_mask + (b_col * deformable_group + deformable_group_index) * kernel_h * kernel_w * height_col * width_col; + + for (int i = 0; i < kernel_h; ++i) + { + for (int j = 0; j < kernel_w; ++j) + { + const int data_offset_h_ptr = ((2 * (i * kernel_w + j)) * height_col + h_col) * width_col + w_col; + const int data_offset_w_ptr = ((2 * (i * kernel_w + j) + 1) * height_col + h_col) * width_col + w_col; + const int data_mask_hw_ptr = ((i * kernel_w + j) * height_col + h_col) * width_col + w_col; + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + const scalar_t mask = data_mask_ptr[data_mask_hw_ptr]; + scalar_t val = static_cast(0); + const scalar_t h_im = h_in + i * dilation_h + offset_h; + const scalar_t w_im = w_in + j * dilation_w + offset_w; + if (h_im > -1 && w_im > -1 && h_im < height && w_im < width) + { + //const scalar_t map_h = i * dilation_h + offset_h; + //const scalar_t map_w = j * dilation_w + offset_w; + //const int cur_height = height - h_in; + //const int cur_width = width - w_in; + //val = dmcn_im2col_bilinear(data_im_ptr, width, cur_height, cur_width, map_h, map_w); + val = mdmcn_2d_im2col_bilinear(data_im_ptr, width, height, width, h_im, w_im); + } + *data_col_ptr = val * mask; + data_col_ptr += batch_size * height_col * width_col; + } + } + } +} + +template +__global__ void modulated_deformable_2d_col2im_gpu_kernel(const int n, + const scalar_t *data_col, const scalar_t *data_offset, + const scalar_t *data_mask, + const int channels, const int height, const int width, + const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, + const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, const int deformable_group, + const int height_col, const int width_col, + scalar_t *grad_im) +{ + CUDA_KERNEL_LOOP(index, n) + { + const int j = (index / width_col / height_col / batch_size) % kernel_w; + const int i = (index / width_col / height_col / batch_size / kernel_w) % kernel_h; + const int c = index / width_col / height_col / batch_size / kernel_w / kernel_h; + // compute the start and end of the output + + const int deformable_group_index = c / channel_per_deformable_group; + + int w_out = index % width_col; + int h_out = (index / width_col) % height_col; + int b = (index / width_col / height_col) % batch_size; + int w_in = w_out * stride_w - pad_w; + int h_in = h_out * stride_h - pad_h; + + const scalar_t *data_offset_ptr = data_offset + (b * deformable_group + deformable_group_index) * 2 * kernel_h * kernel_w * height_col * width_col; + const scalar_t *data_mask_ptr = data_mask + (b * deformable_group + deformable_group_index) * kernel_h * kernel_w * height_col * width_col; + const int data_offset_h_ptr = ((2 * (i * kernel_w + j)) * height_col + h_out) * width_col + w_out; + const int data_offset_w_ptr = ((2 * (i * kernel_w + j) + 1) * height_col + h_out) * width_col + w_out; + const int data_mask_hw_ptr = ((i * kernel_w + j) * height_col + h_out) * width_col + w_out; + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + const scalar_t mask = data_mask_ptr[data_mask_hw_ptr]; + const scalar_t cur_inv_h_data = h_in + i * dilation_h + offset_h; + const scalar_t cur_inv_w_data = w_in + j * dilation_w + offset_w; + + const scalar_t cur_top_grad = data_col[index] * mask; + const int cur_h = (int)cur_inv_h_data; + const int cur_w = (int)cur_inv_w_data; + for (int dy = -2; dy <= 2; dy++) + { + for (int dx = -2; dx <= 2; dx++) + { + if (cur_h + dy >= 0 && cur_h + dy < height && + cur_w + dx >= 0 && cur_w + dx < width && + abs(cur_inv_h_data - (cur_h + dy)) < 1 && + abs(cur_inv_w_data - (cur_w + dx)) < 1) + { + int cur_bottom_grad_pos = ((b * channels + c) * height + cur_h + dy) * width + cur_w + dx; + scalar_t weight = mdmcn_2d_get_gradient_weight(cur_inv_h_data, cur_inv_w_data, cur_h + dy, cur_w + dx, height, width); + atomicAdd(grad_im + cur_bottom_grad_pos, weight * cur_top_grad); + } + } + } + } +} + +template +__global__ void modulated_deformable_2d_col2im_coord_gpu_kernel(const int n, + const scalar_t *data_col, const scalar_t *data_im, + const scalar_t *data_offset, const scalar_t *data_mask, + const int channels, const int height, const int width, + const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, + const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, const int offset_channels, + const int deformable_group, + const int height_col, const int width_col, + scalar_t *grad_offset, scalar_t *grad_mask) +{ + CUDA_KERNEL_LOOP(index, n) + { + scalar_t val = 0, mval = 0; + int w = index % width_col; + int h = (index / width_col) % height_col; + int c = (index / width_col / height_col) % offset_channels; + int b = (index / width_col / height_col) / offset_channels; + // compute the start and end of the output + + const int deformable_group_index = c / (2 * kernel_h * kernel_w); + const int col_step = kernel_h * kernel_w; + int cnt = 0; + const scalar_t *data_col_ptr = data_col + deformable_group_index * channel_per_deformable_group * batch_size * width_col * height_col; + const scalar_t *data_im_ptr = data_im + (b * deformable_group + deformable_group_index) * channel_per_deformable_group / kernel_h / kernel_w * height * width; + const scalar_t *data_offset_ptr = data_offset + (b * deformable_group + deformable_group_index) * 2 * kernel_h * kernel_w * height_col * width_col; + const scalar_t *data_mask_ptr = data_mask + (b * deformable_group + deformable_group_index) * kernel_h * kernel_w * height_col * width_col; + + const int offset_c = c - deformable_group_index * 2 * kernel_h * kernel_w; + + for (int col_c = (offset_c / 2); col_c < channel_per_deformable_group; col_c += col_step) + { + const int col_pos = (((col_c * batch_size + b) * height_col) + h) * width_col + w; + const int bp_dir = offset_c % 2; + + int j = (col_pos / width_col / height_col / batch_size) % kernel_w; + int i = (col_pos / width_col / height_col / batch_size / kernel_w) % kernel_h; + int w_out = col_pos % width_col; + int h_out = (col_pos / width_col) % height_col; + int w_in = w_out * stride_w - pad_w; + int h_in = h_out * stride_h - pad_h; + const int data_offset_h_ptr = (((2 * (i * kernel_w + j)) * height_col + h_out) * width_col + w_out); + const int data_offset_w_ptr = (((2 * (i * kernel_w + j) + 1) * height_col + h_out) * width_col + w_out); + const int data_mask_hw_ptr = (((i * kernel_w + j) * height_col + h_out) * width_col + w_out); + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + const scalar_t mask = data_mask_ptr[data_mask_hw_ptr]; + scalar_t inv_h = h_in + i * dilation_h + offset_h; + scalar_t inv_w = w_in + j * dilation_w + offset_w; + if (inv_h <= -1 || inv_w <= -1 || inv_h >= height || inv_w >= width) + { + inv_h = inv_w = -2; + } + else + { + mval += data_col_ptr[col_pos] * mdmcn_2d_im2col_bilinear(data_im_ptr + cnt * height * width, width, height, width, inv_h, inv_w); + } + const scalar_t weight = mdmcn_2d_get_coordinate_weight( + inv_h, inv_w, + height, width, data_im_ptr + cnt * height * width, width, bp_dir); + val += weight * data_col_ptr[col_pos] * mask; + cnt += 1; + } + // KERNEL_ASSIGN(grad_offset[index], offset_req, val); + grad_offset[index] = val; + if (offset_c % 2 == 0) + // KERNEL_ASSIGN(grad_mask[(((b * deformable_group + deformable_group_index) * kernel_h * kernel_w + offset_c / 2) * height_col + h) * width_col + w], mask_req, mval); + grad_mask[(((b * deformable_group + deformable_group_index) * kernel_h * kernel_w + offset_c / 2) * height_col + h) * width_col + w] = mval; + } +} + +template +void modulated_deformable_2d_im2col_cuda(cudaStream_t stream, + const scalar_t *data_im, const scalar_t *data_offset, + const scalar_t *data_mask, + const int batch_size, const int channels, const int height_im, + const int width_im, + const int height_col, const int width_col, const int kernel_h, + const int kernel_w, + const int pad_h, const int pad_w, const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int deformable_group, scalar_t *data_col) { + // num_axes should be smaller than block size + const int channel_per_deformable_group = channels / deformable_group; + const int num_kernels = channels * batch_size * height_col * width_col; + modulated_deformable_2d_im2col_gpu_kernel + <<>>( + num_kernels, data_im, data_offset, data_mask, height_im, width_im, kernel_h, kernel_w, + pad_h, pad_w, stride_h, stride_w, dilation_h, dilation_w, channel_per_deformable_group, + batch_size, channels, deformable_group, height_col, width_col, data_col); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in modulated_deformable_im2col_cuda: %s\n", cudaGetErrorString(err)); + } + +} + +template +void modulated_deformable_2d_col2im_cuda(cudaStream_t stream, + const scalar_t *data_col, const scalar_t *data_offset, + const scalar_t *data_mask, + const int batch_size, const int channels, const int height_im, + const int width_im, + const int height_col, const int width_col, const int kernel_h, + const int kernel_w, + const int pad_h, const int pad_w, const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int deformable_group, scalar_t *grad_im){ + + const int channel_per_deformable_group = channels / deformable_group; + const int num_kernels = channels * kernel_h * kernel_w * batch_size * height_col * width_col; + modulated_deformable_2d_col2im_gpu_kernel + <<>>( + num_kernels, data_col, data_offset, data_mask, channels, height_im, width_im, + kernel_h, kernel_w, pad_h, pad_h, stride_h, stride_w, + dilation_h, dilation_w, channel_per_deformable_group, + batch_size, deformable_group, height_col, width_col, grad_im); + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in modulated_deformable_col2im_cuda: %s\n", cudaGetErrorString(err)); + } + +} + +template +void modulated_deformable_2d_col2im_coord_cuda(cudaStream_t stream, + const scalar_t *data_col, const scalar_t *data_im, + const scalar_t *data_offset, const scalar_t *data_mask, + const int batch_size, const int channels, const int height_im, + const int width_im, + const int height_col, const int width_col, const int kernel_h, + const int kernel_w, + const int pad_h, const int pad_w, const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int deformable_group, + scalar_t *grad_offset, scalar_t *grad_mask) { + const int num_kernels = batch_size * height_col * width_col * 2 * kernel_h * kernel_w * deformable_group; + const int channel_per_deformable_group = channels * kernel_h * kernel_w / deformable_group; + modulated_deformable_2d_col2im_coord_gpu_kernel + <<>>( + num_kernels, data_col, data_im, data_offset, data_mask, channels, height_im, width_im, + kernel_h, kernel_w, pad_h, pad_w, stride_h, stride_w, + dilation_h, dilation_w, channel_per_deformable_group, + batch_size, 2 * kernel_h * kernel_w * deformable_group, deformable_group, height_col, width_col, + grad_offset, grad_mask); + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in modulated_deformable_col2im_coord_cuda: %s\n", cudaGetErrorString(err)); + } +} \ No newline at end of file diff --git a/detection/mm_modules/DCN/src/cuda/modulated_deform_conv2d_cuda.cu b/detection/mm_modules/DCN/src/cuda/modulated_deform_conv2d_cuda.cu new file mode 100644 index 0000000..18ec02b --- /dev/null +++ b/detection/mm_modules/DCN/src/cuda/modulated_deform_conv2d_cuda.cu @@ -0,0 +1,280 @@ +#include +#include "cuda/modulated_deform_2d_im2col_cuda.cuh" + +#include +#include +#include +#include + +// #include +// #include +// #include + +// extern THCState *state; + +// author: Charles Shang +// https://github.com/torch/cunn/blob/master/lib/THCUNN/generic/SpatialConvolutionMM.cu + + +at::Tensor +modulated_deform_conv2d_cuda_forward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &mask, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + // THCAssertSameGPU(THCudaTensor_checkGPU(state, 5, input, weight, bias, offset, mask)); + + AT_ASSERTM(input.is_contiguous(), "input tensor has to be contiguous"); + AT_ASSERTM(weight.is_contiguous(), "weight tensor has to be contiguous"); + + AT_ASSERTM(input.type().is_cuda(), "input must be a CUDA tensor"); + AT_ASSERTM(weight.type().is_cuda(), "weight must be a CUDA tensor"); + AT_ASSERTM(bias.type().is_cuda(), "bias must be a CUDA tensor"); + AT_ASSERTM(offset.type().is_cuda(), "offset must be a CUDA tensor"); + AT_ASSERTM(mask.type().is_cuda(), "mask must be a CUDA tensor"); + + const int batch = input.size(0); + const int channels = input.size(1); + const int height = input.size(2); + const int width = input.size(3); + + const int channels_out = weight.size(0); + const int channels_kernel = weight.size(1); + const int kernel_h_ = weight.size(2); + const int kernel_w_ = weight.size(3); + + const int im2col_step_ = std::min(batch, im2col_step); + + AT_ASSERTM(batch % im2col_step_ == 0, "batch(%d) must divide im2col_step(%d)", batch, im2col_step_); + + AT_ASSERTM((channels % group == 0) && (channels_out % group == 0), + "channels(%d) and channels_out(%d) must divide group(%d)", channels, channels_out, group); + + // printf("Kernels: %d %d %d %d\n", kernel_h_, kernel_w_, kernel_w, kernel_h); + // printf("Channels: %d %d\n", channels, channels_kernel); + // printf("Channels: %d %d\n", channels_out, channels_kernel); + + AT_ASSERTM(kernel_h_ == kernel_h && kernel_w_ == kernel_w, + "Input shape and kernel shape wont match: (%d x %d vs %d x %d).", kernel_h_, kernel_w, kernel_h_, kernel_w_); + + AT_ASSERTM(channels == (channels_kernel * group), + "Input shape and kernel channels wont match: (%d vs %d).", channels, channels_kernel * group); + + const int height_out = (height + 2 * pad_h - (dilation_h * (kernel_h - 1) + 1)) / stride_h + 1; + const int width_out = (width + 2 * pad_w - (dilation_w * (kernel_w - 1) + 1)) / stride_w + 1; + + auto output = at::empty({batch * height_out * width_out, channels_out}, input.options()); + + // prepare group weight and bias + auto weight_g = weight.view({group, channels_out/group, channels_kernel, kernel_h, kernel_w}); + auto bias_g = bias.view({group, channels_out/group}); + + // define alias for easy use + const int batch_n = im2col_step_; + const int per_input_size = channels * height * width; + const int per_offset_size = offset.size(1) * offset.size(2) * offset.size(3); + const int per_mask_size = mask.size(1) * mask.size(2) * mask.size(3); + auto output_n = output.view({batch/im2col_step_, batch_n * height_out * width_out, channels_out}); + for (int n = 0; n < batch/im2col_step_; ++n) + { + auto columns = at::empty({channels * kernel_h * kernel_w, batch_n * height_out * width_out}, input.options()); + AT_DISPATCH_FLOATING_TYPES(input.type(), "deform_conv_forward_cuda", ([&] { + modulated_deformable_2d_im2col_cuda(at::cuda::getCurrentCUDAStream(), + input.data() + n * im2col_step_ * per_input_size, + offset.data() + n * im2col_step_ * per_offset_size, + mask.data() + n * im2col_step_ * per_mask_size, + batch_n, channels, height, width, + height_out, width_out, kernel_h, kernel_w, + pad_h, pad_w, stride_h, stride_w, dilation_h, dilation_w, + deformable_group, + columns.data()); + + })); + + auto columns_g = columns.view({group, channels/group * kernel_h * kernel_w, batch_n * height_out * width_out}); + auto output_g = output_n.select(0, n).view({batch_n * height_out * width_out, group, channels_out/group}); + for (int g = 0; g < group; ++g) + { + auto columns_gm = columns_g.select(0, g).t(); + auto weight_gm = weight_g.select(0, g).view({channels_out/group, channels_kernel * kernel_h * kernel_w}).t(); + auto output_m = at::addmm(bias_g.select(0, g), columns_gm, weight_gm); + output_g.select(1, g) = output_m.view({batch_n * height_out * width_out, channels_out/group}); + } + + } + + output = output.view({batch, height_out, width_out, channels_out}).permute({0, 3, 1, 2}).contiguous(); + + return output; +} + + +std::vector modulated_deform_conv2d_cuda_backward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &mask, + const at::Tensor &grad_output, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + + AT_ASSERTM(input.is_contiguous(), "input tensor has to be contiguous"); + AT_ASSERTM(weight.is_contiguous(), "weight tensor has to be contiguous"); + + AT_ASSERTM(input.type().is_cuda(), "input must be a CUDA tensor"); + AT_ASSERTM(weight.type().is_cuda(), "weight must be a CUDA tensor"); + AT_ASSERTM(bias.type().is_cuda(), "bias must be a CUDA tensor"); + AT_ASSERTM(offset.type().is_cuda(), "offset must be a CUDA tensor"); + AT_ASSERTM(mask.type().is_cuda(), "mask must be a CUDA tensor"); + + const int batch = input.size(0); + const int channels = input.size(1); + const int height = input.size(2); + const int width = input.size(3); + + const int channels_out = weight.size(0); + const int channels_kernel = weight.size(1); + const int kernel_h_ = weight.size(2); + const int kernel_w_ = weight.size(3); + + const int batch_ = grad_output.size(0); + const int channels_out_ = grad_output.size(1); + const int height_out_ = grad_output.size(2); + const int width_out_ = grad_output.size(3); + + const int im2col_step_ = std::min(im2col_step, batch); + + AT_ASSERTM(batch % im2col_step_ == 0, "batch(%d) must divide im2col_step(%d)", batch, im2col_step_); + + AT_ASSERTM((channels % group == 0) && (channels_out % group == 0), + "channels(%d) and channels_out(%d) must divide group(%d)", channels, channels_out, group); + + AT_ASSERTM(kernel_h_ == kernel_h && kernel_w_ == kernel_w, + "Input shape and kernel shape wont match: (%d x %d vs %d x %d).", kernel_h_, kernel_w, kernel_h_, kernel_w_); + + AT_ASSERTM(channels == (channels_kernel * group), + "Input shape and kernel channels wont match: (%d vs %d).", channels, channels_kernel * group); + + const int height_out = (height + 2 * pad_h - (dilation_h * (kernel_h - 1) + 1)) / stride_h + 1; + const int width_out = (width + 2 * pad_w - (dilation_w * (kernel_w - 1) + 1)) / stride_w + 1; + + AT_ASSERTM(batch == batch_, + "Input shape and grad_out batch wont match: (%d vs %d).", batch, batch_); + + AT_ASSERTM(channels_out == channels_out_, + "Input shape and grad_out channels_out wont match: (%d vs %d).", channels_out, channels_out_); + + AT_ASSERTM(height_out == height_out_ && width_out == width_out_, + "Input shape and grad_out shape wont match: (%d x %d vs %d x %d).", height_out, height_out_, width_out, width_out_); + + auto ones = at::ones({batch * height_out * width_out}, input.options()); + auto columns = at::empty({channels * kernel_h * kernel_w, batch * 1 * height_out * width_out}, input.options()); + + auto grad_input = at::zeros_like(input); + auto grad_weight = at::zeros_like(weight); + auto grad_bias = at::zeros_like(bias); + auto grad_offset = at::zeros_like(offset); + auto grad_mask = at::zeros_like(mask); + + // prepare group weight and bias + auto weight_g = weight.view({group, channels_out/group, channels_kernel, kernel_h, kernel_w}); + auto grad_weight_g = grad_weight.view({group, channels_out/group, channels_kernel, kernel_h, kernel_w}); + auto grad_bias_g = grad_bias.view({group, channels_out/group}); + + const int batch_n = im2col_step_; + const int per_input_size = channels * height * width; + const int per_offset_size = offset.size(1) * offset.size(2) * offset.size(3); + const int per_mask_size = mask.size(1) * mask.size(2) * mask.size(3); + auto grad_output_n = grad_output.view({batch/im2col_step_, batch_n, channels_out, height_out, width_out}); + for (int n = 0; n < batch/im2col_step_; ++n) + { + auto grad_output_g = grad_output_n.select(0, n).view({batch_n, group, channels_out/group, height_out, width_out}); + auto ones = at::ones({batch_n * height_out * width_out}, input.options()); + auto columns = at::empty({channels * kernel_h * kernel_w, batch_n * 1 * height_out * width_out}, input.options()); + auto columns_g = columns.view({group, channels/group * kernel_h * kernel_w, batch_n * height_out * width_out}); + for (int g = 0; g < group; ++g) + { + auto grad_output_gm = grad_output_g.select(1, g).permute({1, 0, 2, 3}).contiguous().view({channels_out/group, batch_n * height_out * width_out}); + auto weight_gm = weight_g.select(0, g).view({channels_out/group, channels_kernel * kernel_h * kernel_w}).t(); + columns_g.select(0, g) = at::mm(weight_gm, grad_output_gm); + } + + AT_DISPATCH_FLOATING_TYPES(input.type(), "deform_conv_backward_cuda", ([&] { + modulated_deformable_2d_col2im_coord_cuda(at::cuda::getCurrentCUDAStream(), + columns.data(), + input.data() + n * im2col_step_ * per_input_size, + offset.data() + n * im2col_step_ * per_offset_size, + mask.data() + n * im2col_step_ * per_mask_size, + batch_n, channels, height, width, + height_out, width_out, kernel_h, kernel_w, + pad_h, pad_w, stride_h, stride_w, + dilation_h, dilation_w, deformable_group, + grad_offset.data() + n * im2col_step_ * per_offset_size, + grad_mask.data() + n * im2col_step_ * per_mask_size); + // gradient w.r.t. input data + modulated_deformable_2d_col2im_cuda(at::cuda::getCurrentCUDAStream(), + columns.data(), + offset.data() + n * im2col_step_ * per_offset_size, + mask.data() + n * im2col_step_ * per_mask_size, + batch_n, channels, height, width, + height_out, width_out, kernel_h, kernel_w, + pad_h, pad_w, stride_h, stride_w, + dilation_h, dilation_w, deformable_group, + grad_input.data() + n * im2col_step_ * per_input_size); + + // gradient w.r.t. weight, dWeight should accumulate across the batch and group + modulated_deformable_2d_im2col_cuda(at::cuda::getCurrentCUDAStream(), + input.data() + n * im2col_step_ * per_input_size, + offset.data() + n * im2col_step_ * per_offset_size, + mask.data() + n * im2col_step_ * per_mask_size, + batch_n, channels, height, width, + height_out, width_out, kernel_h, kernel_w, + pad_h, pad_w, stride_h, stride_w, + dilation_h, dilation_w, deformable_group, + columns.data()); + + })); + + // auto grad_output_m = grad_output.permute({1, 0, 2, 3}).contiguous().view({channels_out, batch * height_out * width_out}); + // grad_weight = at::mm(grad_output_m, columns.t()).view_as(weight); + // grad_bias = at::mv(grad_output_m, ones); + // auto grad_output_g = grad_output.view({batch, group, channels_out/group, height_out, width_out}); + // auto columns_g = columns.view({group, channels/group * kernel_h * kernel_w, batch * height_out * width_out}); + for (int g = 0; g < group; ++g) + { + auto grad_output_gm = grad_output_g.select(1, g).permute({1, 0, 2, 3}).contiguous().view({channels_out/group, batch_n * height_out * width_out}); + auto columns_gm = columns_g.select(0, g).t(); + auto grad_weight_gm = grad_weight_g.select(0, g).view({channels_out/group, channels_kernel * kernel_h * kernel_w}); + auto grad_bias_gm = grad_bias_g.select(0, g); + grad_weight_g.select(0, g) = at::addmm(grad_weight_gm, grad_output_gm, columns_gm).view_as(grad_weight_g.select(0, g)); + grad_bias_g.select(0, g) = at::addmv(grad_bias_gm, grad_output_gm, ones); + } + + } + + return { + grad_input, grad_offset, grad_mask, grad_weight, grad_bias + }; +} diff --git a/detection/mm_modules/DCN/src/cuda/modulated_deform_conv2d_cuda.h b/detection/mm_modules/DCN/src/cuda/modulated_deform_conv2d_cuda.h new file mode 100644 index 0000000..4ee2dce --- /dev/null +++ b/detection/mm_modules/DCN/src/cuda/modulated_deform_conv2d_cuda.h @@ -0,0 +1,40 @@ +#pragma once +#include + +at::Tensor +modulated_deform_conv2d_cuda_forward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &mask, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step); + +std::vector +modulated_deform_conv2d_cuda_backward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &mask, + const at::Tensor &grad_output, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step); + diff --git a/detection/mm_modules/DCN/src/deform_conv2d.h b/detection/mm_modules/DCN/src/deform_conv2d.h new file mode 100644 index 0000000..bf0af29 --- /dev/null +++ b/detection/mm_modules/DCN/src/deform_conv2d.h @@ -0,0 +1,84 @@ +#pragma once + +#include "cpu/deform_conv2d_cpu.h" + +#ifdef WITH_CUDA +#include "cuda/deform_conv2d_cuda.h" +#endif + + +at::Tensor +deform_conv2d_forward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + if (input.type().is_cuda()) + { +#ifdef WITH_CUDA + return deform_conv2d_cuda_forward(input, weight, bias, offset, + kernel_h, kernel_w, + stride_h, stride_w, + pad_h, pad_w, + dilation_h, dilation_w, + group, + deformable_group, + im2col_step); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} + +std::vector +deform_conv2d_backward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &grad_output, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + if (input.type().is_cuda()) + { +#ifdef WITH_CUDA + return deform_conv2d_cuda_backward(input, + weight, + bias, + offset, + grad_output, + kernel_h, kernel_w, + stride_h, stride_w, + pad_h, pad_w, + dilation_h, dilation_w, + group, + deformable_group, + im2col_step); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} + diff --git a/detection/mm_modules/DCN/src/modulated_deform_conv2d.h b/detection/mm_modules/DCN/src/modulated_deform_conv2d.h new file mode 100644 index 0000000..9c8043e --- /dev/null +++ b/detection/mm_modules/DCN/src/modulated_deform_conv2d.h @@ -0,0 +1,87 @@ +#pragma once + +#include "cpu/modulated_deform_conv2d_cpu.h" + +#ifdef WITH_CUDA +#include "cuda/modulated_deform_conv2d_cuda.h" +#endif + + +at::Tensor +modulated_deform_conv2d_forward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &mask, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + if (input.type().is_cuda()) + { +#ifdef WITH_CUDA + return modulated_deform_conv2d_cuda_forward(input, weight, bias, offset, mask, + kernel_h, kernel_w, + stride_h, stride_w, + pad_h, pad_w, + dilation_h, dilation_w, + group, + deformable_group, + im2col_step); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} + +std::vector +modulated_deform_conv2d_backward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &mask, + const at::Tensor &grad_output, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + if (input.type().is_cuda()) + { +#ifdef WITH_CUDA + return modulated_deform_conv2d_cuda_backward(input, + weight, + bias, + offset, + mask, + grad_output, + kernel_h, kernel_w, + stride_h, stride_w, + pad_h, pad_w, + dilation_h, dilation_w, + group, + deformable_group, + im2col_step); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} + diff --git a/detection/mm_modules/DCN/src/vision.cpp b/detection/mm_modules/DCN/src/vision.cpp new file mode 100644 index 0000000..5043fea --- /dev/null +++ b/detection/mm_modules/DCN/src/vision.cpp @@ -0,0 +1,10 @@ + +#include "deform_conv2d.h" +#include "modulated_deform_conv2d.h" + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("deform_conv2d_forward", &deform_conv2d_forward, "deform_conv2d_forward"); + m.def("deform_conv2d_backward", &deform_conv2d_backward, "deform_conv2d_backward"); + m.def("modulated_deform_conv2d_forward", &modulated_deform_conv2d_forward, "modulated_deform_conv2d_forward"); + m.def("modulated_deform_conv2d_backward", &modulated_deform_conv2d_backward, "modulated_deform_conv2d_backward"); +} diff --git a/detection/mm_modules/__init__.py b/detection/mm_modules/__init__.py new file mode 100644 index 0000000..faaaf79 --- /dev/null +++ b/detection/mm_modules/__init__.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + + diff --git a/detection/mm_modules/cocoapi_evaluator.py b/detection/mm_modules/cocoapi_evaluator.py new file mode 100644 index 0000000..0d47637 --- /dev/null +++ b/detection/mm_modules/cocoapi_evaluator.py @@ -0,0 +1,209 @@ +import json +import tempfile +import sys +from tqdm import tqdm + +from pycocotools.cocoeval import COCOeval +from torch.autograd import Variable + +from dataset.cocodataset import * +from dataset.data_augment import ValTransform +from utils.utils import * +from utils import distributed_util +from utils.vis_utils import make_vis, make_pred_vis +import time +import apex + +DEBUG =False + +def _accumulate_predictions_from_multiple_gpus(predictions_per_gpu): + all_predictions = distributed_util.scatter_gather(predictions_per_gpu) + if not distributed_util.is_main_process(): + return + # merge the list of dicts + predictions = [] + for p in all_predictions: + for a in p: + predictions.append(a) + + return predictions + +class COCOAPIEvaluator(): + """ + COCO AP Evaluation class. + All the data in the val2017 dataset are processed \ + and evaluated by COCO API. + """ + def __init__(self, data_dir, img_size, confthre, nmsthre, testset=False, voc=False, vis=False): + """ + Args: + data_dir (str): dataset root directory + img_size (int): image size after preprocess. images are resized \ + to squares whose shape is (img_size, img_size). + confthre (float): + confidence threshold ranging from 0 to 1, \ + which is defined in the config file. + nmsthre (float): + IoU threshold of non-max supression ranging from 0 to 1. + """ + json_f = 'instances_val2017.json' + name='val2017' + if testset: + json_f = 'image_info_test-dev2017.json' + name='test2017' + if voc: + json_f = 'pascal_test2007.json' + + self.testset= testset + self.dataset = COCODataset(data_dir=data_dir, + img_size=img_size, + json_file=json_f, + preproc = ValTransform(rgb_means=(0.485, 0.456, 0.406),std=(0.229, 0.224, 0.225)), + name=name, + voc = voc) + + self.num_images = len(self.dataset) + self.dataloader = torch.utils.data.DataLoader( + self.dataset, batch_size=1, shuffle=False, num_workers=0) + self.img_size = img_size + self.confthre = confthre + self.nmsthre = nmsthre + self.voc = voc + self.vis = vis + + def evaluate(self, model, half=False, distributed=False): + """ + COCO average precision (AP) Evaluation. Iterate inference on the test dataset + and the results are evaluated by COCO API. + Args: + model : model object + Returns: + ap50_95 (float) : calculated COCO AP for IoU=50:95 + ap50 (float) : calculated COCO AP for IoU=50 + """ + if isinstance(model, apex.parallel.DistributedDataParallel): + model = model.module + distributed=True + + model=model.eval() + cuda = torch.cuda.is_available() + if half: + Tensor = torch.cuda.HalfTensor if cuda else torch.HalfTensor + else: + Tensor = torch.cuda.FloatTensor if cuda else torch.FloatTensor + ids = [] + data_dict = [] + img_num = 0 + + indices = list(range(self.num_images)) + if distributed: + dis_indices = indices[distributed_util.get_rank()::distributed_util.get_world_size()] + else: + dis_indices = indices + progress_bar = tqdm if distributed_util.is_main_process() else iter + num_classes = 80 if not self.voc else 20 + + inference_time=0 + nms_time=0 + n_samples=len(dis_indices)-10 + + for k, i in enumerate(progress_bar(dis_indices)): + img, _, info_img, id_ = self.dataset[i] # load a batch + info_img = [float(info) for info in info_img] + id_ = int(id_) + ids.append(id_) + with torch.no_grad(): + img = Variable(img.type(Tensor).unsqueeze(0)) + if k > 9: + start=time.time() + + if self.vis: + outputs,fuse_weights,fused_f = model(img) + else: + outputs = model(img) + + if k > 9: + infer_end=time.time() + inference_time += (infer_end-start) + + outputs = postprocess( + outputs, num_classes, self.confthre, self.nmsthre) + + if k > 9: + nms_end=time.time() + nms_time +=(nms_end-infer_end) + + if outputs[0] is None: + continue + outputs = outputs[0].cpu().data + + bboxes = outputs[:, 0:4] + bboxes[:, 0::2] *= info_img[0] / self.img_size[0] + bboxes[:, 1::2] *= info_img[1] / self.img_size[1] + bboxes[:, 2] = bboxes[:,2] - bboxes[:,0] + bboxes[:, 3] = bboxes[:,3] - bboxes[:,1] + cls = outputs[:, 6] + scores = outputs[:, 4]* outputs[:,5] + for ind in range(bboxes.shape[0]): + label = self.dataset.class_ids[int(cls[ind])] + A = {"image_id": id_, "category_id": label, "bbox": bboxes[ind].numpy().tolist(), + "score": scores[ind].numpy().item(), "segmentation": []} # COCO json format + data_dict.append(A) + + if self.vis: + o_img,_,_,_ = self.dataset.pull_item(i) + make_vis('COCO', i, o_img, fuse_weights, fused_f) + class_names = self.dataset._classes + make_pred_vis('COCO', i, o_img, class_names, bboxes, cls, scores) + + if DEBUG and distributed_util.is_main_process(): + o_img,_ = self.dataset.pull_item(i) + class_names = self.dataset._classes + make_pred_vis('COCO', i, o_img, class_names, bboxes, cls, scores) + + if distributed: + distributed_util.synchronize() + data_dict = _accumulate_predictions_from_multiple_gpus(data_dict) + inference_time = torch.FloatTensor(1).type(Tensor).fill_(inference_time) + nms_time = torch.FloatTensor(1).type(Tensor).fill_(nms_time) + n_samples = torch.LongTensor(1).type(Tensor).fill_(n_samples) + distributed_util.synchronize() + torch.distributed.reduce(inference_time, dst=0) + torch.distributed.reduce(nms_time, dst=0) + torch.distributed.reduce(n_samples, dst=0) + inference_time = inference_time.item() + nms_time = nms_time.item() + n_samples = n_samples.item() + + if not distributed_util.is_main_process(): + return 0, 0 + + + print('Main process Evaluating...') + + annType = ['segm', 'bbox', 'keypoints'] + a_infer_time = 1000*inference_time / (n_samples) + a_nms_time= 1000*nms_time / (n_samples) + + print('Average forward time: %.2f ms, Average NMS time: %.2f ms, Average inference time: %.2f ms' %(a_infer_time, \ + a_nms_time, (a_infer_time+a_nms_time))) + + # Evaluate the Dt (detection) json comparing with the ground truth + if len(data_dict) > 0: + cocoGt = self.dataset.coco + # workaround: temporarily write data to json file because pycocotools can't process dict in py36. + if self.testset: + json.dump(data_dict, open('yolov3_2017.json', 'w')) + cocoDt = cocoGt.loadRes('yolov3_2017.json') + else: + _, tmp = tempfile.mkstemp() + json.dump(data_dict, open(tmp, 'w')) + cocoDt = cocoGt.loadRes(tmp) + cocoEval = COCOeval(self.dataset.coco, cocoDt, annType[1]) + cocoEval.evaluate() + cocoEval.accumulate() + cocoEval.summarize() + return cocoEval.stats[0], cocoEval.stats[1] + else: + return 0, 0 + diff --git a/detection/mm_modules/distributed_util.py b/detection/mm_modules/distributed_util.py new file mode 100644 index 0000000..dcd2479 --- /dev/null +++ b/detection/mm_modules/distributed_util.py @@ -0,0 +1,162 @@ +import os +import pickle +import tempfile +import time + +import torch + + +def get_world_size(): + if not torch.distributed.is_initialized(): + return 1 + return torch.distributed.get_world_size() + + +def get_rank(): + if not torch.distributed.is_initialized(): + return 0 + return torch.distributed.get_rank() + + +def is_main_process(): + if not torch.distributed.is_initialized(): + return True + return torch.distributed.get_rank() == 0 + + +def synchronize(): + """ + Helper function to synchronize between multiple processes when + using distributed training + """ + if not torch.distributed.is_initialized(): + return + world_size = torch.distributed.get_world_size() + rank = torch.distributed.get_rank() + if world_size == 1: + return + + def _send_and_wait(r): + if rank == r: + tensor = torch.tensor(0, device="cuda") + else: + tensor = torch.tensor(1, device="cuda") + torch.distributed.broadcast(tensor, r) + while tensor.item() == 1: + time.sleep(1) + + _send_and_wait(0) + # now sync on the main process + _send_and_wait(1) + + +def _encode(encoded_data, data): + # gets a byte representation for the data + encoded_bytes = pickle.dumps(data) + # convert this byte string into a byte tensor + storage = torch.ByteStorage.from_buffer(encoded_bytes) + tensor = torch.ByteTensor(storage).to("cuda") + # encoding: first byte is the size and then rest is the data + s = tensor.numel() + assert s <= 255, "Can't encode data greater than 255 bytes" + # put the encoded data in encoded_data + encoded_data[0] = s + encoded_data[1: (s + 1)] = tensor + + +def _decode(encoded_data): + size = encoded_data[0] + encoded_tensor = encoded_data[1: (size + 1)].to("cpu") + return pickle.loads(bytearray(encoded_tensor.tolist())) + + +# TODO try to use tensor in shared-memory instead of serializing to disk +# this involves getting the all_gather to work +def scatter_gather(data): + """ + This function gathers data from multiple processes, and returns them + in a list, as they were obtained from each process. + This function is useful for retrieving data from multiple processes, + when launching the code with torch.distributed.launch + Note: this function is slow and should not be used in tight loops, i.e., + do not use it in the training loop. + Arguments: + data: the object to be gathered from multiple processes. + It must be serializable + Returns: + result (list): a list with as many elements as there are processes, + where each element i in the list corresponds to the data that was + gathered from the process of rank i. + """ + # strategy: the main process creates a temporary directory, and communicates + # the location of the temporary directory to all other processes. + # each process will then serialize the data to the folder defined by + # the main process, and then the main process reads all of the serialized + # files and returns them in a list + if not torch.distributed.is_initialized(): + return [data] + synchronize() + # get rank of the current process + rank = torch.distributed.get_rank() + + # the data to communicate should be small + data_to_communicate = torch.empty(256, dtype=torch.uint8, device="cuda") + if rank == 0: + # manually creates a temporary directory, that needs to be cleaned + # afterwards + tmp_dir = tempfile.mkdtemp() + _encode(data_to_communicate, tmp_dir) + + synchronize() + # the main process (rank=0) communicates the data to all processes + torch.distributed.broadcast(data_to_communicate, 0) + + # get the data that was communicated + tmp_dir = _decode(data_to_communicate) + + # each process serializes to a different file + file_template = "file{}.pth" + tmp_file = os.path.join(tmp_dir, file_template.format(rank)) + torch.save(data, tmp_file) + + # synchronize before loading the data + synchronize() + + # only the master process returns the data + if rank == 0: + data_list = [] + world_size = torch.distributed.get_world_size() + for r in range(world_size): + file_path = os.path.join(tmp_dir, file_template.format(r)) + d = torch.load(file_path) + data_list.append(d) + # cleanup + os.remove(file_path) + # cleanup + os.rmdir(tmp_dir) + return data_list + + +def reduce_loss_dict(loss_dict): + """ + Reduce the loss dictionary from all processes so that process with rank + 0 has the averaged results. Returns a dict with the same fields as + loss_dict, after reduction. + """ + world_size = get_world_size() + if world_size < 2: + return loss_dict + with torch.no_grad(): + loss_names = [] + all_losses = [] + for k in sorted(loss_dict.keys()): + loss_names.append(k) + all_losses.append(loss_dict[k]) + all_losses = torch.stack(all_losses, dim=0) + torch.distributed.reduce(all_losses, dst=0) + if torch.distributed.get_rank() == 0: + # only main process gets accumulated, so only divide by + # world_size in this case + all_losses /= world_size + reduced_losses = {k: v for k, v in zip(loss_names, all_losses)} + return reduced_losses diff --git a/detection/mm_modules/fp16_utils/README.md b/detection/mm_modules/fp16_utils/README.md new file mode 100644 index 0000000..941de17 --- /dev/null +++ b/detection/mm_modules/fp16_utils/README.md @@ -0,0 +1,16 @@ +fp16_optimizer.py contains `FP16_Optimizer`, a Python class designed to wrap an existing Pytorch optimizer and automatically enable master parameters and loss scaling in a manner transparent to the user. To use `FP16_Optimizer`, only two lines of one's Python model need to change. + +#### [FP16_Optimizer API documentation](https://nvidia.github.io/apex/fp16_utils.html#automatic-management-of-master-params-loss-scaling) + +#### [Simple examples with FP16_Optimizer](https://github.com/NVIDIA/apex/tree/master/examples/FP16_Optimizer_simple) + +#### [Imagenet with FP16_Optimizer](https://github.com/NVIDIA/apex/tree/master/examples/imagenet) + +#### [word_language_model with FP16_Optimizer](https://github.com/NVIDIA/apex/tree/master/examples/word_language_model) + + +fp16_util.py contains a number of utilities to manually manage master parameters and loss scaling, if the user chooses. + +#### [Manual management documentation](https://nvidia.github.io/apex/fp16_utils.html#manual-master-parameter-management) + +The [Imagenet with FP16_Optimizer](https://github.com/NVIDIA/apex/tree/master/examples/imagenet) and [word_language_model with FP16_Optimizer](https://github.com/NVIDIA/apex/tree/master/examples/word_language_model) directories also contain `main.py` files that demonstrate manual management of master parameters and static loss scaling. These examples illustrate what sort of operations `FP16_Optimizer` is performing automatically. diff --git a/detection/mm_modules/fp16_utils/__init__.py b/detection/mm_modules/fp16_utils/__init__.py new file mode 100644 index 0000000..c7bb1f5 --- /dev/null +++ b/detection/mm_modules/fp16_utils/__init__.py @@ -0,0 +1,16 @@ +from .fp16util import ( + BN_convert_float, + network_to_half, + prep_param_lists, + model_grads_to_master_grads, + master_params_to_model_params, + tofp16, + to_python_float, + clip_grad_norm, + convert_module, + convert_network, + FP16Model, +) + +from .fp16_optimizer import FP16_Optimizer +from .loss_scaler import LossScaler, DynamicLossScaler diff --git a/detection/mm_modules/fp16_utils/fp16_optimizer.py b/detection/mm_modules/fp16_utils/fp16_optimizer.py new file mode 100644 index 0000000..fe999e0 --- /dev/null +++ b/detection/mm_modules/fp16_utils/fp16_optimizer.py @@ -0,0 +1,561 @@ +import torch +from torch import nn +from torch.autograd import Variable +from torch.nn.parameter import Parameter +from torch._utils import _flatten_dense_tensors, _unflatten_dense_tensors + +from .loss_scaler import DynamicLossScaler, LossScaler +from .fp16util import model_grads_to_master_grads, master_params_to_model_params, clip_grad_norm + +# TODO: Update overflow check + downscale to use Carl's fused kernel. +class FP16_Optimizer(object): + """ + :class:`FP16_Optimizer` is designed to wrap an existing PyTorch optimizer, + and manage static or dynamic loss scaling and master weights in a manner transparent to the user. + For standard use, only two lines must be changed: creating the :class:`FP16_Optimizer` instance, + and changing the call to ``backward``. + + Example:: + + model = torch.nn.Linear(D_in, D_out).cuda().half() + optimizer = torch.optim.SGD(model.parameters(), lr=1e-3) + # Name the FP16_Optimizer instance to replace the existing optimizer + # (recommended but not required): + optimizer = FP16_Optimizer(optimizer, static_loss_scale = 128.0) + ... + # loss.backward() becomes: + optimizer.backward(loss) + ... + + Example with dynamic loss scaling:: + + ... + optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True) + # optional arg to control dynamic loss scaling behavior + # dynamic_loss_args={'scale_window' : 500}) + # Usually, dynamic_loss_args is not necessary. + + Args: + init_optimizer (torch.optim.optimizer): Existing optimizer created with the parameters to optimize. Internally, :class:`FP16_Optimizer` replaces the passed optimizer's fp16 parameters, if any, with fp32 master parameters copied from the original ones. :class:`FP16_Optimizer` also stores references to the original fp16 parameters, and updates these fp16 parameters from the master fp32 copy at the end of each :attr:`step`. + static_loss_scale (float, optional, default=1.0): Loss scale used internally to scale gradients computed by the model. Any fp16 gradients will be copied to fp32, then downscaled before being applied to the fp32 master params, so ``static_loss_scale`` should not affect learning rate. + dynamic_loss_scale (bool, optional, default=False): Use dynamic loss scaling. If True, this will override any ``static_loss_scale`` option. + dynamic_loss_args (dict, optional, default=None): Dict of kwargs that will be forwarded to the internal :class:`DynamicLossScaler` instance's constructor. Keys of this dict must match kwargs accepted by :class:`DynamicLossScaler`'s constructor. If ``dynamic_loss_args`` is unspecified, :class:`DynamicLossScaler`'s defaults will be used. + verbose (bool, optional, default=True): By default, FP16_Optimizer's constructor prints out the parameters and parameter groups it is ingesting, as a sanity check. If this becomes annoying (e.g. for large models), it can be disabled by passing ``verbose=False``. ``verbose=False`` will not disable printing when the loss scale is readjusted during dynamic loss scaling. + + ``init_optimizer`` is expected to have been constructed in the ordinary way. + It is recommended (although not required) that the newly constructed :class:`FP16_Optimizer` instance be + named to replace ``init_optimizer``, for two reasons: + First, it means that references to the same name + later in the file will not have to change. + Second, :class:`FP16_Optimizer` reserves the right (as an implementation detail) to + modify ``init_optimizer``. If you do choose a unique name for the new + :class:`FP16_Optimizer` instance, you should only work with this new instance, + because the preexisting optimizer might no longer behave as expected. + + ``init_optimizer`` may be any Pytorch optimizer. + It may contain a mixture of fp16 and fp32 parameters organized into any number of + ``param_groups`` with different hyperparameters. The :class:`FP16_Optimizer` constructor will + ingest these ``param_groups`` and remember them. + + Calls to :: + + loss.backward() + + must be replaced with :: + + optimizer.backward(loss) + + because :class:`FP16_Optimizer` requires ownership of the backward pass to implement + loss scaling and copies to master gradients. + + .. note:: + Loss scaling, either static or dynamic, is orthogonal to learning rate, because gradients + are downscaled before being applied. This means that adjusting the loss scale, or using + dynamic loss scaling, should not require retuning the learning rate or any other + hyperparameters. + + + **Advanced options** + + **Closures**: :class:`FP16_Optimizer` can wrap a Pytorch optimizer that receives a closure. + See docstring for :attr:`step`. + + **Gradient clipping**: Use :attr:`clip_master_grads`. + + **Multiple losses**: If your model accumulates gradients from multiple losses, + this can be made more efficient by supplying ``update_master_grads=False`` + to :attr:`backward`. See docstring for :attr:`backward`. + + **Manually adjusting loss scale**: The current loss scale can be retrieved or set via :: + + print(optimizer.loss_scale) + optimizer.loss_scale = new_loss_scale + + For static loss scaling, manually adjusting the loss scale over time is a reasonable + thing to do. During later epochs, gradients may become smaller, and a + higher loss scale may be required, analogous to scheduling the learning rate. Dynamic loss + scaling is more subtle (see :class:`DynamicLossScaler`) and in this case, manually adjusting + the loss scale is not recommended. + + **Multi_GPU training**: If the wrapped ``init_optimizer`` was created from a model wrapped in + Pytorch DistributedDataParallel or Apex DistributedDataParallel, :class:`FP16_Optimizer` + should still work as intended. + """ + + def __init__(self, + init_optimizer, + static_loss_scale=1.0, + dynamic_loss_scale=False, + dynamic_loss_args=None, + verbose=True): + if not torch.cuda.is_available: + raise SystemError("Cannot use fp16 without CUDA.") + + self.verbose = verbose + + self.optimizer = init_optimizer + # init_state_dict sets up an alternative way to cast per-param state tensors. + # Stashing here in case https://github.com/pytorch/pytorch/issues/7733 makes it necessary. + # init_state_dict = init_optimizer.state_dict() + + self.fp16_groups = [] + self.fp32_from_fp16_groups = [] + self.fp32_from_fp32_groups = [] + for i, param_group in enumerate(self.optimizer.param_groups): + self.maybe_print("FP16_Optimizer processing param group {}:".format(i)) + fp16_params_this_group = [] + fp32_params_this_group = [] + fp32_from_fp16_params_this_group = [] + for i, param in enumerate(param_group['params']): + if param.requires_grad: + if param.type() == 'torch.cuda.HalfTensor': + self.maybe_print("FP16_Optimizer received torch.cuda.HalfTensor with {}" + .format(param.size())) + fp16_params_this_group.append(param) + master_param = param.detach().clone().float() + master_param.requires_grad = True + param_group['params'][i] = master_param + fp32_from_fp16_params_this_group.append(master_param) + # Reset existing state dict key to the new master param. + # We still need to recast per-param state tensors, if any, to FP32. + if param in self.optimizer.state: + self.optimizer.state[master_param] = self.optimizer.state.pop(param) + elif param.type() == 'torch.cuda.FloatTensor': + self.maybe_print("FP16_Optimizer received torch.cuda.FloatTensor with {}" + .format(param.size())) + fp32_params_this_group.append(param) + param_group['params'][i] = param + else: + raise TypeError("Wrapped parameters must be either " + "torch.cuda.FloatTensor or torch.cuda.HalfTensor. " + "Received {}".format(param.type())) + + self.fp16_groups.append(fp16_params_this_group) + self.fp32_from_fp16_groups.append(fp32_from_fp16_params_this_group) + self.fp32_from_fp32_groups.append(fp32_params_this_group) + + # Leverage state_dict() and load_state_dict() to recast preexisting per-param state tensors + self.optimizer.load_state_dict(self.optimizer.state_dict()) + # alternative way to cast per-param state tensors: + # self.optimizer.load_state_dict(init_state_dict) + + if dynamic_loss_scale: + self.dynamic_loss_scale = True + if dynamic_loss_args is not None: + self.loss_scaler = DynamicLossScaler(**dynamic_loss_args) + else: + self.loss_scaler = DynamicLossScaler() + else: + self.dynamic_loss_scale = False + self.loss_scaler = LossScaler(static_loss_scale) + + self.overflow = False + self.first_closure_call_this_step = True + + self.clip_grad_norm = clip_grad_norm + + def maybe_print(self, msg): + if self.verbose: + print(msg) + + def __getstate__(self): + raise RuntimeError("FP16_Optimizer should be serialized using state_dict().") + + def __setstate__(self, state): + raise RuntimeError("FP16_Optimizer should be deserialized using load_state_dict().") + + def zero_grad(self, set_grads_to_None=False): + """ + Zero fp32 and fp16 parameter grads. + """ + # In principle, only the .grad attributes of the model params need to be zeroed, + # because gradients are copied into the FP32 master params. However, we zero + # all gradients owned by the optimizer, just to be safe: + for group in self.optimizer.param_groups: + for p in group['params']: + if set_grads_to_None: + p.grad = None + else: + if p.grad is not None: + p.grad.detach_() + p.grad.zero_() + + # Zero fp16 gradients owned by the model: + for fp16_group in self.fp16_groups: + for param in fp16_group: + if set_grads_to_None: + param.grad = None + else: + if param.grad is not None: + param.grad.detach_() # as in torch.optim.optimizer.zero_grad() + param.grad.zero_() + + def _check_overflow(self): + params = [] + for group in self.fp16_groups: + for param in group: + params.append(param) + for group in self.fp32_from_fp32_groups: + for param in group: + params.append(param) + self.overflow = self.loss_scaler.has_overflow(params) + + def _update_scale(self, has_overflow=False): + self.loss_scaler.update_scale(has_overflow) + + def _master_params_to_model_params(self): + for fp16_group, fp32_from_fp16_group in zip(self.fp16_groups, self.fp32_from_fp16_groups): + master_params_to_model_params(fp16_group, fp32_from_fp16_group) + + # To consider: Integrate distributed with this wrapper by registering a hook on each variable + # that does the overflow check, gradient copy + downscale, and fp32 allreduce in a different stream. + def _model_grads_to_master_grads(self): + for fp16_group, fp32_from_fp16_group in zip(self.fp16_groups, self.fp32_from_fp16_groups): + model_grads_to_master_grads(fp16_group, fp32_from_fp16_group) + + def _downscale_master(self): + if self.loss_scale != 1.0: + for group in self.optimizer.param_groups: + for param in group['params']: + if param.grad is not None: + param.grad.data.mul_(1./self.loss_scale) + + def clip_master_grads(self, max_norm, norm_type=2): + """ + Clips fp32 master gradients via ``torch.nn.utils.clip_grad_norm``. + + Args: + max_norm (float or int): max norm of the gradients + norm_type (float or int): type of the used p-norm. Can be ``'inf'`` for + infinity norm. + + Returns: + Total norm of the current fp32 gradients (viewed as a single vector). + + .. warning:: + Returns -1 if the most recently computed fp16 gradients overflowed (that is, if ``self.overflow`` is ``True``). + """ + if not self.overflow: + fp32_params = [] + for param_group in self.optimizer.param_groups: + for param in param_group['params']: + fp32_params.append(param) + return self.clip_grad_norm(fp32_params, max_norm, norm_type) + else: + return -1 + + def state_dict(self): + """ + Returns a dict containing the current state of this :class:`FP16_Optimizer` instance. + This dict contains attributes of :class:`FP16_Optimizer`, as well as the state_dict + of the contained Pytorch optimizer. + Example:: + + checkpoint = {} + checkpoint['model'] = model.state_dict() + checkpoint['optimizer'] = optimizer.state_dict() + torch.save(checkpoint, "saved.pth") + """ + state_dict = {} + state_dict['loss_scaler'] = self.loss_scaler + state_dict['dynamic_loss_scale'] = self.dynamic_loss_scale + state_dict['overflow'] = self.overflow + state_dict['first_closure_call_this_step'] = self.first_closure_call_this_step + state_dict['optimizer_state_dict'] = self.optimizer.state_dict() + state_dict['fp32_from_fp16'] = self.fp32_from_fp16_groups + return state_dict + + def load_state_dict(self, state_dict): + """ + Loads a state_dict created by an earlier call to state_dict(). + If ``fp16_optimizer_instance`` was constructed from some ``init_optimizer``, + whose parameters in turn came from ``model``, it is expected that the user + will call ``model.load_state_dict()`` before + ``fp16_optimizer_instance.load_state_dict()`` is called. + + Example:: + + model = torch.nn.Linear(D_in, D_out).cuda().half() + optimizer = torch.optim.SGD(model.parameters(), lr=1e-3) + optimizer = FP16_Optimizer(optimizer, static_loss_scale = 128.0) + ... + checkpoint = torch.load("saved.pth") + model.load_state_dict(checkpoint['model']) + optimizer.load_state_dict(checkpoint['optimizer']) + """ + # I think it should actually be ok to reload the optimizer before the model. + self.loss_scaler = state_dict['loss_scaler'] + self.dynamic_loss_scale = state_dict['dynamic_loss_scale'] + self.overflow = state_dict['overflow'] + self.first_closure_call_this_step = state_dict['first_closure_call_this_step'] + self.optimizer.load_state_dict(state_dict['optimizer_state_dict']) + # At this point, the optimizer's references to the model's fp32 parameters are up to date. + # The optimizer's hyperparameters and internal buffers are also up to date. + # However, the fp32 master copies of the model's fp16 params stored by the optimizer are still + # out of date. There are two options. + # 1: Refresh the master params from the model's fp16 params. + # This requires less storage but incurs precision loss. + # 2: Save and restore the fp32 master copies separately. + # We choose option 2. + # + # Pytorch Optimizer.load_state_dict casts saved buffers (e.g. momentum) to the type and device + # of their associated parameters, because it's possible those buffers might not exist yet in + # the current optimizer instance. In our case, as long as the current FP16_Optimizer has been + # constructed in the same way as the one whose state_dict we are loading, the same master params + # are guaranteed to exist, so we can just copy_() from the saved master params. + for current_group, saved_group in zip(self.fp32_from_fp16_groups, state_dict['fp32_from_fp16']): + for current, saved in zip(current_group, saved_group): + current.data.copy_(saved.data) + + def step(self, closure=None): # could add clip option. + """ + If no closure is supplied, :attr:`step` should be called after + ``fp16_optimizer_obj.backward(loss)``. + :attr:`step` updates the fp32 master copy of parameters using the optimizer supplied to + :class:`FP16_Optimizer`'s constructor, then copies the updated fp32 params into the fp16 params + originally referenced by :class:`FP16_Optimizer`'s constructor, so the user may immediately run + another forward pass using their model. + + If a closure is supplied, :attr:`step` may be called without a prior call to + :attr:`backward(loss)`. + This control flow is identical to `ordinary Pytorch optimizer use`_ with closures. + However, the user should take care that any ``loss.backward()`` call within the closure + has been replaced by ``fp16_optimizer_obj.backward(loss)``. + + Args: + closure (optional): Closure that will be supplied to the underlying optimizer originally passed to :class:`FP16_Optimizer`'s constructor. closure should call :attr:`zero_grad()` on the :class:`FP16_Optimizer` object, compute the loss, call :attr:`backward(loss)`, and return the loss. + + Example with closure:: + + # optimizer is assumed to be an FP16_Optimizer object, previously constructed from an + # existing pytorch optimizer. + for input, target in dataset: + def closure(): + optimizer.zero_grad() + output = model(input) + loss = loss_fn(output, target) + # loss.backward() becomes: + optimizer.backward(loss) + return loss + optimizer.step(closure) + + .. warning:: + Currently, calling :attr:`step` with a closure is not compatible with dynamic loss scaling. + + .. _`ordinary Pytorch optimizer use`: + http://pytorch.org/docs/master/optim.html#optimizer-step-closure + """ + + scale = self.loss_scaler.loss_scale + self._update_scale(self.overflow) + + if self.overflow: + print("OVERFLOW! Skipping step. Attempted loss scale: {}, reducing to {}" + .format(scale, self.loss_scale)) + return + + if closure is not None: + retval = self._step_with_closure(closure) + else: + retval = self.optimizer.step() + + self._master_params_to_model_params() + + return retval + + def _step_with_closure(self, closure): + def wrapped_closure(): + # helpful for debugging + # print("Calling wrapped_closure, first_closure_call_this_step = {}" + # .format(self.first_closure_call_this_step)) + if self.first_closure_call_this_step: + # We expect that the fp16 params are initially fresh on entering self.step(), + # so _master_params_to_model_params() is unnecessary the first time wrapped_closure() + # is called within self.optimizer.step(). + self.first_closure_call_this_step = False + else: + # If self.optimizer.step() internally calls wrapped_closure more than once, + # it may update the fp32 params after each call. However, self.optimizer + # doesn't know about the fp16 params at all. If the fp32 params get updated, + # we can't rely on self.optimizer to refresh the fp16 params. We need + # to handle that manually: + self._master_params_to_model_params() + # Our API expects the user to give us ownership of the backward() call by + # replacing all calls to loss.backward() with optimizer.backward(loss). + # This requirement holds whether or not the call to backward() is made within a closure. + # If the user is properly calling optimizer.backward(loss) within "closure," + # calling closure() here will give the fp32 master params fresh gradients + # for the optimizer to play with, so all wrapped_closure needs to do is call + # closure() and return the loss. + temp_loss = closure() + while(self.overflow): + scale = self.loss_scaler.loss_scale + self._update_scale(self.overflow) + print("OVERFLOW within closure! Skipping step. Attempted loss scale: {}, " + "reducing to {}".format(scale, self.loss_scale)) + temp_loss = closure() + return temp_loss + + retval = self.optimizer.step(wrapped_closure) + + self.first_closure_call_this_step = True + + return retval + + def backward(self, loss, update_master_grads=True, retain_graph=False): + """ + :attr:`backward` performs the following conceptual steps: + + 1. fp32_loss = loss.float() (see first Note below) + 2. scaled_loss = fp32_loss*loss_scale + 3. scaled_loss.backward(), which accumulates scaled gradients into the ``.grad`` attributes of the model's leaves (which may be fp16, fp32, or a mixture, depending how your model was defined). + 4. fp16 grads are then copied to the master params' ``.grad`` attributes (see second Note), which are guaranteed to be fp32. + 5. Finally, master grads are divided by loss_scale. + + In this way, after :attr:`backward`, the master params have fresh gradients, + and :attr:`step` may be called. + + .. note:: + :attr:`backward` internally converts the loss to fp32 before applying the loss scale. + This provides some additional safety against overflow if the user has supplied an + fp16 loss value. + However, for maximum overflow safety, the user should + compute the loss criterion (MSE, cross entropy, etc) in fp32 before supplying it to + :attr:`backward`. + + .. warning:: + The gradients found in a model's leaves after the call to + :attr:`backward` should not be regarded as valid in general, + because it's possible + they have been scaled (and in the case of dynamic loss scaling, + the scale factor may change over time). + If the user wants to inspect gradients after a call to :attr:`backward`, + only the master gradients should be regarded as valid. These can be retrieved via + :attr:`inspect_master_grad_data()`. + + Args: + loss: The loss output by the user's model. loss may be either float or half (but see first Note above). + update_master_grads (bool, optional, default=True): Option to copy fp16 grads to fp32 grads on this call. By setting this to False, the user can delay the copy, which is useful to eliminate redundant fp16->fp32 grad copies if :attr:`backward` is being called on multiple losses in one iteration. If set to False, the user becomes responsible for calling :attr:`update_master_grads` before calling :attr:`step`. + retain_graph (bool, optional, default=False): Forwards the usual ``retain_graph=True`` option to the internal call to ``loss.backward``. If ``retain_graph`` is being used to accumulate gradient values from multiple backward passes before calling ``optimizer.step``, passing ``update_master_grads=False`` is also recommended (see Example below). + + Example:: + + # Ordinary operation: + optimizer.backward(loss) + + # Naive operation with multiple losses (technically valid, but less efficient): + # fp32 grads will be correct after the second call, but + # the first call incurs an unnecessary fp16->fp32 grad copy. + optimizer.backward(loss1) + optimizer.backward(loss2) + + # More efficient way to handle multiple losses: + # The fp16->fp32 grad copy is delayed until fp16 grads from all + # losses have been accumulated. + optimizer.backward(loss1, update_master_grads=False) + optimizer.backward(loss2, update_master_grads=False) + optimizer.update_master_grads() + """ + # To consider: try multiple backward passes using retain_grad=True to find + # a loss scale that works. After you find a loss scale that works, do a final dummy + # backward pass with retain_graph=False to tear down the graph. Doing this would avoid + # discarding the iteration, but probably wouldn't improve overall efficiency. + self.loss_scaler.backward(loss.float(), retain_graph=retain_graph) + if update_master_grads: + self.update_master_grads() + + def update_master_grads(self): + """ + Copy the ``.grad`` attribute from stored references to fp16 parameters to + the ``.grad`` attribute of the fp32 master parameters that are directly + updated by the optimizer. :attr:`update_master_grads` only needs to be called if + ``fp16_optimizer_obj.backward`` was called with ``update_master_grads=False``. + """ + if self.dynamic_loss_scale: + self._check_overflow() + if self.overflow: return + self._model_grads_to_master_grads() + self._downscale_master() + + def inspect_master_grad_data(self): + """ + When running with :class:`FP16_Optimizer`, + ``.grad`` attributes of a model's fp16 leaves should not be + regarded as truthful, because they might be scaled. + After a call to :attr:`fp16_optimizer_obj.backward(loss)`, if no overflow was encountered, + the fp32 master params' ``.grad`` + attributes will contain valid gradients properly divided by the loss scale. However, + because :class:`FP16_Optimizer` flattens some parameters, accessing them may be + nonintuitive. :attr:`inspect_master_grad_data` + allows those gradients to be viewed with shapes corresponding to their associated model leaves. + + Returns: + List of lists (one list for each parameter group). The list for each parameter group + is a list of the ``.grad.data`` attributes of the fp32 master params belonging to that group. + """ + if self.overflow: + print("Warning: calling FP16_Optimizer.inspect_master_grad_data while in an overflow state. " + "Gradients are currently invalid (may be inf, nan, or stale). Returning None.") + return None + else: + # The optimizer owns only references to master params. + master_grads_data = [] + for param_group in self.optimizer.param_groups: + master_grads_this_group = [] + for param in param_group['params']: + if param.grad is not None: + master_grads_this_group.append(param.grad.data) + else: + master_grads_this_group.append(None) + master_grads_data.append(master_grads_this_group) + return master_grads_data + + + # Promote loss scale so it can be retrieved or set via "fp16_optimizer_instance.loss_scale" + def _get_loss_scale(self): + return self.loss_scaler.loss_scale + + def _set_loss_scale(self, value): + self.loss_scaler.cur_scale = value + + loss_scale = property(_get_loss_scale, _set_loss_scale) + + # Promote state so it can be retrieved or set via "fp16_optimizer_instance.state" + def _get_state(self): + return self.optimizer.state + + def _set_state(self, value): + self.optimizer.state = value + + state = property(_get_state, _set_state) + + # Promote param_groups so it can be retrieved or set via "fp16_optimizer_instance.param_groups" + # (for example, to adjust the learning rate) + def _get_param_groups(self): + return self.optimizer.param_groups + + def _set_param_groups(self, value): + self.optimizer.param_groups = value + + param_groups = property(_get_param_groups, _set_param_groups) + diff --git a/detection/mm_modules/fp16_utils/fp16util.py b/detection/mm_modules/fp16_utils/fp16util.py new file mode 100644 index 0000000..66c13e4 --- /dev/null +++ b/detection/mm_modules/fp16_utils/fp16util.py @@ -0,0 +1,185 @@ +import torch +import torch.nn as nn +from torch.autograd import Variable +from torch._utils import _flatten_dense_tensors, _unflatten_dense_tensors + + +class tofp16(nn.Module): + """ + Utility module that implements:: + + def forward(self, input): + return input.half() + """ + + def __init__(self): + super(tofp16, self).__init__() + + def forward(self, input): + return input.half() + + +def BN_convert_float(module): + """ + Utility function for network_to_half(). + + Retained for legacy purposes. + """ + if isinstance(module, torch.nn.modules.batchnorm._BatchNorm) and module.affine is True: + module.float() + for child in module.children(): + BN_convert_float(child) + return module + + +def network_to_half(network): + """ + Convert model to half precision in a batchnorm-safe way. + + Retained for legacy purposes. It is recommended to use FP16Model. + """ + return nn.Sequential(tofp16(), BN_convert_float(network.half())) + + +def convert_module(module, dtype): + """ + Converts a module's immediate parameters and buffers to dtype. + """ + for param in module.parameters(recurse=False): + if param is not None: + if param.data.dtype.is_floating_point: + param.data = param.data.to(dtype=dtype) + if param._grad is not None and param._grad.data.dtype.is_floating_point: + param._grad.data = param._grad.data.to(dtype=dtype) + + for buf in module.buffers(recurse=False): + if buf is not None and buf.data.dtype.is_floating_point: + buf.data = buf.data.to(dtype=dtype) + + +def convert_network(network, dtype): + """ + Converts a network's parameters and buffers to dtype. + """ + for module in network.modules(): + if isinstance(module, torch.nn.modules.batchnorm._BatchNorm) and module.affine is True: + continue + convert_module(module, dtype) + return network + + +class FP16Model(nn.Module): + """ + Convert model to half precision in a batchnorm-safe way. + """ + + def __init__(self, network): + super(FP16Model, self).__init__() + self.network = convert_network(network, dtype=torch.half) + + def forward(self, *inputs): + inputs = tuple(t.half() for t in inputs) + return self.network(*inputs) + + +def backwards_debug_hook(grad): + raise RuntimeError("master_params recieved a gradient in the backward pass!") + +def prep_param_lists(model, flat_master=False): + """ + Creates a list of FP32 master parameters for a given model, as in + `Training Neural Networks with Mixed Precision: Real Examples`_. + + Args: + model (torch.nn.Module): Existing Pytorch model + flat_master (bool, optional, default=False): Flatten the master parameters into a single tensor, as a performance optimization. + Returns: + A tuple (``model_params``, ``master_params``). ``model_params`` is a list of the model's parameters for later use with :func:`model_grads_to_master_grads` and :func:`master_params_to_model_params`. ``master_params`` is a list of FP32 master gradients. If ``flat_master=True``, ``master_params`` will be a list with one element. + + Example:: + + model_params, master_params = prep_param_lists(model) + + .. warning:: + Currently, if ``flat_master=True``, all the model's parameters must be the same type. If the model has parameters of different types, use ``flat_master=False``, or use :class:`FP16_Optimizer`. + + .. _`Training Neural Networks with Mixed Precision: Real Examples`: + http://on-demand.gputechconf.com/gtc/2018/video/S81012/ + """ + model_params = [param for param in model.parameters() if param.requires_grad] + + if flat_master: + # Give the user some more useful error messages + try: + # flatten_dense_tensors returns a contiguous flat array. + # http://pytorch.org/docs/master/_modules/torch/_utils.html + master_params = _flatten_dense_tensors([param.data for param in model_params]).float() + except: + print("Error in prep_param_lists: model may contain a mixture of parameters " + "of different types. Use flat_master=False, or use F16_Optimizer.") + raise + master_params = torch.nn.Parameter(master_params) + master_params.requires_grad = True + # master_params.register_hook(backwards_debug_hook) + if master_params.grad is None: + master_params.grad = master_params.new(*master_params.size()) + return model_params, [master_params] + else: + master_params = [param.clone().float().detach() for param in model_params] + for param in master_params: + param.requires_grad = True + return model_params, master_params + + +def model_grads_to_master_grads(model_params, master_params, flat_master=False): + """ + Copy model gradients to master gradients. + + Args: + model_params: List of model parameters created by :func:`prep_param_lists`. + master_params: List of FP32 master parameters created by :func:`prep_param_lists`. If ``master_params`` was created with ``flat_master=True``, ``flat_master=True`` should also be supplied to :func:`model_grads_to_master_grads`. + """ + if flat_master: + # The flattening may incur one more deep copy than is necessary. + master_params[0].grad.data.copy_( + _flatten_dense_tensors([p.grad.data for p in model_params])) + else: + for model, master in zip(model_params, master_params): + if model.grad is not None: + if master.grad is None: + master.grad = Variable(master.data.new(*master.data.size())) + master.grad.data.copy_(model.grad.data) + else: + master.grad = None + + +def master_params_to_model_params(model_params, master_params, flat_master=False): + """ + Copy master parameters to model parameters. + + Args: + model_params: List of model parameters created by :func:`prep_param_lists`. + master_params: List of FP32 master parameters created by :func:`prep_param_lists`. If ``master_params`` was created with ``flat_master=True``, ``flat_master=True`` should also be supplied to :func:`master_params_to_model_params`. + """ + if flat_master: + for model, master in zip(model_params, + _unflatten_dense_tensors(master_params[0].data, model_params)): + model.data.copy_(master) + else: + for model, master in zip(model_params, master_params): + model.data.copy_(master.data) + +# Backward compatibility fixes + +def to_python_float(t): + if hasattr(t, 'item'): + return t.item() + else: + return t[0] + +TORCH_MAJOR = int(torch.__version__.split('.')[0]) +TORCH_MINOR = int(torch.__version__.split('.')[1]) +if TORCH_MAJOR == 0 and TORCH_MINOR <= 4: + clip_grad_norm = torch.nn.utils.clip_grad_norm +else: + clip_grad_norm = torch.nn.utils.clip_grad_norm_ diff --git a/detection/mm_modules/fp16_utils/loss_scaler.py b/detection/mm_modules/fp16_utils/loss_scaler.py new file mode 100644 index 0000000..b9f32fe --- /dev/null +++ b/detection/mm_modules/fp16_utils/loss_scaler.py @@ -0,0 +1,186 @@ +import torch + +# item() is a recent addition, so this helps with backward compatibility. +def to_python_float(t): + if hasattr(t, 'item'): + return t.item() + else: + return t[0] + +class LossScaler: + """ + Class that manages a static loss scale. This class is intended to interact with + :class:`FP16_Optimizer`, and should not be directly manipulated by the user. + + Use of :class:`LossScaler` is enabled via the ``static_loss_scale`` argument to + :class:`FP16_Optimizer`'s constructor. + + Args: + scale (float, optional, default=1.0): The loss scale. + """ + + def __init__(self, scale=1): + self.cur_scale = scale + + # `params` is a list / generator of torch.Variable + def has_overflow(self, params): + return False + + # `x` is a torch.Tensor + def _has_inf_or_nan(x): + return False + + def update_scale(self, overflow): + pass + + @property + def loss_scale(self): + return self.cur_scale + + def scale_gradient(self, module, grad_in, grad_out): + return tuple(self.loss_scale * g for g in grad_in) + + def backward(self, loss, retain_graph=False): + scaled_loss = loss*self.loss_scale + scaled_loss.backward(retain_graph=retain_graph) + +class DynamicLossScaler: + """ + Class that manages dynamic loss scaling. It is recommended to use :class:`DynamicLossScaler` + indirectly, by supplying ``dynamic_loss_scale=True`` to the constructor of + :class:`FP16_Optimizer`. However, it's important to understand how :class:`DynamicLossScaler` + operates, because the default options can be changed using the + the ``dynamic_loss_args`` argument to :class:`FP16_Optimizer`'s constructor. + + Loss scaling is designed to combat the problem of underflowing gradients encountered at long + times when training fp16 networks. Dynamic loss scaling begins by attempting a very high loss + scale. Ironically, this may result in OVERflowing gradients. If overflowing gradients are + encountered, :class:`DynamicLossScaler` informs :class:`FP16_Optimizer` that an overflow has + occurred. + :class:`FP16_Optimizer` then skips the update step for this particular iteration/minibatch, + and :class:`DynamicLossScaler` adjusts the loss scale to a lower value. + If a certain number of iterations occur without overflowing gradients detected, + :class:`DynamicLossScaler` increases the loss scale once more. + In this way :class:`DynamicLossScaler` attempts to "ride the edge" of + always using the highest loss scale possible without incurring overflow. + + Args: + init_scale (float, optional, default=2**32): Initial loss scale attempted by :class:`DynamicLossScaler.` + scale_factor (float, optional, default=2.0): Factor used when adjusting the loss scale. If an overflow is encountered, the loss scale is readjusted to loss scale/``scale_factor``. If ``scale_window`` consecutive iterations take place without an overflow, the loss scale is readjusted to loss_scale*``scale_factor``. + scale_window (int, optional, default=1000): Number of consecutive iterations without an overflow to wait before increasing the loss scale. + """ + + def __init__(self, + init_scale=2**32, + scale_factor=2., + scale_window=1000): + self.cur_scale = init_scale + self.cur_iter = 0 + self.last_overflow_iter = -1 + self.scale_factor = scale_factor + self.scale_window = scale_window + + # `params` is a list / generator of torch.Variable + def has_overflow(self, params): + for p in params: + if p.grad is not None and DynamicLossScaler._has_inf_or_nan(p.grad.data): + return True + + return False + + # `x` is a torch.Tensor + def _has_inf_or_nan(x): + try: + # if x is half, the .float() incurs an additional deep copy, but it's necessary if + # Pytorch's .sum() creates a one-element tensor of the same type as x + # (which is true for some recent version of pytorch). + cpu_sum = float(x.float().sum()) + # More efficient version that can be used if .sum() returns a Python scalar + # cpu_sum = float(x.sum()) + except RuntimeError as instance: + # We want to check if inst is actually an overflow exception. + # RuntimeError could come from a different error. + # If so, we still want the exception to propagate. + if "value cannot be converted" not in instance.args[0]: + raise + return True + else: + if cpu_sum == float('inf') or cpu_sum == -float('inf') or cpu_sum != cpu_sum: + return True + return False + + # `overflow` is boolean indicating whether the gradient overflowed + def update_scale(self, overflow): + if overflow: + # self.cur_scale /= self.scale_factor + self.cur_scale = max(self.cur_scale/self.scale_factor, 1) + self.last_overflow_iter = self.cur_iter + else: + if (self.cur_iter - self.last_overflow_iter) % self.scale_window == 0: + self.cur_scale *= self.scale_factor + self.cur_iter += 1 + + @property + def loss_scale(self): + return self.cur_scale + + def scale_gradient(self, module, grad_in, grad_out): + return tuple(self.loss_scale * g for g in grad_in) + + def backward(self, loss, retain_graph=False): + scaled_loss = loss*self.loss_scale + scaled_loss.backward(retain_graph=retain_graph) + +############################################################## +# Example usage below here -- assuming it's in a separate file +############################################################## +""" +TO-DO separate out into an example. +if __name__ == "__main__": + import torch + from torch.autograd import Variable + from dynamic_loss_scaler import DynamicLossScaler + + # N is batch size; D_in is input dimension; + # H is hidden dimension; D_out is output dimension. + N, D_in, H, D_out = 64, 1000, 100, 10 + + # Create random Tensors to hold inputs and outputs, and wrap them in Variables. + x = Variable(torch.randn(N, D_in), requires_grad=False) + y = Variable(torch.randn(N, D_out), requires_grad=False) + + w1 = Variable(torch.randn(D_in, H), requires_grad=True) + w2 = Variable(torch.randn(H, D_out), requires_grad=True) + parameters = [w1, w2] + + learning_rate = 1e-6 + optimizer = torch.optim.SGD(parameters, lr=learning_rate) + loss_scaler = DynamicLossScaler() + + for t in range(500): + y_pred = x.mm(w1).clamp(min=0).mm(w2) + loss = (y_pred - y).pow(2).sum() * loss_scaler.loss_scale + print('Iter {} loss scale: {}'.format(t, loss_scaler.loss_scale)) + print('Iter {} scaled loss: {}'.format(t, loss.data[0])) + print('Iter {} unscaled loss: {}'.format(t, loss.data[0] / loss_scaler.loss_scale)) + + # Run backprop + optimizer.zero_grad() + loss.backward() + + # Check for overflow + has_overflow = DynamicLossScaler.has_overflow(parameters) + + # If no overflow, unscale grad and update as usual + if not has_overflow: + for param in parameters: + param.grad.data.mul_(1. / loss_scaler.loss_scale) + optimizer.step() + # Otherwise, don't do anything -- ie, skip iteration + else: + print('OVERFLOW!') + + # Update loss scale for next iteration + loss_scaler.update_scale(has_overflow) + +""" diff --git a/detection/mm_modules/utils.py b/detection/mm_modules/utils.py new file mode 100644 index 0000000..c58b062 --- /dev/null +++ b/detection/mm_modules/utils.py @@ -0,0 +1,138 @@ +from __future__ import division +import torch +import torchvision +import numpy as np +import cv2 + +def postprocess(prediction, num_classes, conf_thre=0.7, nms_thre=0.45): + """ + Postprocess for the output of YOLO model + perform box transformation, specify the class for each detection, + and perform class-wise non-maximum suppression. + Args: + prediction (torch tensor): The shape is :math:`(N, B, 4)`. + :math:`N` is the number of predictions, + :math:`B` the number of boxes. The last axis consists of + :math:`xc, yc, w, h` where `xc` and `yc` represent a center + of a bounding box. + num_classes (int): + number of dataset classes. + conf_thre (float): + confidence threshold ranging from 0 to 1, + which is defined in the config file. + nms_thre (float): + IoU threshold of non-max suppression ranging from 0 to 1. + + Returns: + output (list of torch tensor): + + """ + box_corner = prediction.new(prediction.shape) + box_corner[:, :, 0] = prediction[:, :, 0] - prediction[:, :, 2] / 2 + box_corner[:, :, 1] = prediction[:, :, 1] - prediction[:, :, 3] / 2 + box_corner[:, :, 2] = prediction[:, :, 0] + prediction[:, :, 2] / 2 + box_corner[:, :, 3] = prediction[:, :, 1] + prediction[:, :, 3] / 2 + prediction[:, :, :4] = box_corner[:, :, :4] + + output = [None for _ in range(len(prediction))] + for i, image_pred in enumerate(prediction): + + # If none are remaining => process next image + if not image_pred.size(0): + continue + # Get score and class with highest confidence + class_conf, class_pred = torch.max( + image_pred[:, 5:5 + num_classes], 1, keepdim=True) + + conf_mask = (image_pred[:, 4] * class_conf.squeeze() >= conf_thre).squeeze() + # Detections ordered as (x1, y1, x2, y2, obj_conf, class_conf, class_pred) + detections = torch.cat( + (image_pred[:, :5], class_conf, class_pred.float()), 1) + detections = detections[conf_mask] + if not detections.size(0): + continue + + # Iterate through all predicted classes + unique_labels = detections[:, -1].unique() + + for c in unique_labels: + # Get the detections with the particular class + detections_class = detections[detections[:, -1] == c] + nms_out_index = torchvision.ops.nms( + detections_class[:, :4], detections_class[:, 4]*detections_class[:, 5], nms_thre) + detections_class = detections_class[nms_out_index] + if output[i] is None: + output[i] = detections_class + else: + output[i] = torch.cat((output[i], detections_class)) + + return output + + +def bboxes_iou(bboxes_a, bboxes_b, xyxy=True): + """Calculate the Intersection of Unions (IoUs) between bounding boxes. + IoU is calculated as a ratio of area of the intersection + and area of the union. + + Args: + bbox_a (array): An array whose shape is :math:`(N, 4)`. + :math:`N` is the number of bounding boxes. + The dtype should be :obj:`numpy.float32`. + bbox_b (array): An array similar to :obj:`bbox_a`, + whose shape is :math:`(K, 4)`. + The dtype should be :obj:`numpy.float32`. + Returns: + array: + An array whose shape is :math:`(N, K)`. \ + An element at index :math:`(n, k)` contains IoUs between \ + :math:`n` th bounding box in :obj:`bbox_a` and :math:`k` th bounding \ + box in :obj:`bbox_b`. + + from: https://github.com/chainer/chainercv + """ + if bboxes_a.shape[1] != 4 or bboxes_b.shape[1] != 4: + raise IndexError + + if xyxy: + tl = torch.max(bboxes_a[:, None, :2], bboxes_b[:, :2]) + br = torch.min(bboxes_a[:, None, 2:], bboxes_b[:, 2:]) + area_a = torch.prod(bboxes_a[:, 2:] - bboxes_a[:, :2], 1) + area_b = torch.prod(bboxes_b[:, 2:] - bboxes_b[:, :2], 1) + else: + tl = torch.max((bboxes_a[:, None, :2] - bboxes_a[:, None, 2:] / 2), + (bboxes_b[:, :2] - bboxes_b[:, 2:] / 2)) + br = torch.min((bboxes_a[:, None, :2] + bboxes_a[:, None, 2:] / 2), + (bboxes_b[:, :2] + bboxes_b[:, 2:] / 2)) + + area_a = torch.prod(bboxes_a[:, 2:], 1) + area_b = torch.prod(bboxes_b[:, 2:], 1) + en = (tl < br).type(tl.type()).prod(dim=2) + area_i = torch.prod(br - tl, 2) * en # * ((tl < br).all()) + return area_i / (area_a[:, None] + area_b - area_i) + + +def matrix_iou(a,b): + """ + return iou of a and b, numpy version for data augenmentation + """ + lt = np.maximum(a[:, np.newaxis, :2], b[:, :2]) + rb = np.minimum(a[:, np.newaxis, 2:], b[:, 2:]) + + area_i = np.prod(rb - lt, axis=2) * (lt < rb).all(axis=2) + area_a = np.prod(a[:, 2:] - a[:, :2], axis=1) + area_b = np.prod(b[:, 2:] - b[:, :2], axis=1) + return area_i / (area_a[:, np.newaxis] + area_b - area_i+1e-12) + +def visual(img, boxes, scores): + + COLORS = [(255, 0, 0), (0, 255, 0), (0, 0, 255)] + FONT = cv2.FONT_HERSHEY_SIMPLEX + for i in range(boxes.shape[0]): + + cv2.rectangle(img, (int(boxes[i][0]),int(boxes[i][1])),(int(boxes[i][2]),int(boxes[i][3])),COLORS[i%3],2) + cv2.putText(img, 'Object: %.2f'%scores[i],(int(boxes[i][0])-3,int(boxes[i][1])-5), FONT, + 0.4, (0,0,0),2) + + return img + + diff --git a/detection/mm_modules/vis_utils.py b/detection/mm_modules/vis_utils.py new file mode 100644 index 0000000..a322afc --- /dev/null +++ b/detection/mm_modules/vis_utils.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- + +import numpy as np +import os +import matplotlib + +matplotlib.use('AGG') + +import matplotlib.pyplot as plt +import torch +import cv2 +import math +from skimage import transform + +def make_vis(dataset, index, img, fuse_weights, fused_fs): + save_dir = 'vis_output/{}/{}'.format(dataset,index) + os.makedirs(save_dir, exist_ok=True) + + for i in range(len(fuse_weights)): + weights = fuse_weights[i].float().cpu().squeeze().numpy() + max_v = weights.max() + min_v = weights.min() + for j in range(3): + v = weights[j,:,:] + save_name = os.path.join(save_dir, 'level_{}_weight_{}.png'.format(i+1,j+1)) + add_heat(img, v, max_v, min_v, save=save_name) + + fused_f = fused_fs[i].float().cpu().squeeze().numpy() + max_f = fused_f.max() + min_f = fused_f.min() + save_f_name = os.path.join(save_dir, 'fused_feature_level_{}.png'.format(i+1)) + add_heat(img, fused_f, max_f, min_f, save=save_f_name) + +def make_pred_vis(dataset,index, img, class_names, bboxes, cls, scores): + save_preddir = 'vis_output/{}/pred/'.format(dataset) + os.makedirs(save_preddir, exist_ok=True) + + save_pred_name = os.path.join(save_preddir,'{}.png'.format(index)) + + bboxes = bboxes.numpy() + scores = scores.numpy() + cls_ids = cls.numpy() + + im = vis(img, bboxes, scores, cls_ids, class_names) + + cv2.imwrite(save_pred_name, im) + +def vis(img, boxes, scores, cls_ids, conf=0.5, class_names=None, color=None): + + colors = torch.FloatTensor([[1,0,1],[0,0,1],[0,1,1],[0,1,0],[1,1,0],[1,0,0]]); + def get_color(c, x, max_val): + ratio = float(x)/max_val * 5 + i = int(math.floor(ratio)) + j = int(math.ceil(ratio)) + ratio = ratio - i + r = (1-ratio) * colors[i][c] + ratio*colors[j][c] + return int(r*255) + + width = img.shape[1] + height = img.shape[0] + for i in range(len(boxes)): + box = boxes[i] + cls_conf = scores[i] + if cls_conf < conf: + continue + x1 = int(box[0]) + y1 = int(box[1]) + x2 = int(box[0]+box[2]) + y2 = int(box[1]+box[3]) + + + if color: + rgb = color + else: + rgb = (255, 0, 0) + if class_names is not None: + cls_conf = scores[i] + cls_id = int(cls_ids[i]) + class_name = class_names[cls_id] + classes = len(class_names) + offset = cls_id * 123456 % classes + red = get_color(2, offset, classes) + green = get_color(1, offset, classes) + blue = get_color(0, offset, classes) + if color is None: + rgb = (red, green, blue) + img = cv2.putText(img, '%s: %.2f'%(class_name,cls_conf), (x1,y1-5), cv2.FONT_HERSHEY_SIMPLEX, 0.3, rgb, 1) + img = cv2.rectangle(img, (x1,y1), (x2,y2), rgb, 1) + return img + +def add_heat(image, heat_map, max_v, min_v, alpha=0.4, save=None, cmap='jet', axis='off'): + height = image.shape[0] + width = image.shape[1] + + # resize heat map + heat_map_resized = transform.resize(heat_map, (height, width)) + + # normalize heat map + max_value = max_v + min_value = min_v + normalized_heat_map = (heat_map_resized - min_value) / (max_value - min_value) + + # display + plt.imshow(image) + plt.imshow(255 * normalized_heat_map, alpha=alpha, cmap=cmap) + plt.axis(axis) + + if save is not None: + plt.savefig(save, bbox_inches='tight', pad_inches=0) + + + + diff --git a/detection/mm_modules/voc_evaluator.py b/detection/mm_modules/voc_evaluator.py new file mode 100644 index 0000000..1fc5ae8 --- /dev/null +++ b/detection/mm_modules/voc_evaluator.py @@ -0,0 +1,204 @@ +import json +import tempfile +import sys +from tqdm import tqdm + +from pycocotools.cocoeval import COCOeval +from torch.autograd import Variable + +from dataset.vocdataset import * +from dataset.data_augment import ValTransform +from utils.utils import * +from utils import distributed_util +from utils.vis_utils import make_vis, make_pred_vis + +import time + +#DEBUG = True +DEBUG = False + +def _accumulate_predictions_from_multiple_gpus(predictions_per_gpu): + all_predictions = distributed_util.scatter_gather(predictions_per_gpu) + if not distributed_util.is_main_process(): + return + # merge the list of dicts + predictions = {} + for p in all_predictions: + predictions.update(p) + # convert a dict where the key is the index in a list + image_ids = list(sorted(predictions.keys())) + if len(image_ids) != image_ids[-1] + 1: + print('num_imgs: ',len(image_ids)) + print('last img_id: ',image_ids[-1]) + print( + "Number of images that were gathered from multiple processes is not " + "a contiguous set. Some images might be missing from the evaluation" + ) + + # convert to a list + predictions = [predictions[i] for i in image_ids] + return predictions + + +class VOCEvaluator(): + """ + COCO AP Evaluation class. + All the data in the val2017 dataset are processed \ + and evaluated by COCO API. + """ + def __init__(self, data_dir, img_size, confthre, nmsthre,vis=False): + """ + Args: + data_dir (str): dataset root directory + img_size (int): image size after preprocess. images are resized \ + to squares whose shape is (img_size, img_size). + confthre (float): + confidence threshold ranging from 0 to 1, \ + which is defined in the config file. + nmsthre (float): + IoU threshold of non-max supression ranging from 0 to 1. + """ + test_sets = [('2007', 'test'),] + self.dataset = VOCDetection( + root=data_dir, + image_sets = test_sets, + input_dim=img_size, + preproc = ValTransform(rgb_means=(0.485, 0.456, 0.406),std=(0.229, 0.224, 0.225)),) + self.num_images = len(self.dataset) + self.dataloader = torch.utils.data.DataLoader( + self.dataset, batch_size=1, shuffle=False, num_workers=0) + self.img_size = img_size + self.confthre = confthre + self.nmsthre = nmsthre + self.vis=vis + + def evaluate(self, model, half=False): + """ + COCO average precision (AP) Evaluation. Iterate inference on the test dataset + and the results are evaluated by COCO API. + Args: + model : model object + Returns: + ap50_95 (float) : calculated COCO AP for IoU=50:95 + ap50 (float) : calculated COCO AP for IoU=50 + """ + if isinstance(model, torch.nn.parallel.DistributedDataParallel): + model = model.module + model.eval() + cuda = torch.cuda.is_available() + if half: + Tensor = torch.cuda.HalfTensor if cuda else torch.HalfTensor + else: + Tensor = torch.cuda.FloatTensor if cuda else torch.FloatTensor + + ids = [] + data_dict = [] + dataiterator = iter(self.dataloader) + img_num = 0 + indices = list(range(self.num_images)) + dis_indices = indices[distributed_util.get_rank()::distributed_util.get_world_size()] + progress_bar = tqdm if distributed_util.is_main_process() else iter + num_classes = 20 + predictions = {} + + if distributed_util.is_main_process(): + inference_time=0 + nms_time=0 + n_samples=len(dis_indices) + + for i in progress_bar(dis_indices): + img, _, info_img, id_ = self.dataset[i] # load a batch + info_img = [float(info) for info in info_img] + ids.append(id_) + with torch.no_grad(): + img = Variable(img.type(Tensor).unsqueeze(0)) + + if distributed_util.is_main_process() and i > 9: + start=time.time() + + if self.vis: + outputs,fuse_weights,fused_f = model(img) + else: + outputs = model(img) + + if distributed_util.is_main_process() and i > 9: + infer_end=time.time() + inference_time += (infer_end-start) + + outputs = postprocess( + outputs, 20, self.confthre, self.nmsthre) + + + if distributed_util.is_main_process() and i > 9: + nms_end=time.time() + nms_time +=(nms_end-infer_end) + + if outputs[0] is None: + predictions[i] = (None, None, None) + continue + outputs = outputs[0].cpu().data + + bboxes = outputs[:, 0:4] + bboxes[:, 0::2] *= info_img[0] / self.img_size[0] + bboxes[:, 1::2] *= info_img[1] / self.img_size[1] + cls = outputs[:, 6] + scores = outputs[:, 4]* outputs[:,5] + predictions[i] = (bboxes, cls, scores) + + if self.vis: + o_img,_,_,_ = self.dataset.pull_item(i) + make_vis('VOC', i, o_img, fuse_weights, fused_f) + class_names = self.dataset._classes + + bbox = bboxes.clone() + bbox[:, 2] = bbox[:,2] - bbox[:,0] + bbox[:, 3] = bbox[:,3] - bbox[:,1] + + make_pred_vis('VOC', i, o_img, class_names, bbox, cls, scores) + + if DEBUG and distributed_util.is_main_process(): + o_img,_,_,_ = self.dataset.pull_item(i) + class_names = self.dataset._classes + bbox = bboxes.clone() + bbox[:, 2] = bbox[:,2] - bbox[:,0] + bbox[:, 3] = bbox[:,3] - bbox[:,1] + make_pred_vis('VOC', i, o_img, class_names, bbox, cls, scores) + + distributed_util.synchronize() + predictions = _accumulate_predictions_from_multiple_gpus(predictions) + if not distributed_util.is_main_process(): + return 0, 0 + + + print('Main process Evaluating...') + + a_infer_time = 1000*inference_time / (n_samples-10) + a_nms_time= 1000*nms_time / (n_samples-10) + + print('Average forward time: %.2f ms, Average NMS time: %.2f ms, Average inference time: %.2f ms' %(a_infer_time, \ + a_nms_time, (a_infer_time+a_nms_time))) + + all_boxes = [[[] for _ in range(self.num_images)] + for _ in range(num_classes)] + for img_num in range(self.num_images): + bboxes, cls, scores = predictions[img_num] + if bboxes is None: + for j in range(num_classes): + all_boxes[j][img_num] = np.empty([0,5],dtype=np.float32) + continue + for j in range(num_classes): + mask_c = (cls == j) + if sum(mask_c) ==0: + all_boxes[j][img_num] = np.empty([0,5],dtype=np.float32) + continue + + c_dets = torch.cat((bboxes, scores.unsqueeze(1)),dim=1) + all_boxes[j][img_num] = c_dets[mask_c].numpy() + + sys.stdout.write('im_eval: {:d}/{:d} \r'.format(img_num+1, self.num_images)) + sys.stdout.flush() + + with tempfile.TemporaryDirectory() as tempdir: + mAP50, mAP70 = self.dataset.evaluate_detections(all_boxes, tempdir) + return mAP50,mAP70 + diff --git a/detection/mmcv_custom/__init__.py b/detection/mmcv_custom/__init__.py new file mode 100644 index 0000000..7e0e39b --- /dev/null +++ b/detection/mmcv_custom/__init__.py @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- + +from .checkpoint import load_checkpoint + +__all__ = ['load_checkpoint'] diff --git a/detection/mmcv_custom/checkpoint.py b/detection/mmcv_custom/checkpoint.py new file mode 100644 index 0000000..51322c1 --- /dev/null +++ b/detection/mmcv_custom/checkpoint.py @@ -0,0 +1,500 @@ +# Copyright (c) Open-MMLab. All rights reserved. +import io +import os +import os.path as osp +import pkgutil +import time +import warnings +from collections import OrderedDict +from importlib import import_module +from tempfile import TemporaryDirectory + +import torch +import torchvision +from torch.optim import Optimizer +from torch.utils import model_zoo +from torch.nn import functional as F + +import mmcv +from mmcv.fileio import FileClient +from mmcv.fileio import load as load_file +from mmcv.parallel import is_module_wrapper +from mmcv.utils import mkdir_or_exist +from mmcv.runner import get_dist_info + +ENV_MMCV_HOME = 'MMCV_HOME' +ENV_XDG_CACHE_HOME = 'XDG_CACHE_HOME' +DEFAULT_CACHE_DIR = '~/.cache' + + +def _get_mmcv_home(): + mmcv_home = os.path.expanduser( + os.getenv( + ENV_MMCV_HOME, + os.path.join( + os.getenv(ENV_XDG_CACHE_HOME, DEFAULT_CACHE_DIR), 'mmcv'))) + + mkdir_or_exist(mmcv_home) + return mmcv_home + + +def load_state_dict(module, state_dict, strict=False, logger=None): + """Load state_dict to a module. + + This method is modified from :meth:`torch.nn.Module.load_state_dict`. + Default value for ``strict`` is set to ``False`` and the message for + param mismatch will be shown even if strict is False. + + Args: + module (Module): Module that receives the state_dict. + state_dict (OrderedDict): Weights. + strict (bool): whether to strictly enforce that the keys + in :attr:`state_dict` match the keys returned by this module's + :meth:`~torch.nn.Module.state_dict` function. Default: ``False``. + logger (:obj:`logging.Logger`, optional): Logger to log the error + message. If not specified, print function will be used. + """ + unexpected_keys = [] + all_missing_keys = [] + err_msg = [] + + metadata = getattr(state_dict, '_metadata', None) + state_dict = state_dict.copy() + if metadata is not None: + state_dict._metadata = metadata + + # use _load_from_state_dict to enable checkpoint version control + def load(module, prefix=''): + # recursively check parallel module in case that the model has a + # complicated structure, e.g., nn.Module(nn.Module(DDP)) + if is_module_wrapper(module): + module = module.module + local_metadata = {} if metadata is None else metadata.get( + prefix[:-1], {}) + module._load_from_state_dict(state_dict, prefix, local_metadata, True, + all_missing_keys, unexpected_keys, + err_msg) + for name, child in module._modules.items(): + if child is not None: + load(child, prefix + name + '.') + + load(module) + load = None # break load->load reference cycle + + # ignore "num_batches_tracked" of BN layers + missing_keys = [ + key for key in all_missing_keys if 'num_batches_tracked' not in key + ] + + if unexpected_keys: + err_msg.append('unexpected key in source ' + f'state_dict: {", ".join(unexpected_keys)}\n') + if missing_keys: + err_msg.append( + f'missing keys in source state_dict: {", ".join(missing_keys)}\n') + + rank, _ = get_dist_info() + if len(err_msg) > 0 and rank == 0: + err_msg.insert( + 0, 'The model and loaded state dict do not match exactly\n') + err_msg = '\n'.join(err_msg) + if strict: + raise RuntimeError(err_msg) + elif logger is not None: + logger.warning(err_msg) + else: + print(err_msg) + + +def load_url_dist(url, model_dir=None): + """In distributed setting, this function only download checkpoint at local + rank 0.""" + rank, world_size = get_dist_info() + rank = int(os.environ.get('LOCAL_RANK', rank)) + if rank == 0: + checkpoint = model_zoo.load_url(url, model_dir=model_dir) + if world_size > 1: + torch.distributed.barrier() + if rank > 0: + checkpoint = model_zoo.load_url(url, model_dir=model_dir) + return checkpoint + + +def load_pavimodel_dist(model_path, map_location=None): + """In distributed setting, this function only download checkpoint at local + rank 0.""" + try: + from pavi import modelcloud + except ImportError: + raise ImportError( + 'Please install pavi to load checkpoint from modelcloud.') + rank, world_size = get_dist_info() + rank = int(os.environ.get('LOCAL_RANK', rank)) + if rank == 0: + model = modelcloud.get(model_path) + with TemporaryDirectory() as tmp_dir: + downloaded_file = osp.join(tmp_dir, model.name) + model.download(downloaded_file) + checkpoint = torch.load(downloaded_file, map_location=map_location) + if world_size > 1: + torch.distributed.barrier() + if rank > 0: + model = modelcloud.get(model_path) + with TemporaryDirectory() as tmp_dir: + downloaded_file = osp.join(tmp_dir, model.name) + model.download(downloaded_file) + checkpoint = torch.load( + downloaded_file, map_location=map_location) + return checkpoint + + +def load_fileclient_dist(filename, backend, map_location): + """In distributed setting, this function only download checkpoint at local + rank 0.""" + rank, world_size = get_dist_info() + rank = int(os.environ.get('LOCAL_RANK', rank)) + allowed_backends = ['ceph'] + if backend not in allowed_backends: + raise ValueError(f'Load from Backend {backend} is not supported.') + if rank == 0: + fileclient = FileClient(backend=backend) + buffer = io.BytesIO(fileclient.get(filename)) + checkpoint = torch.load(buffer, map_location=map_location) + if world_size > 1: + torch.distributed.barrier() + if rank > 0: + fileclient = FileClient(backend=backend) + buffer = io.BytesIO(fileclient.get(filename)) + checkpoint = torch.load(buffer, map_location=map_location) + return checkpoint + + +def get_torchvision_models(): + model_urls = dict() + for _, name, ispkg in pkgutil.walk_packages(torchvision.models.__path__): + if ispkg: + continue + _zoo = import_module(f'torchvision.models.{name}') + if hasattr(_zoo, 'model_urls'): + _urls = getattr(_zoo, 'model_urls') + model_urls.update(_urls) + return model_urls + + +def get_external_models(): + mmcv_home = _get_mmcv_home() + default_json_path = osp.join(mmcv.__path__[0], 'model_zoo/open_mmlab.json') + default_urls = load_file(default_json_path) + assert isinstance(default_urls, dict) + external_json_path = osp.join(mmcv_home, 'open_mmlab.json') + if osp.exists(external_json_path): + external_urls = load_file(external_json_path) + assert isinstance(external_urls, dict) + default_urls.update(external_urls) + + return default_urls + + +def get_mmcls_models(): + mmcls_json_path = osp.join(mmcv.__path__[0], 'model_zoo/mmcls.json') + mmcls_urls = load_file(mmcls_json_path) + + return mmcls_urls + + +def get_deprecated_model_names(): + deprecate_json_path = osp.join(mmcv.__path__[0], + 'model_zoo/deprecated.json') + deprecate_urls = load_file(deprecate_json_path) + assert isinstance(deprecate_urls, dict) + + return deprecate_urls + + +def _process_mmcls_checkpoint(checkpoint): + state_dict = checkpoint['state_dict'] + new_state_dict = OrderedDict() + for k, v in state_dict.items(): + if k.startswith('backbone.'): + new_state_dict[k[9:]] = v + new_checkpoint = dict(state_dict=new_state_dict) + + return new_checkpoint + + +def _load_checkpoint(filename, map_location=None): + """Load checkpoint from somewhere (modelzoo, file, url). + + Args: + filename (str): Accept local filepath, URL, ``torchvision://xxx``, + ``open-mmlab://xxx``. Please refer to ``docs/model_zoo.md`` for + details. + map_location (str | None): Same as :func:`torch.load`. Default: None. + + Returns: + dict | OrderedDict: The loaded checkpoint. It can be either an + OrderedDict storing model weights or a dict containing other + information, which depends on the checkpoint. + """ + if filename.startswith('modelzoo://'): + warnings.warn('The URL scheme of "modelzoo://" is deprecated, please ' + 'use "torchvision://" instead') + model_urls = get_torchvision_models() + model_name = filename[11:] + checkpoint = load_url_dist(model_urls[model_name]) + elif filename.startswith('torchvision://'): + model_urls = get_torchvision_models() + model_name = filename[14:] + checkpoint = load_url_dist(model_urls[model_name]) + elif filename.startswith('open-mmlab://'): + model_urls = get_external_models() + model_name = filename[13:] + deprecated_urls = get_deprecated_model_names() + if model_name in deprecated_urls: + warnings.warn(f'open-mmlab://{model_name} is deprecated in favor ' + f'of open-mmlab://{deprecated_urls[model_name]}') + model_name = deprecated_urls[model_name] + model_url = model_urls[model_name] + # check if is url + if model_url.startswith(('http://', 'https://')): + checkpoint = load_url_dist(model_url) + else: + filename = osp.join(_get_mmcv_home(), model_url) + if not osp.isfile(filename): + raise IOError(f'{filename} is not a checkpoint file') + checkpoint = torch.load(filename, map_location=map_location) + elif filename.startswith('mmcls://'): + model_urls = get_mmcls_models() + model_name = filename[8:] + checkpoint = load_url_dist(model_urls[model_name]) + checkpoint = _process_mmcls_checkpoint(checkpoint) + elif filename.startswith(('http://', 'https://')): + checkpoint = load_url_dist(filename) + elif filename.startswith('pavi://'): + model_path = filename[7:] + checkpoint = load_pavimodel_dist(model_path, map_location=map_location) + elif filename.startswith('s3://'): + checkpoint = load_fileclient_dist( + filename, backend='ceph', map_location=map_location) + else: + if not osp.isfile(filename): + raise IOError(f'{filename} is not a checkpoint file') + checkpoint = torch.load(filename, map_location=map_location) + return checkpoint + + +def load_checkpoint(model, + filename, + map_location='cpu', + strict=False, + logger=None): + """Load checkpoint from a file or URI. + + Args: + model (Module): Module to load checkpoint. + filename (str): Accept local filepath, URL, ``torchvision://xxx``, + ``open-mmlab://xxx``. Please refer to ``docs/model_zoo.md`` for + details. + map_location (str): Same as :func:`torch.load`. + strict (bool): Whether to allow different params for the model and + checkpoint. + logger (:mod:`logging.Logger` or None): The logger for error message. + + Returns: + dict or OrderedDict: The loaded checkpoint. + """ + checkpoint = _load_checkpoint(filename, map_location) + # OrderedDict is a subclass of dict + if not isinstance(checkpoint, dict): + raise RuntimeError( + f'No state_dict found in checkpoint file {filename}') + # get state_dict from checkpoint + if 'state_dict' in checkpoint: + state_dict = checkpoint['state_dict'] + elif 'model' in checkpoint: + state_dict = checkpoint['model'] + else: + state_dict = checkpoint + # strip prefix of state_dict + if list(state_dict.keys())[0].startswith('module.'): + state_dict = {k[7:]: v for k, v in state_dict.items()} + + # for MoBY, load model of online branch + if sorted(list(state_dict.keys()))[0].startswith('encoder'): + state_dict = {k.replace('encoder.', ''): v for k, v in state_dict.items() if k.startswith('encoder.')} + + # reshape absolute position embedding + if state_dict.get('absolute_pos_embed') is not None: + absolute_pos_embed = state_dict['absolute_pos_embed'] + N1, L, C1 = absolute_pos_embed.size() + N2, C2, H, W = model.absolute_pos_embed.size() + if N1 != N2 or C1 != C2 or L != H*W: + logger.warning("Error in loading absolute_pos_embed, pass") + else: + state_dict['absolute_pos_embed'] = absolute_pos_embed.view(N2, H, W, C2).permute(0, 3, 1, 2) + + # interpolate position bias table if needed + relative_position_bias_table_keys = [k for k in state_dict.keys() if "relative_position_bias_table" in k] + for table_key in relative_position_bias_table_keys: + table_pretrained = state_dict[table_key] + table_current = model.state_dict()[table_key] + L1, nH1 = table_pretrained.size() + L2, nH2 = table_current.size() + if nH1 != nH2: + logger.warning(f"Error in loading {table_key}, pass") + else: + if L1 != L2: + S1 = int(L1 ** 0.5) + S2 = int(L2 ** 0.5) + table_pretrained_resized = F.interpolate( + table_pretrained.permute(1, 0).view(1, nH1, S1, S1), + size=(S2, S2), mode='bicubic') + state_dict[table_key] = table_pretrained_resized.view(nH2, L2).permute(1, 0) + + # load state_dict + load_state_dict(model, state_dict, strict, logger) + return checkpoint + + +def weights_to_cpu(state_dict): + """Copy a model state_dict to cpu. + + Args: + state_dict (OrderedDict): Model weights on GPU. + + Returns: + OrderedDict: Model weights on GPU. + """ + state_dict_cpu = OrderedDict() + for key, val in state_dict.items(): + state_dict_cpu[key] = val.cpu() + return state_dict_cpu + + +def _save_to_state_dict(module, destination, prefix, keep_vars): + """Saves module state to `destination` dictionary. + + This method is modified from :meth:`torch.nn.Module._save_to_state_dict`. + + Args: + module (nn.Module): The module to generate state_dict. + destination (dict): A dict where state will be stored. + prefix (str): The prefix for parameters and buffers used in this + module. + """ + for name, param in module._parameters.items(): + if param is not None: + destination[prefix + name] = param if keep_vars else param.detach() + for name, buf in module._buffers.items(): + # remove check of _non_persistent_buffers_set to allow nn.BatchNorm2d + if buf is not None: + destination[prefix + name] = buf if keep_vars else buf.detach() + + +def get_state_dict(module, destination=None, prefix='', keep_vars=False): + """Returns a dictionary containing a whole state of the module. + + Both parameters and persistent buffers (e.g. running averages) are + included. Keys are corresponding parameter and buffer names. + + This method is modified from :meth:`torch.nn.Module.state_dict` to + recursively check parallel module in case that the model has a complicated + structure, e.g., nn.Module(nn.Module(DDP)). + + Args: + module (nn.Module): The module to generate state_dict. + destination (OrderedDict): Returned dict for the state of the + module. + prefix (str): Prefix of the key. + keep_vars (bool): Whether to keep the variable property of the + parameters. Default: False. + + Returns: + dict: A dictionary containing a whole state of the module. + """ + # recursively check parallel module in case that the model has a + # complicated structure, e.g., nn.Module(nn.Module(DDP)) + if is_module_wrapper(module): + module = module.module + + # below is the same as torch.nn.Module.state_dict() + if destination is None: + destination = OrderedDict() + destination._metadata = OrderedDict() + destination._metadata[prefix[:-1]] = local_metadata = dict( + version=module._version) + _save_to_state_dict(module, destination, prefix, keep_vars) + for name, child in module._modules.items(): + if child is not None: + get_state_dict( + child, destination, prefix + name + '.', keep_vars=keep_vars) + for hook in module._state_dict_hooks.values(): + hook_result = hook(module, destination, prefix, local_metadata) + if hook_result is not None: + destination = hook_result + return destination + + +def save_checkpoint(model, filename, optimizer=None, meta=None): + """Save checkpoint to file. + + The checkpoint will have 3 fields: ``meta``, ``state_dict`` and + ``optimizer``. By default ``meta`` will contain version and time info. + + Args: + model (Module): Module whose params are to be saved. + filename (str): Checkpoint filename. + optimizer (:obj:`Optimizer`, optional): Optimizer to be saved. + meta (dict, optional): Metadata to be saved in checkpoint. + """ + if meta is None: + meta = {} + elif not isinstance(meta, dict): + raise TypeError(f'meta must be a dict or None, but got {type(meta)}') + meta.update(mmcv_version=mmcv.__version__, time=time.asctime()) + + if is_module_wrapper(model): + model = model.module + + if hasattr(model, 'CLASSES') and model.CLASSES is not None: + # save class name to the meta + meta.update(CLASSES=model.CLASSES) + + checkpoint = { + 'meta': meta, + 'state_dict': weights_to_cpu(get_state_dict(model)) + } + # save optimizer state dict in the checkpoint + if isinstance(optimizer, Optimizer): + checkpoint['optimizer'] = optimizer.state_dict() + elif isinstance(optimizer, dict): + checkpoint['optimizer'] = {} + for name, optim in optimizer.items(): + checkpoint['optimizer'][name] = optim.state_dict() + + if filename.startswith('pavi://'): + try: + from pavi import modelcloud + from pavi.exception import NodeNotFoundError + except ImportError: + raise ImportError( + 'Please install pavi to load checkpoint from modelcloud.') + model_path = filename[7:] + root = modelcloud.Folder() + model_dir, model_name = osp.split(model_path) + try: + model = modelcloud.get(model_dir) + except NodeNotFoundError: + model = root.create_training_model(model_dir) + with TemporaryDirectory() as tmp_dir: + checkpoint_file = osp.join(tmp_dir, model_name) + with open(checkpoint_file, 'wb') as f: + torch.save(checkpoint, f) + f.flush() + model.create_file(checkpoint_file, name=model_name) + else: + mmcv.mkdir_or_exist(osp.dirname(filename)) + # immediately flush buffer + with open(filename, 'wb') as f: + torch.save(checkpoint, f) + f.flush() diff --git a/detection/mmcv_custom/runner/__init__.py b/detection/mmcv_custom/runner/__init__.py new file mode 100644 index 0000000..c701cb0 --- /dev/null +++ b/detection/mmcv_custom/runner/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) Open-MMLab. All rights reserved. +from .checkpoint import save_checkpoint +from .epoch_based_runner import EpochBasedRunnerAmp + + +__all__ = [ + 'EpochBasedRunnerAmp', 'save_checkpoint' +] diff --git a/detection/mmcv_custom/runner/checkpoint.py b/detection/mmcv_custom/runner/checkpoint.py new file mode 100644 index 0000000..b04167e --- /dev/null +++ b/detection/mmcv_custom/runner/checkpoint.py @@ -0,0 +1,85 @@ +# Copyright (c) Open-MMLab. All rights reserved. +import os.path as osp +import time +from tempfile import TemporaryDirectory + +import torch +from torch.optim import Optimizer + +import mmcv +from mmcv.parallel import is_module_wrapper +from mmcv.runner.checkpoint import weights_to_cpu, get_state_dict + +try: + import apex +except: + print('apex is not installed') + + +def save_checkpoint(model, filename, optimizer=None, meta=None): + """Save checkpoint to file. + + The checkpoint will have 4 fields: ``meta``, ``state_dict`` and + ``optimizer``, ``amp``. By default ``meta`` will contain version + and time info. + + Args: + model (Module): Module whose params are to be saved. + filename (str): Checkpoint filename. + optimizer (:obj:`Optimizer`, optional): Optimizer to be saved. + meta (dict, optional): Metadata to be saved in checkpoint. + """ + if meta is None: + meta = {} + elif not isinstance(meta, dict): + raise TypeError(f'meta must be a dict or None, but got {type(meta)}') + meta.update(mmcv_version=mmcv.__version__, time=time.asctime()) + + if is_module_wrapper(model): + model = model.module + + if hasattr(model, 'CLASSES') and model.CLASSES is not None: + # save class name to the meta + meta.update(CLASSES=model.CLASSES) + + checkpoint = { + 'meta': meta, + 'state_dict': weights_to_cpu(get_state_dict(model)) + } + # save optimizer state dict in the checkpoint + if isinstance(optimizer, Optimizer): + checkpoint['optimizer'] = optimizer.state_dict() + elif isinstance(optimizer, dict): + checkpoint['optimizer'] = {} + for name, optim in optimizer.items(): + checkpoint['optimizer'][name] = optim.state_dict() + + # save amp state dict in the checkpoint + checkpoint['amp'] = apex.amp.state_dict() + + if filename.startswith('pavi://'): + try: + from pavi import modelcloud + from pavi.exception import NodeNotFoundError + except ImportError: + raise ImportError( + 'Please install pavi to load checkpoint from modelcloud.') + model_path = filename[7:] + root = modelcloud.Folder() + model_dir, model_name = osp.split(model_path) + try: + model = modelcloud.get(model_dir) + except NodeNotFoundError: + model = root.create_training_model(model_dir) + with TemporaryDirectory() as tmp_dir: + checkpoint_file = osp.join(tmp_dir, model_name) + with open(checkpoint_file, 'wb') as f: + torch.save(checkpoint, f) + f.flush() + model.create_file(checkpoint_file, name=model_name) + else: + mmcv.mkdir_or_exist(osp.dirname(filename)) + # immediately flush buffer + with open(filename, 'wb') as f: + torch.save(checkpoint, f) + f.flush() diff --git a/detection/mmcv_custom/runner/epoch_based_runner.py b/detection/mmcv_custom/runner/epoch_based_runner.py new file mode 100644 index 0000000..7cdf3fa --- /dev/null +++ b/detection/mmcv_custom/runner/epoch_based_runner.py @@ -0,0 +1,104 @@ +# Copyright (c) Open-MMLab. All rights reserved. +import os.path as osp +import platform +import shutil + +import torch +from torch.optim import Optimizer + +import mmcv +from mmcv.runner import RUNNERS, EpochBasedRunner +from .checkpoint import save_checkpoint + +try: + import apex +except: + print('apex is not installed') + + +@RUNNERS.register_module() +class EpochBasedRunnerAmp(EpochBasedRunner): + """Epoch-based Runner with AMP support. + + This runner train models epoch by epoch. + """ + + def save_checkpoint(self, + out_dir, + filename_tmpl='epoch_{}.pth', + save_optimizer=True, + meta=None, + create_symlink=True): + """Save the checkpoint. + + Args: + out_dir (str): The directory that checkpoints are saved. + filename_tmpl (str, optional): The checkpoint filename template, + which contains a placeholder for the epoch number. + Defaults to 'epoch_{}.pth'. + save_optimizer (bool, optional): Whether to save the optimizer to + the checkpoint. Defaults to True. + meta (dict, optional): The meta information to be saved in the + checkpoint. Defaults to None. + create_symlink (bool, optional): Whether to create a symlink + "latest.pth" to point to the latest checkpoint. + Defaults to True. + """ + if meta is None: + meta = dict(epoch=self.epoch + 1, iter=self.iter) + elif isinstance(meta, dict): + meta.update(epoch=self.epoch + 1, iter=self.iter) + else: + raise TypeError( + f'meta should be a dict or None, but got {type(meta)}') + if self.meta is not None: + meta.update(self.meta) + + filename = filename_tmpl.format(self.epoch + 1) + filepath = osp.join(out_dir, filename) + optimizer = self.optimizer if save_optimizer else None + save_checkpoint(self.model, filepath, optimizer=optimizer, meta=meta) + # in some environments, `os.symlink` is not supported, you may need to + # set `create_symlink` to False + if create_symlink: + dst_file = osp.join(out_dir, 'latest.pth') + if platform.system() != 'Windows': + mmcv.symlink(filename, dst_file) + else: + shutil.copy(filepath, dst_file) + + def resume(self, + checkpoint, + resume_optimizer=True, + map_location='default'): + if map_location == 'default': + if torch.cuda.is_available(): + device_id = torch.cuda.current_device() + checkpoint = self.load_checkpoint( + checkpoint, + map_location=lambda storage, loc: storage.cuda(device_id)) + else: + checkpoint = self.load_checkpoint(checkpoint) + else: + checkpoint = self.load_checkpoint( + checkpoint, map_location=map_location) + + self._epoch = checkpoint['meta']['epoch'] + self._iter = checkpoint['meta']['iter'] + if 'optimizer' in checkpoint and resume_optimizer: + if isinstance(self.optimizer, Optimizer): + self.optimizer.load_state_dict(checkpoint['optimizer']) + elif isinstance(self.optimizer, dict): + for k in self.optimizer.keys(): + self.optimizer[k].load_state_dict( + checkpoint['optimizer'][k]) + else: + raise TypeError( + 'Optimizer should be dict or torch.optim.Optimizer ' + f'but got {type(self.optimizer)}') + + if 'amp' in checkpoint: + apex.amp.load_state_dict(checkpoint['amp']) + self.logger.info('load amp state dict') + + self.logger.info('resumed epoch %d, iter %d', self.epoch, self.iter) diff --git a/detection/mmdet/__init__.py b/detection/mmdet/__init__.py new file mode 100644 index 0000000..4ca764f --- /dev/null +++ b/detection/mmdet/__init__.py @@ -0,0 +1,28 @@ +import mmcv + +from .version import __version__, short_version + + +def digit_version(version_str): + digit_version = [] + for x in version_str.split('.'): + if x.isdigit(): + digit_version.append(int(x)) + elif x.find('rc') != -1: + patch_version = x.split('rc') + digit_version.append(int(patch_version[0]) - 1) + digit_version.append(int(patch_version[1])) + return digit_version + + +mmcv_minimum_version = '1.2.4' +mmcv_maximum_version = '1.4.0' +mmcv_version = digit_version(mmcv.__version__) + + +# assert (mmcv_version >= digit_version(mmcv_minimum_version) +# and mmcv_version <= digit_version(mmcv_maximum_version)), \ +# f'MMCV=={mmcv.__version__} is used but incompatible. ' \ +# f'Please install mmcv>={mmcv_minimum_version}, <={mmcv_maximum_version}.' + +__all__ = ['__version__', 'short_version'] diff --git a/detection/mmdet/apis/__init__.py b/detection/mmdet/apis/__init__.py new file mode 100644 index 0000000..1d8035b --- /dev/null +++ b/detection/mmdet/apis/__init__.py @@ -0,0 +1,10 @@ +from .inference import (async_inference_detector, inference_detector, + init_detector, show_result_pyplot) +from .test import multi_gpu_test, single_gpu_test +from .train import get_root_logger, set_random_seed, train_detector + +__all__ = [ + 'get_root_logger', 'set_random_seed', 'train_detector', 'init_detector', + 'async_inference_detector', 'inference_detector', 'show_result_pyplot', + 'multi_gpu_test', 'single_gpu_test' +] diff --git a/detection/mmdet/apis/inference.py b/detection/mmdet/apis/inference.py new file mode 100644 index 0000000..464d1e2 --- /dev/null +++ b/detection/mmdet/apis/inference.py @@ -0,0 +1,217 @@ +import warnings + +import mmcv +import numpy as np +import torch +from mmcv.ops import RoIPool +from mmcv.parallel import collate, scatter +from mmcv.runner import load_checkpoint + +from mmdet.core import get_classes +from mmdet.datasets import replace_ImageToTensor +from mmdet.datasets.pipelines import Compose +from mmdet.models import build_detector + + +def init_detector(config, checkpoint=None, device='cuda:0', cfg_options=None): + """Initialize a detector from config file. + + Args: + config (str or :obj:`mmcv.Config`): Config file path or the config + object. + checkpoint (str, optional): Checkpoint path. If left as None, the model + will not load any weights. + cfg_options (dict): Options to override some settings in the used + config. + + Returns: + nn.Module: The constructed detector. + """ + if isinstance(config, str): + config = mmcv.Config.fromfile(config) + elif not isinstance(config, mmcv.Config): + raise TypeError('config must be a filename or Config object, ' + f'but got {type(config)}') + if cfg_options is not None: + config.merge_from_dict(cfg_options) + config.model.pretrained = None + config.model.train_cfg = None + model = build_detector(config.model, test_cfg=config.get('test_cfg')) + if checkpoint is not None: + map_loc = 'cpu' if device == 'cpu' else None + checkpoint = load_checkpoint(model, checkpoint, map_location=map_loc) + if 'CLASSES' in checkpoint.get('meta', {}): + model.CLASSES = checkpoint['meta']['CLASSES'] + else: + warnings.simplefilter('once') + warnings.warn('Class names are not saved in the checkpoint\'s ' + 'meta data, use COCO classes by default.') + model.CLASSES = get_classes('coco') + model.cfg = config # save the config in the model for convenience + model.to(device) + model.eval() + return model + + +class LoadImage(object): + """Deprecated. + + A simple pipeline to load image. + """ + + def __call__(self, results): + """Call function to load images into results. + + Args: + results (dict): A result dict contains the file name + of the image to be read. + Returns: + dict: ``results`` will be returned containing loaded image. + """ + warnings.simplefilter('once') + warnings.warn('`LoadImage` is deprecated and will be removed in ' + 'future releases. You may use `LoadImageFromWebcam` ' + 'from `mmdet.datasets.pipelines.` instead.') + if isinstance(results['img'], str): + results['filename'] = results['img'] + results['ori_filename'] = results['img'] + else: + results['filename'] = None + results['ori_filename'] = None + img = mmcv.imread(results['img']) + results['img'] = img + results['img_fields'] = ['img'] + results['img_shape'] = img.shape + results['ori_shape'] = img.shape + return results + + +def inference_detector(model, imgs): + """Inference image(s) with the detector. + + Args: + model (nn.Module): The loaded detector. + imgs (str/ndarray or list[str/ndarray] or tuple[str/ndarray]): + Either image files or loaded images. + + Returns: + If imgs is a list or tuple, the same length list type results + will be returned, otherwise return the detection results directly. + """ + + if isinstance(imgs, (list, tuple)): + is_batch = True + else: + imgs = [imgs] + is_batch = False + + cfg = model.cfg + device = next(model.parameters()).device # model device + + if isinstance(imgs[0], np.ndarray): + cfg = cfg.copy() + # set loading pipeline type + cfg.data.test.pipeline[0].type = 'LoadImageFromWebcam' + + cfg.data.test.pipeline = replace_ImageToTensor(cfg.data.test.pipeline) + test_pipeline = Compose(cfg.data.test.pipeline) + + datas = [] + for img in imgs: + # prepare data + if isinstance(img, np.ndarray): + # directly add img + data = dict(img=img) + else: + # add information into dict + data = dict(img_info=dict(filename=img), img_prefix=None) + # build the data pipeline + data = test_pipeline(data) + datas.append(data) + + data = collate(datas, samples_per_gpu=len(imgs)) + # just get the actual data from DataContainer + data['img_metas'] = [img_metas.data[0] for img_metas in data['img_metas']] + data['img'] = [img.data[0] for img in data['img']] + if next(model.parameters()).is_cuda: + # scatter to specified GPU + data = scatter(data, [device])[0] + else: + for m in model.modules(): + assert not isinstance( + m, RoIPool + ), 'CPU inference with RoIPool is not supported currently.' + + # forward the model + with torch.no_grad(): + results = model(return_loss=False, rescale=True, **data) + + if not is_batch: + return results[0] + else: + return results + + +async def async_inference_detector(model, img): + """Async inference image(s) with the detector. + + Args: + model (nn.Module): The loaded detector. + img (str | ndarray): Either image files or loaded images. + + Returns: + Awaitable detection results. + """ + cfg = model.cfg + device = next(model.parameters()).device # model device + # prepare data + if isinstance(img, np.ndarray): + # directly add img + data = dict(img=img) + cfg = cfg.copy() + # set loading pipeline type + cfg.data.test.pipeline[0].type = 'LoadImageFromWebcam' + else: + # add information into dict + data = dict(img_info=dict(filename=img), img_prefix=None) + # build the data pipeline + test_pipeline = Compose(cfg.data.test.pipeline) + data = test_pipeline(data) + data = scatter(collate([data], samples_per_gpu=1), [device])[0] + + # We don't restore `torch.is_grad_enabled()` value during concurrent + # inference since execution can overlap + torch.set_grad_enabled(False) + result = await model.aforward_test(rescale=True, **data) + return result + + +def show_result_pyplot(model, + img, + result, + score_thr=0.3, + title='result', + wait_time=0): + """Visualize the detection results on the image. + + Args: + model (nn.Module): The loaded detector. + img (str or np.ndarray): Image filename or loaded image. + result (tuple[list] or list): The detection result, can be either + (bbox, segm) or just bbox. + score_thr (float): The threshold to visualize the bboxes and masks. + title (str): Title of the pyplot figure. + wait_time (float): Value of waitKey param. + Default: 0. + """ + if hasattr(model, 'module'): + model = model.module + model.show_result( + img, + result, + score_thr=score_thr, + show=True, + wait_time=wait_time, + win_name=title, + bbox_color=(72, 101, 241), + text_color=(72, 101, 241)) diff --git a/detection/mmdet/apis/test.py b/detection/mmdet/apis/test.py new file mode 100644 index 0000000..e54b1b8 --- /dev/null +++ b/detection/mmdet/apis/test.py @@ -0,0 +1,190 @@ +import os.path as osp +import pickle +import shutil +import tempfile +import time + +import mmcv +import torch +import torch.distributed as dist +from mmcv.image import tensor2imgs +from mmcv.runner import get_dist_info + +from mmdet.core import encode_mask_results + + +def single_gpu_test(model, + data_loader, + show=False, + out_dir=None, + show_score_thr=0.3): + model.eval() + results = [] + dataset = data_loader.dataset + prog_bar = mmcv.ProgressBar(len(dataset)) + for i, data in enumerate(data_loader): + with torch.no_grad(): + result = model(return_loss=False, rescale=True, **data) + + batch_size = len(result) + if show or out_dir: + if batch_size == 1 and isinstance(data['img'][0], torch.Tensor): + img_tensor = data['img'][0] + else: + img_tensor = data['img'][0].data[0] + img_metas = data['img_metas'][0].data[0] + imgs = tensor2imgs(img_tensor, **img_metas[0]['img_norm_cfg']) + assert len(imgs) == len(img_metas) + + for i, (img, img_meta) in enumerate(zip(imgs, img_metas)): + h, w, _ = img_meta['img_shape'] + img_show = img[:h, :w, :] + + ori_h, ori_w = img_meta['ori_shape'][:-1] + img_show = mmcv.imresize(img_show, (ori_w, ori_h)) + + if out_dir: + out_file = osp.join(out_dir, img_meta['ori_filename']) + else: + out_file = None + + model.module.show_result( + img_show, + result[i], + show=show, + out_file=out_file, + score_thr=show_score_thr) + + # encode mask results + if isinstance(result[0], tuple): + result = [(bbox_results, encode_mask_results(mask_results)) + for bbox_results, mask_results in result] + results.extend(result) + + for _ in range(batch_size): + prog_bar.update() + return results + + +def multi_gpu_test(model, data_loader, tmpdir=None, gpu_collect=False): + """Test model with multiple gpus. + + This method tests model with multiple gpus and collects the results + under two different modes: gpu and cpu modes. By setting 'gpu_collect=True' + it encodes results to gpu tensors and use gpu communication for results + collection. On cpu mode it saves the results on different gpus to 'tmpdir' + and collects them by the rank 0 worker. + + Args: + model (nn.Module): Model to be tested. + data_loader (nn.Dataloader): Pytorch data loader. + tmpdir (str): Path of directory to save the temporary results from + different gpus under cpu mode. + gpu_collect (bool): Option to use either gpu or cpu to collect results. + + Returns: + list: The prediction results. + """ + model.eval() + results = [] + dataset = data_loader.dataset + rank, world_size = get_dist_info() + if rank == 0: + prog_bar = mmcv.ProgressBar(len(dataset)) + time.sleep(2) # This line can prevent deadlock problem in some cases. + for i, data in enumerate(data_loader): + with torch.no_grad(): + result = model(return_loss=False, rescale=True, **data) + # encode mask results + if isinstance(result[0], tuple): + result = [(bbox_results, encode_mask_results(mask_results)) + for bbox_results, mask_results in result] + results.extend(result) + + if rank == 0: + batch_size = len(result) + for _ in range(batch_size * world_size): + prog_bar.update() + + # collect results from all ranks + if gpu_collect: + results = collect_results_gpu(results, len(dataset)) + else: + results = collect_results_cpu(results, len(dataset), tmpdir) + return results + + +def collect_results_cpu(result_part, size, tmpdir=None): + rank, world_size = get_dist_info() + # create a tmp dir if it is not specified + if tmpdir is None: + MAX_LEN = 512 + # 32 is whitespace + dir_tensor = torch.full((MAX_LEN, ), + 32, + dtype=torch.uint8, + device='cuda') + if rank == 0: + mmcv.mkdir_or_exist('.dist_test') + tmpdir = tempfile.mkdtemp(dir='.dist_test') + tmpdir = torch.tensor( + bytearray(tmpdir.encode()), dtype=torch.uint8, device='cuda') + dir_tensor[:len(tmpdir)] = tmpdir + dist.broadcast(dir_tensor, 0) + tmpdir = dir_tensor.cpu().numpy().tobytes().decode().rstrip() + else: + mmcv.mkdir_or_exist(tmpdir) + # dump the part result to the dir + mmcv.dump(result_part, osp.join(tmpdir, f'part_{rank}.pkl')) + dist.barrier() + # collect all parts + if rank != 0: + return None + else: + # load results of all parts from tmp dir + part_list = [] + for i in range(world_size): + part_file = osp.join(tmpdir, f'part_{i}.pkl') + part_list.append(mmcv.load(part_file)) + # sort the results + ordered_results = [] + for res in zip(*part_list): + ordered_results.extend(list(res)) + # the dataloader may pad some samples + ordered_results = ordered_results[:size] + # remove tmp dir + shutil.rmtree(tmpdir) + return ordered_results + + +def collect_results_gpu(result_part, size): + rank, world_size = get_dist_info() + # dump result part to tensor with pickle + part_tensor = torch.tensor( + bytearray(pickle.dumps(result_part)), dtype=torch.uint8, device='cuda') + # gather all result part tensor shape + shape_tensor = torch.tensor(part_tensor.shape, device='cuda') + shape_list = [shape_tensor.clone() for _ in range(world_size)] + dist.all_gather(shape_list, shape_tensor) + # padding result part tensor to max length + shape_max = torch.tensor(shape_list).max() + part_send = torch.zeros(shape_max, dtype=torch.uint8, device='cuda') + part_send[:shape_tensor[0]] = part_tensor + part_recv_list = [ + part_tensor.new_zeros(shape_max) for _ in range(world_size) + ] + # gather all result part + dist.all_gather(part_recv_list, part_send) + + if rank == 0: + part_list = [] + for recv, shape in zip(part_recv_list, shape_list): + part_list.append( + pickle.loads(recv[:shape[0]].cpu().numpy().tobytes())) + # sort the results + ordered_results = [] + for res in zip(*part_list): + ordered_results.extend(list(res)) + # the dataloader may pad some samples + ordered_results = ordered_results[:size] + return ordered_results diff --git a/detection/mmdet/apis/train.py b/detection/mmdet/apis/train.py new file mode 100644 index 0000000..7f2f1f9 --- /dev/null +++ b/detection/mmdet/apis/train.py @@ -0,0 +1,185 @@ +import random +import warnings + +import numpy as np +import torch +from mmcv.parallel import MMDataParallel, MMDistributedDataParallel +from mmcv.runner import (HOOKS, DistSamplerSeedHook, EpochBasedRunner, + Fp16OptimizerHook, OptimizerHook, build_optimizer, + build_runner) +from mmcv.utils import build_from_cfg + +from mmdet.core import DistEvalHook, EvalHook +from mmdet.datasets import (build_dataloader, build_dataset, + replace_ImageToTensor) +from mmdet.utils import get_root_logger +from mmcv_custom.runner import EpochBasedRunnerAmp +try: + import apex +except: + print('apex is not installed') + + +def set_random_seed(seed, deterministic=False): + """Set random seed. + + Args: + seed (int): Seed to be used. + deterministic (bool): Whether to set the deterministic option for + CUDNN backend, i.e., set `torch.backends.cudnn.deterministic` + to True and `torch.backends.cudnn.benchmark` to False. + Default: False. + """ + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + if deterministic: + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + + +def train_detector(model, + dataset, + cfg, + distributed=False, + validate=False, + timestamp=None, + meta=None): + logger = get_root_logger(cfg.log_level) + + # prepare data loaders + dataset = dataset if isinstance(dataset, (list, tuple)) else [dataset] + if 'imgs_per_gpu' in cfg.data: + logger.warning('"imgs_per_gpu" is deprecated in MMDet V2.0. ' + 'Please use "samples_per_gpu" instead') + if 'samples_per_gpu' in cfg.data: + logger.warning( + f'Got "imgs_per_gpu"={cfg.data.imgs_per_gpu} and ' + f'"samples_per_gpu"={cfg.data.samples_per_gpu}, "imgs_per_gpu"' + f'={cfg.data.imgs_per_gpu} is used in this experiments') + else: + logger.warning( + 'Automatically set "samples_per_gpu"="imgs_per_gpu"=' + f'{cfg.data.imgs_per_gpu} in this experiments') + cfg.data.samples_per_gpu = cfg.data.imgs_per_gpu + + data_loaders = [ + build_dataloader( + ds, + cfg.data.samples_per_gpu, + cfg.data.workers_per_gpu, + # cfg.gpus will be ignored if distributed + len(cfg.gpu_ids), + dist=distributed, + seed=cfg.seed) for ds in dataset + ] + + # build optimizer + optimizer = build_optimizer(model, cfg.optimizer) + + # use apex fp16 optimizer + if cfg.optimizer_config.get("type", None) and cfg.optimizer_config["type"] == "DistOptimizerHook": + if cfg.optimizer_config.get("use_fp16", False): + model, optimizer = apex.amp.initialize( + model.cuda(), optimizer, opt_level="O1") + for m in model.modules(): + if hasattr(m, "fp16_enabled"): + m.fp16_enabled = True + + # put model on gpus + if distributed: + find_unused_parameters = cfg.get('find_unused_parameters', False) + # Sets the `find_unused_parameters` parameter in + # torch.nn.parallel.DistributedDataParallel + model = MMDistributedDataParallel( + model.cuda(), + device_ids=[torch.cuda.current_device()], + broadcast_buffers=False, + find_unused_parameters=find_unused_parameters) + else: + model = MMDataParallel( + model.cuda(cfg.gpu_ids[0]), device_ids=cfg.gpu_ids) + + if 'runner' not in cfg: + cfg.runner = { + 'type': 'EpochBasedRunner', + 'max_epochs': cfg.total_epochs + } + warnings.warn( + 'config is now expected to have a `runner` section, ' + 'please set `runner` in your config.', UserWarning) + else: + if 'total_epochs' in cfg: + assert cfg.total_epochs == cfg.runner.max_epochs + + # build runner + runner = build_runner( + cfg.runner, + default_args=dict( + model=model, + optimizer=optimizer, + work_dir=cfg.work_dir, + logger=logger, + meta=meta)) + + # an ugly workaround to make .log and .log.json filenames the same + runner.timestamp = timestamp + + # fp16 setting + fp16_cfg = cfg.get('fp16', None) + if fp16_cfg is not None: + optimizer_config = Fp16OptimizerHook( + **cfg.optimizer_config, **fp16_cfg, distributed=distributed) + elif distributed and 'type' not in cfg.optimizer_config: + optimizer_config = OptimizerHook(**cfg.optimizer_config) + else: + optimizer_config = cfg.optimizer_config + + # register hooks + runner.register_training_hooks(cfg.lr_config, optimizer_config, + cfg.checkpoint_config, cfg.log_config, + cfg.get('momentum_config', None)) + if distributed: + if isinstance(runner, EpochBasedRunner): + runner.register_hook(DistSamplerSeedHook()) + + # register eval hooks + if validate: + # Support batch_size > 1 in validation + val_samples_per_gpu = cfg.data.val.pop('samples_per_gpu', 1) + if val_samples_per_gpu > 1: + # Replace 'ImageToTensor' to 'DefaultFormatBundle' + cfg.data.val.pipeline = replace_ImageToTensor( + cfg.data.val.pipeline) + val_dataset = build_dataset(cfg.data.val, dict(test_mode=True)) + val_dataloader = build_dataloader( + val_dataset, + samples_per_gpu=val_samples_per_gpu, + workers_per_gpu=cfg.data.workers_per_gpu, + dist=distributed, + shuffle=False) + eval_cfg = cfg.get('evaluation', {}) + eval_cfg['by_epoch'] = cfg.runner['type'] != 'IterBasedRunner' + eval_hook = DistEvalHook if distributed else EvalHook + runner.register_hook(eval_hook(val_dataloader, **eval_cfg)) + + # user-defined hooks + if cfg.get('custom_hooks', None): + custom_hooks = cfg.custom_hooks + assert isinstance(custom_hooks, list), \ + f'custom_hooks expect list type, but got {type(custom_hooks)}' + for hook_cfg in cfg.custom_hooks: + assert isinstance(hook_cfg, dict), \ + 'Each item in custom_hooks expects dict type, but got ' \ + f'{type(hook_cfg)}' + hook_cfg = hook_cfg.copy() + priority = hook_cfg.pop('priority', 'NORMAL') + hook = build_from_cfg(hook_cfg, HOOKS) + runner.register_hook(hook, priority=priority) + + if cfg.resume_from: + runner.resume(cfg.resume_from) + elif cfg.load_from: + runner.load_checkpoint(cfg.load_from) + runner.run(data_loaders, cfg.workflow) diff --git a/detection/mmdet/core/__init__.py b/detection/mmdet/core/__init__.py new file mode 100644 index 0000000..e812391 --- /dev/null +++ b/detection/mmdet/core/__init__.py @@ -0,0 +1,7 @@ +from .anchor import * # noqa: F401, F403 +from .bbox import * # noqa: F401, F403 +from .evaluation import * # noqa: F401, F403 +from .export import * # noqa: F401, F403 +from .mask import * # noqa: F401, F403 +from .post_processing import * # noqa: F401, F403 +from .utils import * # noqa: F401, F403 diff --git a/detection/mmdet/core/anchor/__init__.py b/detection/mmdet/core/anchor/__init__.py new file mode 100644 index 0000000..5838ff3 --- /dev/null +++ b/detection/mmdet/core/anchor/__init__.py @@ -0,0 +1,11 @@ +from .anchor_generator import (AnchorGenerator, LegacyAnchorGenerator, + YOLOAnchorGenerator) +from .builder import ANCHOR_GENERATORS, build_anchor_generator +from .point_generator import PointGenerator +from .utils import anchor_inside_flags, calc_region, images_to_levels + +__all__ = [ + 'AnchorGenerator', 'LegacyAnchorGenerator', 'anchor_inside_flags', + 'PointGenerator', 'images_to_levels', 'calc_region', + 'build_anchor_generator', 'ANCHOR_GENERATORS', 'YOLOAnchorGenerator' +] diff --git a/detection/mmdet/core/anchor/anchor_generator.py b/detection/mmdet/core/anchor/anchor_generator.py new file mode 100644 index 0000000..388d260 --- /dev/null +++ b/detection/mmdet/core/anchor/anchor_generator.py @@ -0,0 +1,727 @@ +import mmcv +import numpy as np +import torch +from torch.nn.modules.utils import _pair + +from .builder import ANCHOR_GENERATORS + + +@ANCHOR_GENERATORS.register_module() +class AnchorGenerator(object): + """Standard anchor generator for 2D anchor-based detectors. + + Args: + strides (list[int] | list[tuple[int, int]]): Strides of anchors + in multiple feature levels in order (w, h). + ratios (list[float]): The list of ratios between the height and width + of anchors in a single level. + scales (list[int] | None): Anchor scales for anchors in a single level. + It cannot be set at the same time if `octave_base_scale` and + `scales_per_octave` are set. + base_sizes (list[int] | None): The basic sizes + of anchors in multiple levels. + If None is given, strides will be used as base_sizes. + (If strides are non square, the shortest stride is taken.) + scale_major (bool): Whether to multiply scales first when generating + base anchors. If true, the anchors in the same row will have the + same scales. By default it is True in V2.0 + octave_base_scale (int): The base scale of octave. + scales_per_octave (int): Number of scales for each octave. + `octave_base_scale` and `scales_per_octave` are usually used in + retinanet and the `scales` should be None when they are set. + centers (list[tuple[float, float]] | None): The centers of the anchor + relative to the feature grid center in multiple feature levels. + By default it is set to be None and not used. If a list of tuple of + float is given, they will be used to shift the centers of anchors. + center_offset (float): The offset of center in proportion to anchors' + width and height. By default it is 0 in V2.0. + + Examples: + >>> from mmdet.core import AnchorGenerator + >>> self = AnchorGenerator([16], [1.], [1.], [9]) + >>> all_anchors = self.grid_anchors([(2, 2)], device='cpu') + >>> print(all_anchors) + [tensor([[-4.5000, -4.5000, 4.5000, 4.5000], + [11.5000, -4.5000, 20.5000, 4.5000], + [-4.5000, 11.5000, 4.5000, 20.5000], + [11.5000, 11.5000, 20.5000, 20.5000]])] + >>> self = AnchorGenerator([16, 32], [1.], [1.], [9, 18]) + >>> all_anchors = self.grid_anchors([(2, 2), (1, 1)], device='cpu') + >>> print(all_anchors) + [tensor([[-4.5000, -4.5000, 4.5000, 4.5000], + [11.5000, -4.5000, 20.5000, 4.5000], + [-4.5000, 11.5000, 4.5000, 20.5000], + [11.5000, 11.5000, 20.5000, 20.5000]]), \ + tensor([[-9., -9., 9., 9.]])] + """ + + def __init__(self, + strides, + ratios, + scales=None, + base_sizes=None, + scale_major=True, + octave_base_scale=None, + scales_per_octave=None, + centers=None, + center_offset=0.): + # check center and center_offset + if center_offset != 0: + assert centers is None, 'center cannot be set when center_offset' \ + f'!=0, {centers} is given.' + if not (0 <= center_offset <= 1): + raise ValueError('center_offset should be in range [0, 1], ' + f'{center_offset} is given.') + if centers is not None: + assert len(centers) == len(strides), \ + 'The number of strides should be the same as centers, got ' \ + f'{strides} and {centers}' + + # calculate base sizes of anchors + self.strides = [_pair(stride) for stride in strides] + self.base_sizes = [min(stride) for stride in self.strides + ] if base_sizes is None else base_sizes + assert len(self.base_sizes) == len(self.strides), \ + 'The number of strides should be the same as base sizes, got ' \ + f'{self.strides} and {self.base_sizes}' + + # calculate scales of anchors + assert ((octave_base_scale is not None + and scales_per_octave is not None) ^ (scales is not None)), \ + 'scales and octave_base_scale with scales_per_octave cannot' \ + ' be set at the same time' + if scales is not None: + self.scales = torch.Tensor(scales) + elif octave_base_scale is not None and scales_per_octave is not None: + octave_scales = np.array( + [2**(i / scales_per_octave) for i in range(scales_per_octave)]) + scales = octave_scales * octave_base_scale + self.scales = torch.Tensor(scales) + else: + raise ValueError('Either scales or octave_base_scale with ' + 'scales_per_octave should be set') + + self.octave_base_scale = octave_base_scale + self.scales_per_octave = scales_per_octave + self.ratios = torch.Tensor(ratios) + self.scale_major = scale_major + self.centers = centers + self.center_offset = center_offset + self.base_anchors = self.gen_base_anchors() + + @property + def num_base_anchors(self): + """list[int]: total number of base anchors in a feature grid""" + return [base_anchors.size(0) for base_anchors in self.base_anchors] + + @property + def num_levels(self): + """int: number of feature levels that the generator will be applied""" + return len(self.strides) + + def gen_base_anchors(self): + """Generate base anchors. + + Returns: + list(torch.Tensor): Base anchors of a feature grid in multiple \ + feature levels. + """ + multi_level_base_anchors = [] + for i, base_size in enumerate(self.base_sizes): + center = None + if self.centers is not None: + center = self.centers[i] + multi_level_base_anchors.append( + self.gen_single_level_base_anchors( + base_size, + scales=self.scales, + ratios=self.ratios, + center=center)) + return multi_level_base_anchors + + def gen_single_level_base_anchors(self, + base_size, + scales, + ratios, + center=None): + """Generate base anchors of a single level. + + Args: + base_size (int | float): Basic size of an anchor. + scales (torch.Tensor): Scales of the anchor. + ratios (torch.Tensor): The ratio between between the height + and width of anchors in a single level. + center (tuple[float], optional): The center of the base anchor + related to a single feature grid. Defaults to None. + + Returns: + torch.Tensor: Anchors in a single-level feature maps. + """ + w = base_size + h = base_size + if center is None: + x_center = self.center_offset * w + y_center = self.center_offset * h + else: + x_center, y_center = center + + h_ratios = torch.sqrt(ratios) + w_ratios = 1 / h_ratios + if self.scale_major: + ws = (w * w_ratios[:, None] * scales[None, :]).view(-1) + hs = (h * h_ratios[:, None] * scales[None, :]).view(-1) + else: + ws = (w * scales[:, None] * w_ratios[None, :]).view(-1) + hs = (h * scales[:, None] * h_ratios[None, :]).view(-1) + + # use float anchor and the anchor's center is aligned with the + # pixel center + base_anchors = [ + x_center - 0.5 * ws, y_center - 0.5 * hs, x_center + 0.5 * ws, + y_center + 0.5 * hs + ] + base_anchors = torch.stack(base_anchors, dim=-1) + + return base_anchors + + def _meshgrid(self, x, y, row_major=True): + """Generate mesh grid of x and y. + + Args: + x (torch.Tensor): Grids of x dimension. + y (torch.Tensor): Grids of y dimension. + row_major (bool, optional): Whether to return y grids first. + Defaults to True. + + Returns: + tuple[torch.Tensor]: The mesh grids of x and y. + """ + # use shape instead of len to keep tracing while exporting to onnx + xx = x.repeat(y.shape[0]) + yy = y.view(-1, 1).repeat(1, x.shape[0]).view(-1) + if row_major: + return xx, yy + else: + return yy, xx + + def grid_anchors(self, featmap_sizes, device='cuda'): + """Generate grid anchors in multiple feature levels. + + Args: + featmap_sizes (list[tuple]): List of feature map sizes in + multiple feature levels. + device (str): Device where the anchors will be put on. + + Return: + list[torch.Tensor]: Anchors in multiple feature levels. \ + The sizes of each tensor should be [N, 4], where \ + N = width * height * num_base_anchors, width and height \ + are the sizes of the corresponding feature level, \ + num_base_anchors is the number of anchors for that level. + """ + assert self.num_levels == len(featmap_sizes) + multi_level_anchors = [] + for i in range(self.num_levels): + anchors = self.single_level_grid_anchors( + self.base_anchors[i].to(device), + featmap_sizes[i], + self.strides[i], + device=device) + multi_level_anchors.append(anchors) + return multi_level_anchors + + def single_level_grid_anchors(self, + base_anchors, + featmap_size, + stride=(16, 16), + device='cuda'): + """Generate grid anchors of a single level. + + Note: + This function is usually called by method ``self.grid_anchors``. + + Args: + base_anchors (torch.Tensor): The base anchors of a feature grid. + featmap_size (tuple[int]): Size of the feature maps. + stride (tuple[int], optional): Stride of the feature map in order + (w, h). Defaults to (16, 16). + device (str, optional): Device the tensor will be put on. + Defaults to 'cuda'. + + Returns: + torch.Tensor: Anchors in the overall feature maps. + """ + # keep as Tensor, so that we can covert to ONNX correctly + feat_h, feat_w = featmap_size + shift_x = torch.arange(0, feat_w, device=device) * stride[0] + shift_y = torch.arange(0, feat_h, device=device) * stride[1] + + shift_xx, shift_yy = self._meshgrid(shift_x, shift_y) + shifts = torch.stack([shift_xx, shift_yy, shift_xx, shift_yy], dim=-1) + shifts = shifts.type_as(base_anchors) + # first feat_w elements correspond to the first row of shifts + # add A anchors (1, A, 4) to K shifts (K, 1, 4) to get + # shifted anchors (K, A, 4), reshape to (K*A, 4) + + all_anchors = base_anchors[None, :, :] + shifts[:, None, :] + all_anchors = all_anchors.view(-1, 4) + # first A rows correspond to A anchors of (0, 0) in feature map, + # then (0, 1), (0, 2), ... + return all_anchors + + def valid_flags(self, featmap_sizes, pad_shape, device='cuda'): + """Generate valid flags of anchors in multiple feature levels. + + Args: + featmap_sizes (list(tuple)): List of feature map sizes in + multiple feature levels. + pad_shape (tuple): The padded shape of the image. + device (str): Device where the anchors will be put on. + + Return: + list(torch.Tensor): Valid flags of anchors in multiple levels. + """ + assert self.num_levels == len(featmap_sizes) + multi_level_flags = [] + for i in range(self.num_levels): + anchor_stride = self.strides[i] + feat_h, feat_w = featmap_sizes[i] + h, w = pad_shape[:2] + valid_feat_h = min(int(np.ceil(h / anchor_stride[1])), feat_h) + valid_feat_w = min(int(np.ceil(w / anchor_stride[0])), feat_w) + flags = self.single_level_valid_flags((feat_h, feat_w), + (valid_feat_h, valid_feat_w), + self.num_base_anchors[i], + device=device) + multi_level_flags.append(flags) + return multi_level_flags + + def single_level_valid_flags(self, + featmap_size, + valid_size, + num_base_anchors, + device='cuda'): + """Generate the valid flags of anchor in a single feature map. + + Args: + featmap_size (tuple[int]): The size of feature maps. + valid_size (tuple[int]): The valid size of the feature maps. + num_base_anchors (int): The number of base anchors. + device (str, optional): Device where the flags will be put on. + Defaults to 'cuda'. + + Returns: + torch.Tensor: The valid flags of each anchor in a single level \ + feature map. + """ + feat_h, feat_w = featmap_size + valid_h, valid_w = valid_size + assert valid_h <= feat_h and valid_w <= feat_w + valid_x = torch.zeros(feat_w, dtype=torch.bool, device=device) + valid_y = torch.zeros(feat_h, dtype=torch.bool, device=device) + valid_x[:valid_w] = 1 + valid_y[:valid_h] = 1 + valid_xx, valid_yy = self._meshgrid(valid_x, valid_y) + valid = valid_xx & valid_yy + valid = valid[:, None].expand(valid.size(0), + num_base_anchors).contiguous().view(-1) + return valid + + def __repr__(self): + """str: a string that describes the module""" + indent_str = ' ' + repr_str = self.__class__.__name__ + '(\n' + repr_str += f'{indent_str}strides={self.strides},\n' + repr_str += f'{indent_str}ratios={self.ratios},\n' + repr_str += f'{indent_str}scales={self.scales},\n' + repr_str += f'{indent_str}base_sizes={self.base_sizes},\n' + repr_str += f'{indent_str}scale_major={self.scale_major},\n' + repr_str += f'{indent_str}octave_base_scale=' + repr_str += f'{self.octave_base_scale},\n' + repr_str += f'{indent_str}scales_per_octave=' + repr_str += f'{self.scales_per_octave},\n' + repr_str += f'{indent_str}num_levels={self.num_levels}\n' + repr_str += f'{indent_str}centers={self.centers},\n' + repr_str += f'{indent_str}center_offset={self.center_offset})' + return repr_str + + +@ANCHOR_GENERATORS.register_module() +class SSDAnchorGenerator(AnchorGenerator): + """Anchor generator for SSD. + + Args: + strides (list[int] | list[tuple[int, int]]): Strides of anchors + in multiple feature levels. + ratios (list[float]): The list of ratios between the height and width + of anchors in a single level. + basesize_ratio_range (tuple(float)): Ratio range of anchors. + input_size (int): Size of feature map, 300 for SSD300, + 512 for SSD512. + scale_major (bool): Whether to multiply scales first when generating + base anchors. If true, the anchors in the same row will have the + same scales. It is always set to be False in SSD. + """ + + def __init__(self, + strides, + ratios, + basesize_ratio_range, + input_size=300, + scale_major=True): + assert len(strides) == len(ratios) + assert mmcv.is_tuple_of(basesize_ratio_range, float) + + self.strides = [_pair(stride) for stride in strides] + self.input_size = input_size + self.centers = [(stride[0] / 2., stride[1] / 2.) + for stride in self.strides] + self.basesize_ratio_range = basesize_ratio_range + + # calculate anchor ratios and sizes + min_ratio, max_ratio = basesize_ratio_range + min_ratio = int(min_ratio * 100) + max_ratio = int(max_ratio * 100) + step = int(np.floor(max_ratio - min_ratio) / (self.num_levels - 2)) + min_sizes = [] + max_sizes = [] + for ratio in range(int(min_ratio), int(max_ratio) + 1, step): + min_sizes.append(int(self.input_size * ratio / 100)) + max_sizes.append(int(self.input_size * (ratio + step) / 100)) + if self.input_size == 300: + if basesize_ratio_range[0] == 0.15: # SSD300 COCO + min_sizes.insert(0, int(self.input_size * 7 / 100)) + max_sizes.insert(0, int(self.input_size * 15 / 100)) + elif basesize_ratio_range[0] == 0.2: # SSD300 VOC + min_sizes.insert(0, int(self.input_size * 10 / 100)) + max_sizes.insert(0, int(self.input_size * 20 / 100)) + else: + raise ValueError( + 'basesize_ratio_range[0] should be either 0.15' + 'or 0.2 when input_size is 300, got ' + f'{basesize_ratio_range[0]}.') + elif self.input_size == 512: + if basesize_ratio_range[0] == 0.1: # SSD512 COCO + min_sizes.insert(0, int(self.input_size * 4 / 100)) + max_sizes.insert(0, int(self.input_size * 10 / 100)) + elif basesize_ratio_range[0] == 0.15: # SSD512 VOC + min_sizes.insert(0, int(self.input_size * 7 / 100)) + max_sizes.insert(0, int(self.input_size * 15 / 100)) + else: + raise ValueError('basesize_ratio_range[0] should be either 0.1' + 'or 0.15 when input_size is 512, got' + f' {basesize_ratio_range[0]}.') + else: + raise ValueError('Only support 300 or 512 in SSDAnchorGenerator' + f', got {self.input_size}.') + + anchor_ratios = [] + anchor_scales = [] + for k in range(len(self.strides)): + scales = [1., np.sqrt(max_sizes[k] / min_sizes[k])] + anchor_ratio = [1.] + for r in ratios[k]: + anchor_ratio += [1 / r, r] # 4 or 6 ratio + anchor_ratios.append(torch.Tensor(anchor_ratio)) + anchor_scales.append(torch.Tensor(scales)) + + self.base_sizes = min_sizes + self.scales = anchor_scales + self.ratios = anchor_ratios + self.scale_major = scale_major + self.center_offset = 0 + self.base_anchors = self.gen_base_anchors() + + def gen_base_anchors(self): + """Generate base anchors. + + Returns: + list(torch.Tensor): Base anchors of a feature grid in multiple \ + feature levels. + """ + multi_level_base_anchors = [] + for i, base_size in enumerate(self.base_sizes): + base_anchors = self.gen_single_level_base_anchors( + base_size, + scales=self.scales[i], + ratios=self.ratios[i], + center=self.centers[i]) + indices = list(range(len(self.ratios[i]))) + indices.insert(1, len(indices)) + base_anchors = torch.index_select(base_anchors, 0, + torch.LongTensor(indices)) + multi_level_base_anchors.append(base_anchors) + return multi_level_base_anchors + + def __repr__(self): + """str: a string that describes the module""" + indent_str = ' ' + repr_str = self.__class__.__name__ + '(\n' + repr_str += f'{indent_str}strides={self.strides},\n' + repr_str += f'{indent_str}scales={self.scales},\n' + repr_str += f'{indent_str}scale_major={self.scale_major},\n' + repr_str += f'{indent_str}input_size={self.input_size},\n' + repr_str += f'{indent_str}scales={self.scales},\n' + repr_str += f'{indent_str}ratios={self.ratios},\n' + repr_str += f'{indent_str}num_levels={self.num_levels},\n' + repr_str += f'{indent_str}base_sizes={self.base_sizes},\n' + repr_str += f'{indent_str}basesize_ratio_range=' + repr_str += f'{self.basesize_ratio_range})' + return repr_str + + +@ANCHOR_GENERATORS.register_module() +class LegacyAnchorGenerator(AnchorGenerator): + """Legacy anchor generator used in MMDetection V1.x. + + Note: + Difference to the V2.0 anchor generator: + + 1. The center offset of V1.x anchors are set to be 0.5 rather than 0. + 2. The width/height are minused by 1 when calculating the anchors' \ + centers and corners to meet the V1.x coordinate system. + 3. The anchors' corners are quantized. + + Args: + strides (list[int] | list[tuple[int]]): Strides of anchors + in multiple feature levels. + ratios (list[float]): The list of ratios between the height and width + of anchors in a single level. + scales (list[int] | None): Anchor scales for anchors in a single level. + It cannot be set at the same time if `octave_base_scale` and + `scales_per_octave` are set. + base_sizes (list[int]): The basic sizes of anchors in multiple levels. + If None is given, strides will be used to generate base_sizes. + scale_major (bool): Whether to multiply scales first when generating + base anchors. If true, the anchors in the same row will have the + same scales. By default it is True in V2.0 + octave_base_scale (int): The base scale of octave. + scales_per_octave (int): Number of scales for each octave. + `octave_base_scale` and `scales_per_octave` are usually used in + retinanet and the `scales` should be None when they are set. + centers (list[tuple[float, float]] | None): The centers of the anchor + relative to the feature grid center in multiple feature levels. + By default it is set to be None and not used. It a list of float + is given, this list will be used to shift the centers of anchors. + center_offset (float): The offset of center in propotion to anchors' + width and height. By default it is 0.5 in V2.0 but it should be 0.5 + in v1.x models. + + Examples: + >>> from mmdet.core import LegacyAnchorGenerator + >>> self = LegacyAnchorGenerator( + >>> [16], [1.], [1.], [9], center_offset=0.5) + >>> all_anchors = self.grid_anchors(((2, 2),), device='cpu') + >>> print(all_anchors) + [tensor([[ 0., 0., 8., 8.], + [16., 0., 24., 8.], + [ 0., 16., 8., 24.], + [16., 16., 24., 24.]])] + """ + + def gen_single_level_base_anchors(self, + base_size, + scales, + ratios, + center=None): + """Generate base anchors of a single level. + + Note: + The width/height of anchors are minused by 1 when calculating \ + the centers and corners to meet the V1.x coordinate system. + + Args: + base_size (int | float): Basic size of an anchor. + scales (torch.Tensor): Scales of the anchor. + ratios (torch.Tensor): The ratio between between the height. + and width of anchors in a single level. + center (tuple[float], optional): The center of the base anchor + related to a single feature grid. Defaults to None. + + Returns: + torch.Tensor: Anchors in a single-level feature map. + """ + w = base_size + h = base_size + if center is None: + x_center = self.center_offset * (w - 1) + y_center = self.center_offset * (h - 1) + else: + x_center, y_center = center + + h_ratios = torch.sqrt(ratios) + w_ratios = 1 / h_ratios + if self.scale_major: + ws = (w * w_ratios[:, None] * scales[None, :]).view(-1) + hs = (h * h_ratios[:, None] * scales[None, :]).view(-1) + else: + ws = (w * scales[:, None] * w_ratios[None, :]).view(-1) + hs = (h * scales[:, None] * h_ratios[None, :]).view(-1) + + # use float anchor and the anchor's center is aligned with the + # pixel center + base_anchors = [ + x_center - 0.5 * (ws - 1), y_center - 0.5 * (hs - 1), + x_center + 0.5 * (ws - 1), y_center + 0.5 * (hs - 1) + ] + base_anchors = torch.stack(base_anchors, dim=-1).round() + + return base_anchors + + +@ANCHOR_GENERATORS.register_module() +class LegacySSDAnchorGenerator(SSDAnchorGenerator, LegacyAnchorGenerator): + """Legacy anchor generator used in MMDetection V1.x. + + The difference between `LegacySSDAnchorGenerator` and `SSDAnchorGenerator` + can be found in `LegacyAnchorGenerator`. + """ + + def __init__(self, + strides, + ratios, + basesize_ratio_range, + input_size=300, + scale_major=True): + super(LegacySSDAnchorGenerator, + self).__init__(strides, ratios, basesize_ratio_range, input_size, + scale_major) + self.centers = [((stride - 1) / 2., (stride - 1) / 2.) + for stride in strides] + self.base_anchors = self.gen_base_anchors() + + +@ANCHOR_GENERATORS.register_module() +class YOLOAnchorGenerator(AnchorGenerator): + """Anchor generator for YOLO. + + Args: + strides (list[int] | list[tuple[int, int]]): Strides of anchors + in multiple feature levels. + base_sizes (list[list[tuple[int, int]]]): The basic sizes + of anchors in multiple levels. + """ + + def __init__(self, strides, base_sizes): + self.strides = [_pair(stride) for stride in strides] + self.centers = [(stride[0] / 2., stride[1] / 2.) + for stride in self.strides] + self.base_sizes = [] + num_anchor_per_level = len(base_sizes[0]) + for base_sizes_per_level in base_sizes: + assert num_anchor_per_level == len(base_sizes_per_level) + self.base_sizes.append( + [_pair(base_size) for base_size in base_sizes_per_level]) + self.base_anchors = self.gen_base_anchors() + + @property + def num_levels(self): + """int: number of feature levels that the generator will be applied""" + return len(self.base_sizes) + + def gen_base_anchors(self): + """Generate base anchors. + + Returns: + list(torch.Tensor): Base anchors of a feature grid in multiple \ + feature levels. + """ + multi_level_base_anchors = [] + for i, base_sizes_per_level in enumerate(self.base_sizes): + center = None + if self.centers is not None: + center = self.centers[i] + multi_level_base_anchors.append( + self.gen_single_level_base_anchors(base_sizes_per_level, + center)) + return multi_level_base_anchors + + def gen_single_level_base_anchors(self, base_sizes_per_level, center=None): + """Generate base anchors of a single level. + + Args: + base_sizes_per_level (list[tuple[int, int]]): Basic sizes of + anchors. + center (tuple[float], optional): The center of the base anchor + related to a single feature grid. Defaults to None. + + Returns: + torch.Tensor: Anchors in a single-level feature maps. + """ + x_center, y_center = center + base_anchors = [] + for base_size in base_sizes_per_level: + w, h = base_size + + # use float anchor and the anchor's center is aligned with the + # pixel center + base_anchor = torch.Tensor([ + x_center - 0.5 * w, y_center - 0.5 * h, x_center + 0.5 * w, + y_center + 0.5 * h + ]) + base_anchors.append(base_anchor) + base_anchors = torch.stack(base_anchors, dim=0) + + return base_anchors + + def responsible_flags(self, featmap_sizes, gt_bboxes, device='cuda'): + """Generate responsible anchor flags of grid cells in multiple scales. + + Args: + featmap_sizes (list(tuple)): List of feature map sizes in multiple + feature levels. + gt_bboxes (Tensor): Ground truth boxes, shape (n, 4). + device (str): Device where the anchors will be put on. + + Return: + list(torch.Tensor): responsible flags of anchors in multiple level + """ + assert self.num_levels == len(featmap_sizes) + multi_level_responsible_flags = [] + for i in range(self.num_levels): + anchor_stride = self.strides[i] + flags = self.single_level_responsible_flags( + featmap_sizes[i], + gt_bboxes, + anchor_stride, + self.num_base_anchors[i], + device=device) + multi_level_responsible_flags.append(flags) + return multi_level_responsible_flags + + def single_level_responsible_flags(self, + featmap_size, + gt_bboxes, + stride, + num_base_anchors, + device='cuda'): + """Generate the responsible flags of anchor in a single feature map. + + Args: + featmap_size (tuple[int]): The size of feature maps. + gt_bboxes (Tensor): Ground truth boxes, shape (n, 4). + stride (tuple(int)): stride of current level + num_base_anchors (int): The number of base anchors. + device (str, optional): Device where the flags will be put on. + Defaults to 'cuda'. + + Returns: + torch.Tensor: The valid flags of each anchor in a single level \ + feature map. + """ + feat_h, feat_w = featmap_size + gt_bboxes_cx = ((gt_bboxes[:, 0] + gt_bboxes[:, 2]) * 0.5).to(device) + gt_bboxes_cy = ((gt_bboxes[:, 1] + gt_bboxes[:, 3]) * 0.5).to(device) + gt_bboxes_grid_x = torch.floor(gt_bboxes_cx / stride[0]).long() + gt_bboxes_grid_y = torch.floor(gt_bboxes_cy / stride[1]).long() + + # row major indexing + gt_bboxes_grid_idx = gt_bboxes_grid_y * feat_w + gt_bboxes_grid_x + + responsible_grid = torch.zeros( + feat_h * feat_w, dtype=torch.uint8, device=device) + responsible_grid[gt_bboxes_grid_idx] = 1 + + responsible_grid = responsible_grid[:, None].expand( + responsible_grid.size(0), num_base_anchors).contiguous().view(-1) + return responsible_grid diff --git a/detection/mmdet/core/anchor/builder.py b/detection/mmdet/core/anchor/builder.py new file mode 100644 index 0000000..d79b448 --- /dev/null +++ b/detection/mmdet/core/anchor/builder.py @@ -0,0 +1,7 @@ +from mmcv.utils import Registry, build_from_cfg + +ANCHOR_GENERATORS = Registry('Anchor generator') + + +def build_anchor_generator(cfg, default_args=None): + return build_from_cfg(cfg, ANCHOR_GENERATORS, default_args) diff --git a/detection/mmdet/core/anchor/point_generator.py b/detection/mmdet/core/anchor/point_generator.py new file mode 100644 index 0000000..e6fbd98 --- /dev/null +++ b/detection/mmdet/core/anchor/point_generator.py @@ -0,0 +1,37 @@ +import torch + +from .builder import ANCHOR_GENERATORS + + +@ANCHOR_GENERATORS.register_module() +class PointGenerator(object): + + def _meshgrid(self, x, y, row_major=True): + xx = x.repeat(len(y)) + yy = y.view(-1, 1).repeat(1, len(x)).view(-1) + if row_major: + return xx, yy + else: + return yy, xx + + def grid_points(self, featmap_size, stride=16, device='cuda'): + feat_h, feat_w = featmap_size + shift_x = torch.arange(0., feat_w, device=device) * stride + shift_y = torch.arange(0., feat_h, device=device) * stride + shift_xx, shift_yy = self._meshgrid(shift_x, shift_y) + stride = shift_x.new_full((shift_xx.shape[0], ), stride) + shifts = torch.stack([shift_xx, shift_yy, stride], dim=-1) + all_points = shifts.to(device) + return all_points + + def valid_flags(self, featmap_size, valid_size, device='cuda'): + feat_h, feat_w = featmap_size + valid_h, valid_w = valid_size + assert valid_h <= feat_h and valid_w <= feat_w + valid_x = torch.zeros(feat_w, dtype=torch.bool, device=device) + valid_y = torch.zeros(feat_h, dtype=torch.bool, device=device) + valid_x[:valid_w] = 1 + valid_y[:valid_h] = 1 + valid_xx, valid_yy = self._meshgrid(valid_x, valid_y) + valid = valid_xx & valid_yy + return valid diff --git a/detection/mmdet/core/anchor/utils.py b/detection/mmdet/core/anchor/utils.py new file mode 100644 index 0000000..ab9b53f --- /dev/null +++ b/detection/mmdet/core/anchor/utils.py @@ -0,0 +1,71 @@ +import torch + + +def images_to_levels(target, num_levels): + """Convert targets by image to targets by feature level. + + [target_img0, target_img1] -> [target_level0, target_level1, ...] + """ + target = torch.stack(target, 0) + level_targets = [] + start = 0 + for n in num_levels: + end = start + n + # level_targets.append(target[:, start:end].squeeze(0)) + level_targets.append(target[:, start:end]) + start = end + return level_targets + + +def anchor_inside_flags(flat_anchors, + valid_flags, + img_shape, + allowed_border=0): + """Check whether the anchors are inside the border. + + Args: + flat_anchors (torch.Tensor): Flatten anchors, shape (n, 4). + valid_flags (torch.Tensor): An existing valid flags of anchors. + img_shape (tuple(int)): Shape of current image. + allowed_border (int, optional): The border to allow the valid anchor. + Defaults to 0. + + Returns: + torch.Tensor: Flags indicating whether the anchors are inside a \ + valid range. + """ + img_h, img_w = img_shape[:2] + if allowed_border >= 0: + inside_flags = valid_flags & \ + (flat_anchors[:, 0] >= -allowed_border) & \ + (flat_anchors[:, 1] >= -allowed_border) & \ + (flat_anchors[:, 2] < img_w + allowed_border) & \ + (flat_anchors[:, 3] < img_h + allowed_border) + else: + inside_flags = valid_flags + return inside_flags + + +def calc_region(bbox, ratio, featmap_size=None): + """Calculate a proportional bbox region. + + The bbox center are fixed and the new h' and w' is h * ratio and w * ratio. + + Args: + bbox (Tensor): Bboxes to calculate regions, shape (n, 4). + ratio (float): Ratio of the output region. + featmap_size (tuple): Feature map size used for clipping the boundary. + + Returns: + tuple: x1, y1, x2, y2 + """ + x1 = torch.round((1 - ratio) * bbox[0] + ratio * bbox[2]).long() + y1 = torch.round((1 - ratio) * bbox[1] + ratio * bbox[3]).long() + x2 = torch.round(ratio * bbox[0] + (1 - ratio) * bbox[2]).long() + y2 = torch.round(ratio * bbox[1] + (1 - ratio) * bbox[3]).long() + if featmap_size is not None: + x1 = x1.clamp(min=0, max=featmap_size[1]) + y1 = y1.clamp(min=0, max=featmap_size[0]) + x2 = x2.clamp(min=0, max=featmap_size[1]) + y2 = y2.clamp(min=0, max=featmap_size[0]) + return (x1, y1, x2, y2) diff --git a/detection/mmdet/core/bbox/__init__.py b/detection/mmdet/core/bbox/__init__.py new file mode 100644 index 0000000..a353729 --- /dev/null +++ b/detection/mmdet/core/bbox/__init__.py @@ -0,0 +1,27 @@ +from .assigners import (AssignResult, BaseAssigner, CenterRegionAssigner, + MaxIoUAssigner, RegionAssigner) +from .builder import build_assigner, build_bbox_coder, build_sampler +from .coder import (BaseBBoxCoder, DeltaXYWHBBoxCoder, PseudoBBoxCoder, + TBLRBBoxCoder) +from .iou_calculators import BboxOverlaps2D, bbox_overlaps +from .samplers import (BaseSampler, CombinedSampler, + InstanceBalancedPosSampler, IoUBalancedNegSampler, + OHEMSampler, PseudoSampler, RandomSampler, + SamplingResult, ScoreHLRSampler) +from .transforms import (bbox2distance, bbox2result, bbox2roi, + bbox_cxcywh_to_xyxy, bbox_flip, bbox_mapping, + bbox_mapping_back, bbox_rescale, bbox_xyxy_to_cxcywh, + distance2bbox, roi2bbox) + +__all__ = [ + 'bbox_overlaps', 'BboxOverlaps2D', 'BaseAssigner', 'MaxIoUAssigner', + 'AssignResult', 'BaseSampler', 'PseudoSampler', 'RandomSampler', + 'InstanceBalancedPosSampler', 'IoUBalancedNegSampler', 'CombinedSampler', + 'OHEMSampler', 'SamplingResult', 'ScoreHLRSampler', 'build_assigner', + 'build_sampler', 'bbox_flip', 'bbox_mapping', 'bbox_mapping_back', + 'bbox2roi', 'roi2bbox', 'bbox2result', 'distance2bbox', 'bbox2distance', + 'build_bbox_coder', 'BaseBBoxCoder', 'PseudoBBoxCoder', + 'DeltaXYWHBBoxCoder', 'TBLRBBoxCoder', 'CenterRegionAssigner', + 'bbox_rescale', 'bbox_cxcywh_to_xyxy', 'bbox_xyxy_to_cxcywh', + 'RegionAssigner' +] diff --git a/detection/mmdet/core/bbox/assigners/__init__.py b/detection/mmdet/core/bbox/assigners/__init__.py new file mode 100644 index 0000000..95e34a8 --- /dev/null +++ b/detection/mmdet/core/bbox/assigners/__init__.py @@ -0,0 +1,16 @@ +from .approx_max_iou_assigner import ApproxMaxIoUAssigner +from .assign_result import AssignResult +from .atss_assigner import ATSSAssigner +from .base_assigner import BaseAssigner +from .center_region_assigner import CenterRegionAssigner +from .grid_assigner import GridAssigner +from .hungarian_assigner import HungarianAssigner +from .max_iou_assigner import MaxIoUAssigner +from .point_assigner import PointAssigner +from .region_assigner import RegionAssigner + +__all__ = [ + 'BaseAssigner', 'MaxIoUAssigner', 'ApproxMaxIoUAssigner', 'AssignResult', + 'PointAssigner', 'ATSSAssigner', 'CenterRegionAssigner', 'GridAssigner', + 'HungarianAssigner', 'RegionAssigner' +] diff --git a/detection/mmdet/core/bbox/assigners/approx_max_iou_assigner.py b/detection/mmdet/core/bbox/assigners/approx_max_iou_assigner.py new file mode 100644 index 0000000..6d07656 --- /dev/null +++ b/detection/mmdet/core/bbox/assigners/approx_max_iou_assigner.py @@ -0,0 +1,145 @@ +import torch + +from ..builder import BBOX_ASSIGNERS +from ..iou_calculators import build_iou_calculator +from .max_iou_assigner import MaxIoUAssigner + + +@BBOX_ASSIGNERS.register_module() +class ApproxMaxIoUAssigner(MaxIoUAssigner): + """Assign a corresponding gt bbox or background to each bbox. + + Each proposals will be assigned with an integer indicating the ground truth + index. (semi-positive index: gt label (0-based), -1: background) + + - -1: negative sample, no assigned gt + - semi-positive integer: positive sample, index (0-based) of assigned gt + + Args: + pos_iou_thr (float): IoU threshold for positive bboxes. + neg_iou_thr (float or tuple): IoU threshold for negative bboxes. + min_pos_iou (float): Minimum iou for a bbox to be considered as a + positive bbox. Positive samples can have smaller IoU than + pos_iou_thr due to the 4th step (assign max IoU sample to each gt). + gt_max_assign_all (bool): Whether to assign all bboxes with the same + highest overlap with some gt to that gt. + ignore_iof_thr (float): IoF threshold for ignoring bboxes (if + `gt_bboxes_ignore` is specified). Negative values mean not + ignoring any bboxes. + ignore_wrt_candidates (bool): Whether to compute the iof between + `bboxes` and `gt_bboxes_ignore`, or the contrary. + match_low_quality (bool): Whether to allow quality matches. This is + usually allowed for RPN and single stage detectors, but not allowed + in the second stage. + gpu_assign_thr (int): The upper bound of the number of GT for GPU + assign. When the number of gt is above this threshold, will assign + on CPU device. Negative values mean not assign on CPU. + """ + + def __init__(self, + pos_iou_thr, + neg_iou_thr, + min_pos_iou=.0, + gt_max_assign_all=True, + ignore_iof_thr=-1, + ignore_wrt_candidates=True, + match_low_quality=True, + gpu_assign_thr=-1, + iou_calculator=dict(type='BboxOverlaps2D')): + self.pos_iou_thr = pos_iou_thr + self.neg_iou_thr = neg_iou_thr + self.min_pos_iou = min_pos_iou + self.gt_max_assign_all = gt_max_assign_all + self.ignore_iof_thr = ignore_iof_thr + self.ignore_wrt_candidates = ignore_wrt_candidates + self.gpu_assign_thr = gpu_assign_thr + self.match_low_quality = match_low_quality + self.iou_calculator = build_iou_calculator(iou_calculator) + + def assign(self, + approxs, + squares, + approxs_per_octave, + gt_bboxes, + gt_bboxes_ignore=None, + gt_labels=None): + """Assign gt to approxs. + + This method assign a gt bbox to each group of approxs (bboxes), + each group of approxs is represent by a base approx (bbox) and + will be assigned with -1, or a semi-positive number. + background_label (-1) means negative sample, + semi-positive number is the index (0-based) of assigned gt. + The assignment is done in following steps, the order matters. + + 1. assign every bbox to background_label (-1) + 2. use the max IoU of each group of approxs to assign + 2. assign proposals whose iou with all gts < neg_iou_thr to background + 3. for each bbox, if the iou with its nearest gt >= pos_iou_thr, + assign it to that bbox + 4. for each gt bbox, assign its nearest proposals (may be more than + one) to itself + + Args: + approxs (Tensor): Bounding boxes to be assigned, + shape(approxs_per_octave*n, 4). + squares (Tensor): Base Bounding boxes to be assigned, + shape(n, 4). + approxs_per_octave (int): number of approxs per octave + gt_bboxes (Tensor): Groundtruth boxes, shape (k, 4). + gt_bboxes_ignore (Tensor, optional): Ground truth bboxes that are + labelled as `ignored`, e.g., crowd boxes in COCO. + gt_labels (Tensor, optional): Label of gt_bboxes, shape (k, ). + + Returns: + :obj:`AssignResult`: The assign result. + """ + num_squares = squares.size(0) + num_gts = gt_bboxes.size(0) + + if num_squares == 0 or num_gts == 0: + # No predictions and/or truth, return empty assignment + overlaps = approxs.new(num_gts, num_squares) + assign_result = self.assign_wrt_overlaps(overlaps, gt_labels) + return assign_result + + # re-organize anchors by approxs_per_octave x num_squares + approxs = torch.transpose( + approxs.view(num_squares, approxs_per_octave, 4), 0, + 1).contiguous().view(-1, 4) + assign_on_cpu = True if (self.gpu_assign_thr > 0) and ( + num_gts > self.gpu_assign_thr) else False + # compute overlap and assign gt on CPU when number of GT is large + if assign_on_cpu: + device = approxs.device + approxs = approxs.cpu() + gt_bboxes = gt_bboxes.cpu() + if gt_bboxes_ignore is not None: + gt_bboxes_ignore = gt_bboxes_ignore.cpu() + if gt_labels is not None: + gt_labels = gt_labels.cpu() + all_overlaps = self.iou_calculator(approxs, gt_bboxes) + + overlaps, _ = all_overlaps.view(approxs_per_octave, num_squares, + num_gts).max(dim=0) + overlaps = torch.transpose(overlaps, 0, 1) + + if (self.ignore_iof_thr > 0 and gt_bboxes_ignore is not None + and gt_bboxes_ignore.numel() > 0 and squares.numel() > 0): + if self.ignore_wrt_candidates: + ignore_overlaps = self.iou_calculator( + squares, gt_bboxes_ignore, mode='iof') + ignore_max_overlaps, _ = ignore_overlaps.max(dim=1) + else: + ignore_overlaps = self.iou_calculator( + gt_bboxes_ignore, squares, mode='iof') + ignore_max_overlaps, _ = ignore_overlaps.max(dim=0) + overlaps[:, ignore_max_overlaps > self.ignore_iof_thr] = -1 + + assign_result = self.assign_wrt_overlaps(overlaps, gt_labels) + if assign_on_cpu: + assign_result.gt_inds = assign_result.gt_inds.to(device) + assign_result.max_overlaps = assign_result.max_overlaps.to(device) + if assign_result.labels is not None: + assign_result.labels = assign_result.labels.to(device) + return assign_result diff --git a/detection/mmdet/core/bbox/assigners/assign_result.py b/detection/mmdet/core/bbox/assigners/assign_result.py new file mode 100644 index 0000000..4639fbd --- /dev/null +++ b/detection/mmdet/core/bbox/assigners/assign_result.py @@ -0,0 +1,204 @@ +import torch + +from mmdet.utils import util_mixins + + +class AssignResult(util_mixins.NiceRepr): + """Stores assignments between predicted and truth boxes. + + Attributes: + num_gts (int): the number of truth boxes considered when computing this + assignment + + gt_inds (LongTensor): for each predicted box indicates the 1-based + index of the assigned truth box. 0 means unassigned and -1 means + ignore. + + max_overlaps (FloatTensor): the iou between the predicted box and its + assigned truth box. + + labels (None | LongTensor): If specified, for each predicted box + indicates the category label of the assigned truth box. + + Example: + >>> # An assign result between 4 predicted boxes and 9 true boxes + >>> # where only two boxes were assigned. + >>> num_gts = 9 + >>> max_overlaps = torch.LongTensor([0, .5, .9, 0]) + >>> gt_inds = torch.LongTensor([-1, 1, 2, 0]) + >>> labels = torch.LongTensor([0, 3, 4, 0]) + >>> self = AssignResult(num_gts, gt_inds, max_overlaps, labels) + >>> print(str(self)) # xdoctest: +IGNORE_WANT + + >>> # Force addition of gt labels (when adding gt as proposals) + >>> new_labels = torch.LongTensor([3, 4, 5]) + >>> self.add_gt_(new_labels) + >>> print(str(self)) # xdoctest: +IGNORE_WANT + + """ + + def __init__(self, num_gts, gt_inds, max_overlaps, labels=None): + self.num_gts = num_gts + self.gt_inds = gt_inds + self.max_overlaps = max_overlaps + self.labels = labels + # Interface for possible user-defined properties + self._extra_properties = {} + + @property + def num_preds(self): + """int: the number of predictions in this assignment""" + return len(self.gt_inds) + + def set_extra_property(self, key, value): + """Set user-defined new property.""" + assert key not in self.info + self._extra_properties[key] = value + + def get_extra_property(self, key): + """Get user-defined property.""" + return self._extra_properties.get(key, None) + + @property + def info(self): + """dict: a dictionary of info about the object""" + basic_info = { + 'num_gts': self.num_gts, + 'num_preds': self.num_preds, + 'gt_inds': self.gt_inds, + 'max_overlaps': self.max_overlaps, + 'labels': self.labels, + } + basic_info.update(self._extra_properties) + return basic_info + + def __nice__(self): + """str: a "nice" summary string describing this assign result""" + parts = [] + parts.append(f'num_gts={self.num_gts!r}') + if self.gt_inds is None: + parts.append(f'gt_inds={self.gt_inds!r}') + else: + parts.append(f'gt_inds.shape={tuple(self.gt_inds.shape)!r}') + if self.max_overlaps is None: + parts.append(f'max_overlaps={self.max_overlaps!r}') + else: + parts.append('max_overlaps.shape=' + f'{tuple(self.max_overlaps.shape)!r}') + if self.labels is None: + parts.append(f'labels={self.labels!r}') + else: + parts.append(f'labels.shape={tuple(self.labels.shape)!r}') + return ', '.join(parts) + + @classmethod + def random(cls, **kwargs): + """Create random AssignResult for tests or debugging. + + Args: + num_preds: number of predicted boxes + num_gts: number of true boxes + p_ignore (float): probability of a predicted box assinged to an + ignored truth + p_assigned (float): probability of a predicted box not being + assigned + p_use_label (float | bool): with labels or not + rng (None | int | numpy.random.RandomState): seed or state + + Returns: + :obj:`AssignResult`: Randomly generated assign results. + + Example: + >>> from mmdet.core.bbox.assigners.assign_result import * # NOQA + >>> self = AssignResult.random() + >>> print(self.info) + """ + from mmdet.core.bbox import demodata + rng = demodata.ensure_rng(kwargs.get('rng', None)) + + num_gts = kwargs.get('num_gts', None) + num_preds = kwargs.get('num_preds', None) + p_ignore = kwargs.get('p_ignore', 0.3) + p_assigned = kwargs.get('p_assigned', 0.7) + p_use_label = kwargs.get('p_use_label', 0.5) + num_classes = kwargs.get('p_use_label', 3) + + if num_gts is None: + num_gts = rng.randint(0, 8) + if num_preds is None: + num_preds = rng.randint(0, 16) + + if num_gts == 0: + max_overlaps = torch.zeros(num_preds, dtype=torch.float32) + gt_inds = torch.zeros(num_preds, dtype=torch.int64) + if p_use_label is True or p_use_label < rng.rand(): + labels = torch.zeros(num_preds, dtype=torch.int64) + else: + labels = None + else: + import numpy as np + # Create an overlap for each predicted box + max_overlaps = torch.from_numpy(rng.rand(num_preds)) + + # Construct gt_inds for each predicted box + is_assigned = torch.from_numpy(rng.rand(num_preds) < p_assigned) + # maximum number of assignments constraints + n_assigned = min(num_preds, min(num_gts, is_assigned.sum())) + + assigned_idxs = np.where(is_assigned)[0] + rng.shuffle(assigned_idxs) + assigned_idxs = assigned_idxs[0:n_assigned] + assigned_idxs.sort() + + is_assigned[:] = 0 + is_assigned[assigned_idxs] = True + + is_ignore = torch.from_numpy( + rng.rand(num_preds) < p_ignore) & is_assigned + + gt_inds = torch.zeros(num_preds, dtype=torch.int64) + + true_idxs = np.arange(num_gts) + rng.shuffle(true_idxs) + true_idxs = torch.from_numpy(true_idxs) + gt_inds[is_assigned] = true_idxs[:n_assigned] + + gt_inds = torch.from_numpy( + rng.randint(1, num_gts + 1, size=num_preds)) + gt_inds[is_ignore] = -1 + gt_inds[~is_assigned] = 0 + max_overlaps[~is_assigned] = 0 + + if p_use_label is True or p_use_label < rng.rand(): + if num_classes == 0: + labels = torch.zeros(num_preds, dtype=torch.int64) + else: + labels = torch.from_numpy( + # remind that we set FG labels to [0, num_class-1] + # since mmdet v2.0 + # BG cat_id: num_class + rng.randint(0, num_classes, size=num_preds)) + labels[~is_assigned] = 0 + else: + labels = None + + self = cls(num_gts, gt_inds, max_overlaps, labels) + return self + + def add_gt_(self, gt_labels): + """Add ground truth as assigned results. + + Args: + gt_labels (torch.Tensor): Labels of gt boxes + """ + self_inds = torch.arange( + 1, len(gt_labels) + 1, dtype=torch.long, device=gt_labels.device) + self.gt_inds = torch.cat([self_inds, self.gt_inds]) + + self.max_overlaps = torch.cat( + [self.max_overlaps.new_ones(len(gt_labels)), self.max_overlaps]) + + if self.labels is not None: + self.labels = torch.cat([gt_labels, self.labels]) diff --git a/detection/mmdet/core/bbox/assigners/atss_assigner.py b/detection/mmdet/core/bbox/assigners/atss_assigner.py new file mode 100644 index 0000000..d4fe9d0 --- /dev/null +++ b/detection/mmdet/core/bbox/assigners/atss_assigner.py @@ -0,0 +1,178 @@ +import torch + +from ..builder import BBOX_ASSIGNERS +from ..iou_calculators import build_iou_calculator +from .assign_result import AssignResult +from .base_assigner import BaseAssigner + + +@BBOX_ASSIGNERS.register_module() +class ATSSAssigner(BaseAssigner): + """Assign a corresponding gt bbox or background to each bbox. + + Each proposals will be assigned with `0` or a positive integer + indicating the ground truth index. + + - 0: negative sample, no assigned gt + - positive integer: positive sample, index (1-based) of assigned gt + + Args: + topk (float): number of bbox selected in each level + """ + + def __init__(self, + topk, + iou_calculator=dict(type='BboxOverlaps2D'), + ignore_iof_thr=-1): + self.topk = topk + self.iou_calculator = build_iou_calculator(iou_calculator) + self.ignore_iof_thr = ignore_iof_thr + + # https://github.com/sfzhang15/ATSS/blob/master/atss_core/modeling/rpn/atss/loss.py + + def assign(self, + bboxes, + num_level_bboxes, + gt_bboxes, + gt_bboxes_ignore=None, + gt_labels=None): + """Assign gt to bboxes. + + The assignment is done in following steps + + 1. compute iou between all bbox (bbox of all pyramid levels) and gt + 2. compute center distance between all bbox and gt + 3. on each pyramid level, for each gt, select k bbox whose center + are closest to the gt center, so we total select k*l bbox as + candidates for each gt + 4. get corresponding iou for the these candidates, and compute the + mean and std, set mean + std as the iou threshold + 5. select these candidates whose iou are greater than or equal to + the threshold as positive + 6. limit the positive sample's center in gt + + + Args: + bboxes (Tensor): Bounding boxes to be assigned, shape(n, 4). + num_level_bboxes (List): num of bboxes in each level + gt_bboxes (Tensor): Groundtruth boxes, shape (k, 4). + gt_bboxes_ignore (Tensor, optional): Ground truth bboxes that are + labelled as `ignored`, e.g., crowd boxes in COCO. + gt_labels (Tensor, optional): Label of gt_bboxes, shape (k, ). + + Returns: + :obj:`AssignResult`: The assign result. + """ + INF = 100000000 + bboxes = bboxes[:, :4] + num_gt, num_bboxes = gt_bboxes.size(0), bboxes.size(0) + + # compute iou between all bbox and gt + overlaps = self.iou_calculator(bboxes, gt_bboxes) + + # assign 0 by default + assigned_gt_inds = overlaps.new_full((num_bboxes, ), + 0, + dtype=torch.long) + + if num_gt == 0 or num_bboxes == 0: + # No ground truth or boxes, return empty assignment + max_overlaps = overlaps.new_zeros((num_bboxes, )) + if num_gt == 0: + # No truth, assign everything to background + assigned_gt_inds[:] = 0 + if gt_labels is None: + assigned_labels = None + else: + assigned_labels = overlaps.new_full((num_bboxes, ), + -1, + dtype=torch.long) + return AssignResult( + num_gt, assigned_gt_inds, max_overlaps, labels=assigned_labels) + + # compute center distance between all bbox and gt + gt_cx = (gt_bboxes[:, 0] + gt_bboxes[:, 2]) / 2.0 + gt_cy = (gt_bboxes[:, 1] + gt_bboxes[:, 3]) / 2.0 + gt_points = torch.stack((gt_cx, gt_cy), dim=1) + + bboxes_cx = (bboxes[:, 0] + bboxes[:, 2]) / 2.0 + bboxes_cy = (bboxes[:, 1] + bboxes[:, 3]) / 2.0 + bboxes_points = torch.stack((bboxes_cx, bboxes_cy), dim=1) + + distances = (bboxes_points[:, None, :] - + gt_points[None, :, :]).pow(2).sum(-1).sqrt() + + if (self.ignore_iof_thr > 0 and gt_bboxes_ignore is not None + and gt_bboxes_ignore.numel() > 0 and bboxes.numel() > 0): + ignore_overlaps = self.iou_calculator( + bboxes, gt_bboxes_ignore, mode='iof') + ignore_max_overlaps, _ = ignore_overlaps.max(dim=1) + ignore_idxs = ignore_max_overlaps > self.ignore_iof_thr + distances[ignore_idxs, :] = INF + assigned_gt_inds[ignore_idxs] = -1 + + # Selecting candidates based on the center distance + candidate_idxs = [] + start_idx = 0 + for level, bboxes_per_level in enumerate(num_level_bboxes): + # on each pyramid level, for each gt, + # select k bbox whose center are closest to the gt center + end_idx = start_idx + bboxes_per_level + distances_per_level = distances[start_idx:end_idx, :] + selectable_k = min(self.topk, bboxes_per_level) + _, topk_idxs_per_level = distances_per_level.topk( + selectable_k, dim=0, largest=False) + candidate_idxs.append(topk_idxs_per_level + start_idx) + start_idx = end_idx + candidate_idxs = torch.cat(candidate_idxs, dim=0) + + # get corresponding iou for the these candidates, and compute the + # mean and std, set mean + std as the iou threshold + candidate_overlaps = overlaps[candidate_idxs, torch.arange(num_gt)] + overlaps_mean_per_gt = candidate_overlaps.mean(0) + overlaps_std_per_gt = candidate_overlaps.std(0) + overlaps_thr_per_gt = overlaps_mean_per_gt + overlaps_std_per_gt + + is_pos = candidate_overlaps >= overlaps_thr_per_gt[None, :] + + # limit the positive sample's center in gt + for gt_idx in range(num_gt): + candidate_idxs[:, gt_idx] += gt_idx * num_bboxes + ep_bboxes_cx = bboxes_cx.view(1, -1).expand( + num_gt, num_bboxes).contiguous().view(-1) + ep_bboxes_cy = bboxes_cy.view(1, -1).expand( + num_gt, num_bboxes).contiguous().view(-1) + candidate_idxs = candidate_idxs.view(-1) + + # calculate the left, top, right, bottom distance between positive + # bbox center and gt side + l_ = ep_bboxes_cx[candidate_idxs].view(-1, num_gt) - gt_bboxes[:, 0] + t_ = ep_bboxes_cy[candidate_idxs].view(-1, num_gt) - gt_bboxes[:, 1] + r_ = gt_bboxes[:, 2] - ep_bboxes_cx[candidate_idxs].view(-1, num_gt) + b_ = gt_bboxes[:, 3] - ep_bboxes_cy[candidate_idxs].view(-1, num_gt) + is_in_gts = torch.stack([l_, t_, r_, b_], dim=1).min(dim=1)[0] > 0.01 + is_pos = is_pos & is_in_gts + + # if an anchor box is assigned to multiple gts, + # the one with the highest IoU will be selected. + overlaps_inf = torch.full_like(overlaps, + -INF).t().contiguous().view(-1) + index = candidate_idxs.view(-1)[is_pos.view(-1)] + overlaps_inf[index] = overlaps.t().contiguous().view(-1)[index] + overlaps_inf = overlaps_inf.view(num_gt, -1).t() + + max_overlaps, argmax_overlaps = overlaps_inf.max(dim=1) + assigned_gt_inds[ + max_overlaps != -INF] = argmax_overlaps[max_overlaps != -INF] + 1 + + if gt_labels is not None: + assigned_labels = assigned_gt_inds.new_full((num_bboxes, ), -1) + pos_inds = torch.nonzero( + assigned_gt_inds > 0, as_tuple=False).squeeze() + if pos_inds.numel() > 0: + assigned_labels[pos_inds] = gt_labels[ + assigned_gt_inds[pos_inds] - 1] + else: + assigned_labels = None + return AssignResult( + num_gt, assigned_gt_inds, max_overlaps, labels=assigned_labels) diff --git a/detection/mmdet/core/bbox/assigners/base_assigner.py b/detection/mmdet/core/bbox/assigners/base_assigner.py new file mode 100644 index 0000000..1ff0160 --- /dev/null +++ b/detection/mmdet/core/bbox/assigners/base_assigner.py @@ -0,0 +1,9 @@ +from abc import ABCMeta, abstractmethod + + +class BaseAssigner(metaclass=ABCMeta): + """Base assigner that assigns boxes to ground truth boxes.""" + + @abstractmethod + def assign(self, bboxes, gt_bboxes, gt_bboxes_ignore=None, gt_labels=None): + """Assign boxes to either a ground truth boxes or a negative boxes.""" diff --git a/detection/mmdet/core/bbox/assigners/center_region_assigner.py b/detection/mmdet/core/bbox/assigners/center_region_assigner.py new file mode 100644 index 0000000..488e3b6 --- /dev/null +++ b/detection/mmdet/core/bbox/assigners/center_region_assigner.py @@ -0,0 +1,335 @@ +import torch + +from ..builder import BBOX_ASSIGNERS +from ..iou_calculators import build_iou_calculator +from .assign_result import AssignResult +from .base_assigner import BaseAssigner + + +def scale_boxes(bboxes, scale): + """Expand an array of boxes by a given scale. + + Args: + bboxes (Tensor): Shape (m, 4) + scale (float): The scale factor of bboxes + + Returns: + (Tensor): Shape (m, 4). Scaled bboxes + """ + assert bboxes.size(1) == 4 + w_half = (bboxes[:, 2] - bboxes[:, 0]) * .5 + h_half = (bboxes[:, 3] - bboxes[:, 1]) * .5 + x_c = (bboxes[:, 2] + bboxes[:, 0]) * .5 + y_c = (bboxes[:, 3] + bboxes[:, 1]) * .5 + + w_half *= scale + h_half *= scale + + boxes_scaled = torch.zeros_like(bboxes) + boxes_scaled[:, 0] = x_c - w_half + boxes_scaled[:, 2] = x_c + w_half + boxes_scaled[:, 1] = y_c - h_half + boxes_scaled[:, 3] = y_c + h_half + return boxes_scaled + + +def is_located_in(points, bboxes): + """Are points located in bboxes. + + Args: + points (Tensor): Points, shape: (m, 2). + bboxes (Tensor): Bounding boxes, shape: (n, 4). + + Return: + Tensor: Flags indicating if points are located in bboxes, shape: (m, n). + """ + assert points.size(1) == 2 + assert bboxes.size(1) == 4 + return (points[:, 0].unsqueeze(1) > bboxes[:, 0].unsqueeze(0)) & \ + (points[:, 0].unsqueeze(1) < bboxes[:, 2].unsqueeze(0)) & \ + (points[:, 1].unsqueeze(1) > bboxes[:, 1].unsqueeze(0)) & \ + (points[:, 1].unsqueeze(1) < bboxes[:, 3].unsqueeze(0)) + + +def bboxes_area(bboxes): + """Compute the area of an array of bboxes. + + Args: + bboxes (Tensor): The coordinates ox bboxes. Shape: (m, 4) + + Returns: + Tensor: Area of the bboxes. Shape: (m, ) + """ + assert bboxes.size(1) == 4 + w = (bboxes[:, 2] - bboxes[:, 0]) + h = (bboxes[:, 3] - bboxes[:, 1]) + areas = w * h + return areas + + +@BBOX_ASSIGNERS.register_module() +class CenterRegionAssigner(BaseAssigner): + """Assign pixels at the center region of a bbox as positive. + + Each proposals will be assigned with `-1`, `0`, or a positive integer + indicating the ground truth index. + - -1: negative samples + - semi-positive numbers: positive sample, index (0-based) of assigned gt + + Args: + pos_scale (float): Threshold within which pixels are + labelled as positive. + neg_scale (float): Threshold above which pixels are + labelled as positive. + min_pos_iof (float): Minimum iof of a pixel with a gt to be + labelled as positive. Default: 1e-2 + ignore_gt_scale (float): Threshold within which the pixels + are ignored when the gt is labelled as shadowed. Default: 0.5 + foreground_dominate (bool): If True, the bbox will be assigned as + positive when a gt's kernel region overlaps with another's shadowed + (ignored) region, otherwise it is set as ignored. Default to False. + """ + + def __init__(self, + pos_scale, + neg_scale, + min_pos_iof=1e-2, + ignore_gt_scale=0.5, + foreground_dominate=False, + iou_calculator=dict(type='BboxOverlaps2D')): + self.pos_scale = pos_scale + self.neg_scale = neg_scale + self.min_pos_iof = min_pos_iof + self.ignore_gt_scale = ignore_gt_scale + self.foreground_dominate = foreground_dominate + self.iou_calculator = build_iou_calculator(iou_calculator) + + def get_gt_priorities(self, gt_bboxes): + """Get gt priorities according to their areas. + + Smaller gt has higher priority. + + Args: + gt_bboxes (Tensor): Ground truth boxes, shape (k, 4). + + Returns: + Tensor: The priority of gts so that gts with larger priority is \ + more likely to be assigned. Shape (k, ) + """ + gt_areas = bboxes_area(gt_bboxes) + # Rank all gt bbox areas. Smaller objects has larger priority + _, sort_idx = gt_areas.sort(descending=True) + sort_idx = sort_idx.argsort() + return sort_idx + + def assign(self, bboxes, gt_bboxes, gt_bboxes_ignore=None, gt_labels=None): + """Assign gt to bboxes. + + This method assigns gts to every bbox (proposal/anchor), each bbox \ + will be assigned with -1, or a semi-positive number. -1 means \ + negative sample, semi-positive number is the index (0-based) of \ + assigned gt. + + Args: + bboxes (Tensor): Bounding boxes to be assigned, shape(n, 4). + gt_bboxes (Tensor): Groundtruth boxes, shape (k, 4). + gt_bboxes_ignore (tensor, optional): Ground truth bboxes that are + labelled as `ignored`, e.g., crowd boxes in COCO. + gt_labels (tensor, optional): Label of gt_bboxes, shape (num_gts,). + + Returns: + :obj:`AssignResult`: The assigned result. Note that \ + shadowed_labels of shape (N, 2) is also added as an \ + `assign_result` attribute. `shadowed_labels` is a tensor \ + composed of N pairs of anchor_ind, class_label], where N \ + is the number of anchors that lie in the outer region of a \ + gt, anchor_ind is the shadowed anchor index and class_label \ + is the shadowed class label. + + Example: + >>> self = CenterRegionAssigner(0.2, 0.2) + >>> bboxes = torch.Tensor([[0, 0, 10, 10], [10, 10, 20, 20]]) + >>> gt_bboxes = torch.Tensor([[0, 0, 10, 10]]) + >>> assign_result = self.assign(bboxes, gt_bboxes) + >>> expected_gt_inds = torch.LongTensor([1, 0]) + >>> assert torch.all(assign_result.gt_inds == expected_gt_inds) + """ + # There are in total 5 steps in the pixel assignment + # 1. Find core (the center region, say inner 0.2) + # and shadow (the relatively ourter part, say inner 0.2-0.5) + # regions of every gt. + # 2. Find all prior bboxes that lie in gt_core and gt_shadow regions + # 3. Assign prior bboxes in gt_core with a one-hot id of the gt in + # the image. + # 3.1. For overlapping objects, the prior bboxes in gt_core is + # assigned with the object with smallest area + # 4. Assign prior bboxes with class label according to its gt id. + # 4.1. Assign -1 to prior bboxes lying in shadowed gts + # 4.2. Assign positive prior boxes with the corresponding label + # 5. Find pixels lying in the shadow of an object and assign them with + # background label, but set the loss weight of its corresponding + # gt to zero. + assert bboxes.size(1) == 4, 'bboxes must have size of 4' + # 1. Find core positive and shadow region of every gt + gt_core = scale_boxes(gt_bboxes, self.pos_scale) + gt_shadow = scale_boxes(gt_bboxes, self.neg_scale) + + # 2. Find prior bboxes that lie in gt_core and gt_shadow regions + bbox_centers = (bboxes[:, 2:4] + bboxes[:, 0:2]) / 2 + # The center points lie within the gt boxes + is_bbox_in_gt = is_located_in(bbox_centers, gt_bboxes) + # Only calculate bbox and gt_core IoF. This enables small prior bboxes + # to match large gts + bbox_and_gt_core_overlaps = self.iou_calculator( + bboxes, gt_core, mode='iof') + # The center point of effective priors should be within the gt box + is_bbox_in_gt_core = is_bbox_in_gt & ( + bbox_and_gt_core_overlaps > self.min_pos_iof) # shape (n, k) + + is_bbox_in_gt_shadow = ( + self.iou_calculator(bboxes, gt_shadow, mode='iof') > + self.min_pos_iof) + # Rule out center effective positive pixels + is_bbox_in_gt_shadow &= (~is_bbox_in_gt_core) + + num_gts, num_bboxes = gt_bboxes.size(0), bboxes.size(0) + if num_gts == 0 or num_bboxes == 0: + # If no gts exist, assign all pixels to negative + assigned_gt_ids = \ + is_bbox_in_gt_core.new_zeros((num_bboxes,), + dtype=torch.long) + pixels_in_gt_shadow = assigned_gt_ids.new_empty((0, 2)) + else: + # Step 3: assign a one-hot gt id to each pixel, and smaller objects + # have high priority to assign the pixel. + sort_idx = self.get_gt_priorities(gt_bboxes) + assigned_gt_ids, pixels_in_gt_shadow = \ + self.assign_one_hot_gt_indices(is_bbox_in_gt_core, + is_bbox_in_gt_shadow, + gt_priority=sort_idx) + + if gt_bboxes_ignore is not None and gt_bboxes_ignore.numel() > 0: + # No ground truth or boxes, return empty assignment + gt_bboxes_ignore = scale_boxes( + gt_bboxes_ignore, scale=self.ignore_gt_scale) + is_bbox_in_ignored_gts = is_located_in(bbox_centers, + gt_bboxes_ignore) + is_bbox_in_ignored_gts = is_bbox_in_ignored_gts.any(dim=1) + assigned_gt_ids[is_bbox_in_ignored_gts] = -1 + + # 4. Assign prior bboxes with class label according to its gt id. + assigned_labels = None + shadowed_pixel_labels = None + if gt_labels is not None: + # Default assigned label is the background (-1) + assigned_labels = assigned_gt_ids.new_full((num_bboxes, ), -1) + pos_inds = torch.nonzero( + assigned_gt_ids > 0, as_tuple=False).squeeze() + if pos_inds.numel() > 0: + assigned_labels[pos_inds] = gt_labels[assigned_gt_ids[pos_inds] + - 1] + # 5. Find pixels lying in the shadow of an object + shadowed_pixel_labels = pixels_in_gt_shadow.clone() + if pixels_in_gt_shadow.numel() > 0: + pixel_idx, gt_idx =\ + pixels_in_gt_shadow[:, 0], pixels_in_gt_shadow[:, 1] + assert (assigned_gt_ids[pixel_idx] != gt_idx).all(), \ + 'Some pixels are dually assigned to ignore and gt!' + shadowed_pixel_labels[:, 1] = gt_labels[gt_idx - 1] + override = ( + assigned_labels[pixel_idx] == shadowed_pixel_labels[:, 1]) + if self.foreground_dominate: + # When a pixel is both positive and shadowed, set it as pos + shadowed_pixel_labels = shadowed_pixel_labels[~override] + else: + # When a pixel is both pos and shadowed, set it as shadowed + assigned_labels[pixel_idx[override]] = -1 + assigned_gt_ids[pixel_idx[override]] = 0 + + assign_result = AssignResult( + num_gts, assigned_gt_ids, None, labels=assigned_labels) + # Add shadowed_labels as assign_result property. Shape: (num_shadow, 2) + assign_result.set_extra_property('shadowed_labels', + shadowed_pixel_labels) + return assign_result + + def assign_one_hot_gt_indices(self, + is_bbox_in_gt_core, + is_bbox_in_gt_shadow, + gt_priority=None): + """Assign only one gt index to each prior box. + + Gts with large gt_priority are more likely to be assigned. + + Args: + is_bbox_in_gt_core (Tensor): Bool tensor indicating the bbox center + is in the core area of a gt (e.g. 0-0.2). + Shape: (num_prior, num_gt). + is_bbox_in_gt_shadow (Tensor): Bool tensor indicating the bbox + center is in the shadowed area of a gt (e.g. 0.2-0.5). + Shape: (num_prior, num_gt). + gt_priority (Tensor): Priorities of gts. The gt with a higher + priority is more likely to be assigned to the bbox when the bbox + match with multiple gts. Shape: (num_gt, ). + + Returns: + tuple: Returns (assigned_gt_inds, shadowed_gt_inds). + + - assigned_gt_inds: The assigned gt index of each prior bbox \ + (i.e. index from 1 to num_gts). Shape: (num_prior, ). + - shadowed_gt_inds: shadowed gt indices. It is a tensor of \ + shape (num_ignore, 2) with first column being the \ + shadowed prior bbox indices and the second column the \ + shadowed gt indices (1-based). + """ + num_bboxes, num_gts = is_bbox_in_gt_core.shape + + if gt_priority is None: + gt_priority = torch.arange( + num_gts, device=is_bbox_in_gt_core.device) + assert gt_priority.size(0) == num_gts + # The bigger gt_priority, the more preferable to be assigned + # The assigned inds are by default 0 (background) + assigned_gt_inds = is_bbox_in_gt_core.new_zeros((num_bboxes, ), + dtype=torch.long) + # Shadowed bboxes are assigned to be background. But the corresponding + # label is ignored during loss calculation, which is done through + # shadowed_gt_inds + shadowed_gt_inds = torch.nonzero(is_bbox_in_gt_shadow, as_tuple=False) + if is_bbox_in_gt_core.sum() == 0: # No gt match + shadowed_gt_inds[:, 1] += 1 # 1-based. For consistency issue + return assigned_gt_inds, shadowed_gt_inds + + # The priority of each prior box and gt pair. If one prior box is + # matched bo multiple gts. Only the pair with the highest priority + # is saved + pair_priority = is_bbox_in_gt_core.new_full((num_bboxes, num_gts), + -1, + dtype=torch.long) + + # Each bbox could match with multiple gts. + # The following codes deal with this situation + # Matched bboxes (to any gt). Shape: (num_pos_anchor, ) + inds_of_match = torch.any(is_bbox_in_gt_core, dim=1) + # The matched gt index of each positive bbox. Length >= num_pos_anchor + # , since one bbox could match multiple gts + matched_bbox_gt_inds = torch.nonzero( + is_bbox_in_gt_core, as_tuple=False)[:, 1] + # Assign priority to each bbox-gt pair. + pair_priority[is_bbox_in_gt_core] = gt_priority[matched_bbox_gt_inds] + _, argmax_priority = pair_priority[inds_of_match].max(dim=1) + assigned_gt_inds[inds_of_match] = argmax_priority + 1 # 1-based + # Zero-out the assigned anchor box to filter the shadowed gt indices + is_bbox_in_gt_core[inds_of_match, argmax_priority] = 0 + # Concat the shadowed indices due to overlapping with that out side of + # effective scale. shape: (total_num_ignore, 2) + shadowed_gt_inds = torch.cat( + (shadowed_gt_inds, torch.nonzero( + is_bbox_in_gt_core, as_tuple=False)), + dim=0) + # `is_bbox_in_gt_core` should be changed back to keep arguments intact. + is_bbox_in_gt_core[inds_of_match, argmax_priority] = 1 + # 1-based shadowed gt indices, to be consistent with `assigned_gt_inds` + if shadowed_gt_inds.numel() > 0: + shadowed_gt_inds[:, 1] += 1 + return assigned_gt_inds, shadowed_gt_inds diff --git a/detection/mmdet/core/bbox/assigners/grid_assigner.py b/detection/mmdet/core/bbox/assigners/grid_assigner.py new file mode 100644 index 0000000..7390ea6 --- /dev/null +++ b/detection/mmdet/core/bbox/assigners/grid_assigner.py @@ -0,0 +1,155 @@ +import torch + +from ..builder import BBOX_ASSIGNERS +from ..iou_calculators import build_iou_calculator +from .assign_result import AssignResult +from .base_assigner import BaseAssigner + + +@BBOX_ASSIGNERS.register_module() +class GridAssigner(BaseAssigner): + """Assign a corresponding gt bbox or background to each bbox. + + Each proposals will be assigned with `-1`, `0`, or a positive integer + indicating the ground truth index. + + - -1: don't care + - 0: negative sample, no assigned gt + - positive integer: positive sample, index (1-based) of assigned gt + + Args: + pos_iou_thr (float): IoU threshold for positive bboxes. + neg_iou_thr (float or tuple): IoU threshold for negative bboxes. + min_pos_iou (float): Minimum iou for a bbox to be considered as a + positive bbox. Positive samples can have smaller IoU than + pos_iou_thr due to the 4th step (assign max IoU sample to each gt). + gt_max_assign_all (bool): Whether to assign all bboxes with the same + highest overlap with some gt to that gt. + """ + + def __init__(self, + pos_iou_thr, + neg_iou_thr, + min_pos_iou=.0, + gt_max_assign_all=True, + iou_calculator=dict(type='BboxOverlaps2D')): + self.pos_iou_thr = pos_iou_thr + self.neg_iou_thr = neg_iou_thr + self.min_pos_iou = min_pos_iou + self.gt_max_assign_all = gt_max_assign_all + self.iou_calculator = build_iou_calculator(iou_calculator) + + def assign(self, bboxes, box_responsible_flags, gt_bboxes, gt_labels=None): + """Assign gt to bboxes. The process is very much like the max iou + assigner, except that positive samples are constrained within the cell + that the gt boxes fell in. + + This method assign a gt bbox to every bbox (proposal/anchor), each bbox + will be assigned with -1, 0, or a positive number. -1 means don't care, + 0 means negative sample, positive number is the index (1-based) of + assigned gt. + The assignment is done in following steps, the order matters. + + 1. assign every bbox to -1 + 2. assign proposals whose iou with all gts <= neg_iou_thr to 0 + 3. for each bbox within a cell, if the iou with its nearest gt > + pos_iou_thr and the center of that gt falls inside the cell, + assign it to that bbox + 4. for each gt bbox, assign its nearest proposals within the cell the + gt bbox falls in to itself. + + Args: + bboxes (Tensor): Bounding boxes to be assigned, shape(n, 4). + box_responsible_flags (Tensor): flag to indicate whether box is + responsible for prediction, shape(n, ) + gt_bboxes (Tensor): Groundtruth boxes, shape (k, 4). + gt_labels (Tensor, optional): Label of gt_bboxes, shape (k, ). + + Returns: + :obj:`AssignResult`: The assign result. + """ + num_gts, num_bboxes = gt_bboxes.size(0), bboxes.size(0) + + # compute iou between all gt and bboxes + overlaps = self.iou_calculator(gt_bboxes, bboxes) + + # 1. assign -1 by default + assigned_gt_inds = overlaps.new_full((num_bboxes, ), + -1, + dtype=torch.long) + + if num_gts == 0 or num_bboxes == 0: + # No ground truth or boxes, return empty assignment + max_overlaps = overlaps.new_zeros((num_bboxes, )) + if num_gts == 0: + # No truth, assign everything to background + assigned_gt_inds[:] = 0 + if gt_labels is None: + assigned_labels = None + else: + assigned_labels = overlaps.new_full((num_bboxes, ), + -1, + dtype=torch.long) + return AssignResult( + num_gts, + assigned_gt_inds, + max_overlaps, + labels=assigned_labels) + + # 2. assign negative: below + # for each anchor, which gt best overlaps with it + # for each anchor, the max iou of all gts + # shape of max_overlaps == argmax_overlaps == num_bboxes + max_overlaps, argmax_overlaps = overlaps.max(dim=0) + + if isinstance(self.neg_iou_thr, float): + assigned_gt_inds[(max_overlaps >= 0) + & (max_overlaps <= self.neg_iou_thr)] = 0 + elif isinstance(self.neg_iou_thr, (tuple, list)): + assert len(self.neg_iou_thr) == 2 + assigned_gt_inds[(max_overlaps > self.neg_iou_thr[0]) + & (max_overlaps <= self.neg_iou_thr[1])] = 0 + + # 3. assign positive: falls into responsible cell and above + # positive IOU threshold, the order matters. + # the prior condition of comparision is to filter out all + # unrelated anchors, i.e. not box_responsible_flags + overlaps[:, ~box_responsible_flags.type(torch.bool)] = -1. + + # calculate max_overlaps again, but this time we only consider IOUs + # for anchors responsible for prediction + max_overlaps, argmax_overlaps = overlaps.max(dim=0) + + # for each gt, which anchor best overlaps with it + # for each gt, the max iou of all proposals + # shape of gt_max_overlaps == gt_argmax_overlaps == num_gts + gt_max_overlaps, gt_argmax_overlaps = overlaps.max(dim=1) + + pos_inds = (max_overlaps > + self.pos_iou_thr) & box_responsible_flags.type(torch.bool) + assigned_gt_inds[pos_inds] = argmax_overlaps[pos_inds] + 1 + + # 4. assign positive to max overlapped anchors within responsible cell + for i in range(num_gts): + if gt_max_overlaps[i] > self.min_pos_iou: + if self.gt_max_assign_all: + max_iou_inds = (overlaps[i, :] == gt_max_overlaps[i]) & \ + box_responsible_flags.type(torch.bool) + assigned_gt_inds[max_iou_inds] = i + 1 + elif box_responsible_flags[gt_argmax_overlaps[i]]: + assigned_gt_inds[gt_argmax_overlaps[i]] = i + 1 + + # assign labels of positive anchors + if gt_labels is not None: + assigned_labels = assigned_gt_inds.new_full((num_bboxes, ), -1) + pos_inds = torch.nonzero( + assigned_gt_inds > 0, as_tuple=False).squeeze() + if pos_inds.numel() > 0: + assigned_labels[pos_inds] = gt_labels[ + assigned_gt_inds[pos_inds] - 1] + + else: + assigned_labels = None + + return AssignResult( + num_gts, assigned_gt_inds, max_overlaps, labels=assigned_labels) diff --git a/detection/mmdet/core/bbox/assigners/hungarian_assigner.py b/detection/mmdet/core/bbox/assigners/hungarian_assigner.py new file mode 100644 index 0000000..e10cc14 --- /dev/null +++ b/detection/mmdet/core/bbox/assigners/hungarian_assigner.py @@ -0,0 +1,145 @@ +import torch + +from ..builder import BBOX_ASSIGNERS +from ..match_costs import build_match_cost +from ..transforms import bbox_cxcywh_to_xyxy +from .assign_result import AssignResult +from .base_assigner import BaseAssigner + +try: + from scipy.optimize import linear_sum_assignment +except ImportError: + linear_sum_assignment = None + + +@BBOX_ASSIGNERS.register_module() +class HungarianAssigner(BaseAssigner): + """Computes one-to-one matching between predictions and ground truth. + + This class computes an assignment between the targets and the predictions + based on the costs. The costs are weighted sum of three components: + classification cost, regression L1 cost and regression iou cost. The + targets don't include the no_object, so generally there are more + predictions than targets. After the one-to-one matching, the un-matched + are treated as backgrounds. Thus each query prediction will be assigned + with `0` or a positive integer indicating the ground truth index: + + - 0: negative sample, no assigned gt + - positive integer: positive sample, index (1-based) of assigned gt + + Args: + cls_weight (int | float, optional): The scale factor for classification + cost. Default 1.0. + bbox_weight (int | float, optional): The scale factor for regression + L1 cost. Default 1.0. + iou_weight (int | float, optional): The scale factor for regression + iou cost. Default 1.0. + iou_calculator (dict | optional): The config for the iou calculation. + Default type `BboxOverlaps2D`. + iou_mode (str | optional): "iou" (intersection over union), "iof" + (intersection over foreground), or "giou" (generalized + intersection over union). Default "giou". + """ + + def __init__(self, + cls_cost=dict(type='ClassificationCost', weight=1.), + reg_cost=dict(type='BBoxL1Cost', weight=1.0), + iou_cost=dict(type='IoUCost', iou_mode='giou', weight=1.0)): + self.cls_cost = build_match_cost(cls_cost) + self.reg_cost = build_match_cost(reg_cost) + self.iou_cost = build_match_cost(iou_cost) + + def assign(self, + bbox_pred, + cls_pred, + gt_bboxes, + gt_labels, + img_meta, + gt_bboxes_ignore=None, + eps=1e-7): + """Computes one-to-one matching based on the weighted costs. + + This method assign each query prediction to a ground truth or + background. The `assigned_gt_inds` with -1 means don't care, + 0 means negative sample, and positive number is the index (1-based) + of assigned gt. + The assignment is done in the following steps, the order matters. + + 1. assign every prediction to -1 + 2. compute the weighted costs + 3. do Hungarian matching on CPU based on the costs + 4. assign all to 0 (background) first, then for each matched pair + between predictions and gts, treat this prediction as foreground + and assign the corresponding gt index (plus 1) to it. + + Args: + bbox_pred (Tensor): Predicted boxes with normalized coordinates + (cx, cy, w, h), which are all in range [0, 1]. Shape + [num_query, 4]. + cls_pred (Tensor): Predicted classification logits, shape + [num_query, num_class]. + gt_bboxes (Tensor): Ground truth boxes with unnormalized + coordinates (x1, y1, x2, y2). Shape [num_gt, 4]. + gt_labels (Tensor): Label of `gt_bboxes`, shape (num_gt,). + img_meta (dict): Meta information for current image. + gt_bboxes_ignore (Tensor, optional): Ground truth bboxes that are + labelled as `ignored`. Default None. + eps (int | float, optional): A value added to the denominator for + numerical stability. Default 1e-7. + + Returns: + :obj:`AssignResult`: The assigned result. + """ + assert gt_bboxes_ignore is None, \ + 'Only case when gt_bboxes_ignore is None is supported.' + num_gts, num_bboxes = gt_bboxes.size(0), bbox_pred.size(0) + + # 1. assign -1 by default + assigned_gt_inds = bbox_pred.new_full((num_bboxes, ), + -1, + dtype=torch.long) + assigned_labels = bbox_pred.new_full((num_bboxes, ), + -1, + dtype=torch.long) + if num_gts == 0 or num_bboxes == 0: + # No ground truth or boxes, return empty assignment + if num_gts == 0: + # No ground truth, assign all to background + assigned_gt_inds[:] = 0 + return AssignResult( + num_gts, assigned_gt_inds, None, labels=assigned_labels) + img_h, img_w, _ = img_meta['img_shape'] + factor = gt_bboxes.new_tensor([img_w, img_h, img_w, + img_h]).unsqueeze(0) + + # 2. compute the weighted costs + # classification and bboxcost. + cls_cost = self.cls_cost(cls_pred, gt_labels) + # regression L1 cost + normalize_gt_bboxes = gt_bboxes / factor + reg_cost = self.reg_cost(bbox_pred, normalize_gt_bboxes) + # regression iou cost, defaultly giou is used in official DETR. + bboxes = bbox_cxcywh_to_xyxy(bbox_pred) * factor + iou_cost = self.iou_cost(bboxes, gt_bboxes) + # weighted sum of above three costs + cost = cls_cost + reg_cost + iou_cost + + # 3. do Hungarian matching on CPU using linear_sum_assignment + cost = cost.detach().cpu() + if linear_sum_assignment is None: + raise ImportError('Please run "pip install scipy" ' + 'to install scipy first.') + matched_row_inds, matched_col_inds = linear_sum_assignment(cost) + matched_row_inds = torch.from_numpy(matched_row_inds).to( + bbox_pred.device) + matched_col_inds = torch.from_numpy(matched_col_inds).to( + bbox_pred.device) + + # 4. assign backgrounds and foregrounds + # assign all indices to backgrounds first + assigned_gt_inds[:] = 0 + # assign foregrounds based on matching results + assigned_gt_inds[matched_row_inds] = matched_col_inds + 1 + assigned_labels[matched_row_inds] = gt_labels[matched_col_inds] + return AssignResult( + num_gts, assigned_gt_inds, None, labels=assigned_labels) diff --git a/detection/mmdet/core/bbox/assigners/max_iou_assigner.py b/detection/mmdet/core/bbox/assigners/max_iou_assigner.py new file mode 100644 index 0000000..5cf4c4b --- /dev/null +++ b/detection/mmdet/core/bbox/assigners/max_iou_assigner.py @@ -0,0 +1,212 @@ +import torch + +from ..builder import BBOX_ASSIGNERS +from ..iou_calculators import build_iou_calculator +from .assign_result import AssignResult +from .base_assigner import BaseAssigner + + +@BBOX_ASSIGNERS.register_module() +class MaxIoUAssigner(BaseAssigner): + """Assign a corresponding gt bbox or background to each bbox. + + Each proposals will be assigned with `-1`, or a semi-positive integer + indicating the ground truth index. + + - -1: negative sample, no assigned gt + - semi-positive integer: positive sample, index (0-based) of assigned gt + + Args: + pos_iou_thr (float): IoU threshold for positive bboxes. + neg_iou_thr (float or tuple): IoU threshold for negative bboxes. + min_pos_iou (float): Minimum iou for a bbox to be considered as a + positive bbox. Positive samples can have smaller IoU than + pos_iou_thr due to the 4th step (assign max IoU sample to each gt). + gt_max_assign_all (bool): Whether to assign all bboxes with the same + highest overlap with some gt to that gt. + ignore_iof_thr (float): IoF threshold for ignoring bboxes (if + `gt_bboxes_ignore` is specified). Negative values mean not + ignoring any bboxes. + ignore_wrt_candidates (bool): Whether to compute the iof between + `bboxes` and `gt_bboxes_ignore`, or the contrary. + match_low_quality (bool): Whether to allow low quality matches. This is + usually allowed for RPN and single stage detectors, but not allowed + in the second stage. Details are demonstrated in Step 4. + gpu_assign_thr (int): The upper bound of the number of GT for GPU + assign. When the number of gt is above this threshold, will assign + on CPU device. Negative values mean not assign on CPU. + """ + + def __init__(self, + pos_iou_thr, + neg_iou_thr, + min_pos_iou=.0, + gt_max_assign_all=True, + ignore_iof_thr=-1, + ignore_wrt_candidates=True, + match_low_quality=True, + gpu_assign_thr=-1, + iou_calculator=dict(type='BboxOverlaps2D')): + self.pos_iou_thr = pos_iou_thr + self.neg_iou_thr = neg_iou_thr + self.min_pos_iou = min_pos_iou + self.gt_max_assign_all = gt_max_assign_all + self.ignore_iof_thr = ignore_iof_thr + self.ignore_wrt_candidates = ignore_wrt_candidates + self.gpu_assign_thr = gpu_assign_thr + self.match_low_quality = match_low_quality + self.iou_calculator = build_iou_calculator(iou_calculator) + + def assign(self, bboxes, gt_bboxes, gt_bboxes_ignore=None, gt_labels=None): + """Assign gt to bboxes. + + This method assign a gt bbox to every bbox (proposal/anchor), each bbox + will be assigned with -1, or a semi-positive number. -1 means negative + sample, semi-positive number is the index (0-based) of assigned gt. + The assignment is done in following steps, the order matters. + + 1. assign every bbox to the background + 2. assign proposals whose iou with all gts < neg_iou_thr to 0 + 3. for each bbox, if the iou with its nearest gt >= pos_iou_thr, + assign it to that bbox + 4. for each gt bbox, assign its nearest proposals (may be more than + one) to itself + + Args: + bboxes (Tensor): Bounding boxes to be assigned, shape(n, 4). + gt_bboxes (Tensor): Groundtruth boxes, shape (k, 4). + gt_bboxes_ignore (Tensor, optional): Ground truth bboxes that are + labelled as `ignored`, e.g., crowd boxes in COCO. + gt_labels (Tensor, optional): Label of gt_bboxes, shape (k, ). + + Returns: + :obj:`AssignResult`: The assign result. + + Example: + >>> self = MaxIoUAssigner(0.5, 0.5) + >>> bboxes = torch.Tensor([[0, 0, 10, 10], [10, 10, 20, 20]]) + >>> gt_bboxes = torch.Tensor([[0, 0, 10, 9]]) + >>> assign_result = self.assign(bboxes, gt_bboxes) + >>> expected_gt_inds = torch.LongTensor([1, 0]) + >>> assert torch.all(assign_result.gt_inds == expected_gt_inds) + """ + assign_on_cpu = True if (self.gpu_assign_thr > 0) and ( + gt_bboxes.shape[0] > self.gpu_assign_thr) else False + # compute overlap and assign gt on CPU when number of GT is large + if assign_on_cpu: + device = bboxes.device + bboxes = bboxes.cpu() + gt_bboxes = gt_bboxes.cpu() + if gt_bboxes_ignore is not None: + gt_bboxes_ignore = gt_bboxes_ignore.cpu() + if gt_labels is not None: + gt_labels = gt_labels.cpu() + + overlaps = self.iou_calculator(gt_bboxes, bboxes) + + if (self.ignore_iof_thr > 0 and gt_bboxes_ignore is not None + and gt_bboxes_ignore.numel() > 0 and bboxes.numel() > 0): + if self.ignore_wrt_candidates: + ignore_overlaps = self.iou_calculator( + bboxes, gt_bboxes_ignore, mode='iof') + ignore_max_overlaps, _ = ignore_overlaps.max(dim=1) + else: + ignore_overlaps = self.iou_calculator( + gt_bboxes_ignore, bboxes, mode='iof') + ignore_max_overlaps, _ = ignore_overlaps.max(dim=0) + overlaps[:, ignore_max_overlaps > self.ignore_iof_thr] = -1 + + assign_result = self.assign_wrt_overlaps(overlaps, gt_labels) + if assign_on_cpu: + assign_result.gt_inds = assign_result.gt_inds.to(device) + assign_result.max_overlaps = assign_result.max_overlaps.to(device) + if assign_result.labels is not None: + assign_result.labels = assign_result.labels.to(device) + return assign_result + + def assign_wrt_overlaps(self, overlaps, gt_labels=None): + """Assign w.r.t. the overlaps of bboxes with gts. + + Args: + overlaps (Tensor): Overlaps between k gt_bboxes and n bboxes, + shape(k, n). + gt_labels (Tensor, optional): Labels of k gt_bboxes, shape (k, ). + + Returns: + :obj:`AssignResult`: The assign result. + """ + num_gts, num_bboxes = overlaps.size(0), overlaps.size(1) + + # 1. assign -1 by default + assigned_gt_inds = overlaps.new_full((num_bboxes, ), + -1, + dtype=torch.long) + + if num_gts == 0 or num_bboxes == 0: + # No ground truth or boxes, return empty assignment + max_overlaps = overlaps.new_zeros((num_bboxes, )) + if num_gts == 0: + # No truth, assign everything to background + assigned_gt_inds[:] = 0 + if gt_labels is None: + assigned_labels = None + else: + assigned_labels = overlaps.new_full((num_bboxes, ), + -1, + dtype=torch.long) + return AssignResult( + num_gts, + assigned_gt_inds, + max_overlaps, + labels=assigned_labels) + + # for each anchor, which gt best overlaps with it + # for each anchor, the max iou of all gts + max_overlaps, argmax_overlaps = overlaps.max(dim=0) + # for each gt, which anchor best overlaps with it + # for each gt, the max iou of all proposals + gt_max_overlaps, gt_argmax_overlaps = overlaps.max(dim=1) + + # 2. assign negative: below + # the negative inds are set to be 0 + if isinstance(self.neg_iou_thr, float): + assigned_gt_inds[(max_overlaps >= 0) + & (max_overlaps < self.neg_iou_thr)] = 0 + elif isinstance(self.neg_iou_thr, tuple): + assert len(self.neg_iou_thr) == 2 + assigned_gt_inds[(max_overlaps >= self.neg_iou_thr[0]) + & (max_overlaps < self.neg_iou_thr[1])] = 0 + + # 3. assign positive: above positive IoU threshold + pos_inds = max_overlaps >= self.pos_iou_thr + assigned_gt_inds[pos_inds] = argmax_overlaps[pos_inds] + 1 + + if self.match_low_quality: + # Low-quality matching will overwrite the assigned_gt_inds assigned + # in Step 3. Thus, the assigned gt might not be the best one for + # prediction. + # For example, if bbox A has 0.9 and 0.8 iou with GT bbox 1 & 2, + # bbox 1 will be assigned as the best target for bbox A in step 3. + # However, if GT bbox 2's gt_argmax_overlaps = A, bbox A's + # assigned_gt_inds will be overwritten to be bbox B. + # This might be the reason that it is not used in ROI Heads. + for i in range(num_gts): + if gt_max_overlaps[i] >= self.min_pos_iou: + if self.gt_max_assign_all: + max_iou_inds = overlaps[i, :] == gt_max_overlaps[i] + assigned_gt_inds[max_iou_inds] = i + 1 + else: + assigned_gt_inds[gt_argmax_overlaps[i]] = i + 1 + + if gt_labels is not None: + assigned_labels = assigned_gt_inds.new_full((num_bboxes, ), -1) + pos_inds = torch.nonzero( + assigned_gt_inds > 0, as_tuple=False).squeeze() + if pos_inds.numel() > 0: + assigned_labels[pos_inds] = gt_labels[ + assigned_gt_inds[pos_inds] - 1] + else: + assigned_labels = None + + return AssignResult( + num_gts, assigned_gt_inds, max_overlaps, labels=assigned_labels) diff --git a/detection/mmdet/core/bbox/assigners/point_assigner.py b/detection/mmdet/core/bbox/assigners/point_assigner.py new file mode 100644 index 0000000..fb8f5e4 --- /dev/null +++ b/detection/mmdet/core/bbox/assigners/point_assigner.py @@ -0,0 +1,133 @@ +import torch + +from ..builder import BBOX_ASSIGNERS +from .assign_result import AssignResult +from .base_assigner import BaseAssigner + + +@BBOX_ASSIGNERS.register_module() +class PointAssigner(BaseAssigner): + """Assign a corresponding gt bbox or background to each point. + + Each proposals will be assigned with `0`, or a positive integer + indicating the ground truth index. + + - 0: negative sample, no assigned gt + - positive integer: positive sample, index (1-based) of assigned gt + """ + + def __init__(self, scale=4, pos_num=3): + self.scale = scale + self.pos_num = pos_num + + def assign(self, points, gt_bboxes, gt_bboxes_ignore=None, gt_labels=None): + """Assign gt to points. + + This method assign a gt bbox to every points set, each points set + will be assigned with the background_label (-1), or a label number. + -1 is background, and semi-positive number is the index (0-based) of + assigned gt. + The assignment is done in following steps, the order matters. + + 1. assign every points to the background_label (-1) + 2. A point is assigned to some gt bbox if + (i) the point is within the k closest points to the gt bbox + (ii) the distance between this point and the gt is smaller than + other gt bboxes + + Args: + points (Tensor): points to be assigned, shape(n, 3) while last + dimension stands for (x, y, stride). + gt_bboxes (Tensor): Groundtruth boxes, shape (k, 4). + gt_bboxes_ignore (Tensor, optional): Ground truth bboxes that are + labelled as `ignored`, e.g., crowd boxes in COCO. + NOTE: currently unused. + gt_labels (Tensor, optional): Label of gt_bboxes, shape (k, ). + + Returns: + :obj:`AssignResult`: The assign result. + """ + num_points = points.shape[0] + num_gts = gt_bboxes.shape[0] + + if num_gts == 0 or num_points == 0: + # If no truth assign everything to the background + assigned_gt_inds = points.new_full((num_points, ), + 0, + dtype=torch.long) + if gt_labels is None: + assigned_labels = None + else: + assigned_labels = points.new_full((num_points, ), + -1, + dtype=torch.long) + return AssignResult( + num_gts, assigned_gt_inds, None, labels=assigned_labels) + + points_xy = points[:, :2] + points_stride = points[:, 2] + points_lvl = torch.log2( + points_stride).int() # [3...,4...,5...,6...,7...] + lvl_min, lvl_max = points_lvl.min(), points_lvl.max() + + # assign gt box + gt_bboxes_xy = (gt_bboxes[:, :2] + gt_bboxes[:, 2:]) / 2 + gt_bboxes_wh = (gt_bboxes[:, 2:] - gt_bboxes[:, :2]).clamp(min=1e-6) + scale = self.scale + gt_bboxes_lvl = ((torch.log2(gt_bboxes_wh[:, 0] / scale) + + torch.log2(gt_bboxes_wh[:, 1] / scale)) / 2).int() + gt_bboxes_lvl = torch.clamp(gt_bboxes_lvl, min=lvl_min, max=lvl_max) + + # stores the assigned gt index of each point + assigned_gt_inds = points.new_zeros((num_points, ), dtype=torch.long) + # stores the assigned gt dist (to this point) of each point + assigned_gt_dist = points.new_full((num_points, ), float('inf')) + points_range = torch.arange(points.shape[0]) + + for idx in range(num_gts): + gt_lvl = gt_bboxes_lvl[idx] + # get the index of points in this level + lvl_idx = gt_lvl == points_lvl + points_index = points_range[lvl_idx] + # get the points in this level + lvl_points = points_xy[lvl_idx, :] + # get the center point of gt + gt_point = gt_bboxes_xy[[idx], :] + # get width and height of gt + gt_wh = gt_bboxes_wh[[idx], :] + # compute the distance between gt center and + # all points in this level + points_gt_dist = ((lvl_points - gt_point) / gt_wh).norm(dim=1) + # find the nearest k points to gt center in this level + min_dist, min_dist_index = torch.topk( + points_gt_dist, self.pos_num, largest=False) + # the index of nearest k points to gt center in this level + min_dist_points_index = points_index[min_dist_index] + # The less_than_recorded_index stores the index + # of min_dist that is less then the assigned_gt_dist. Where + # assigned_gt_dist stores the dist from previous assigned gt + # (if exist) to each point. + less_than_recorded_index = min_dist < assigned_gt_dist[ + min_dist_points_index] + # The min_dist_points_index stores the index of points satisfy: + # (1) it is k nearest to current gt center in this level. + # (2) it is closer to current gt center than other gt center. + min_dist_points_index = min_dist_points_index[ + less_than_recorded_index] + # assign the result + assigned_gt_inds[min_dist_points_index] = idx + 1 + assigned_gt_dist[min_dist_points_index] = min_dist[ + less_than_recorded_index] + + if gt_labels is not None: + assigned_labels = assigned_gt_inds.new_full((num_points, ), -1) + pos_inds = torch.nonzero( + assigned_gt_inds > 0, as_tuple=False).squeeze() + if pos_inds.numel() > 0: + assigned_labels[pos_inds] = gt_labels[ + assigned_gt_inds[pos_inds] - 1] + else: + assigned_labels = None + + return AssignResult( + num_gts, assigned_gt_inds, None, labels=assigned_labels) diff --git a/detection/mmdet/core/bbox/assigners/region_assigner.py b/detection/mmdet/core/bbox/assigners/region_assigner.py new file mode 100644 index 0000000..2e8464b --- /dev/null +++ b/detection/mmdet/core/bbox/assigners/region_assigner.py @@ -0,0 +1,221 @@ +import torch + +from mmdet.core import anchor_inside_flags +from ..builder import BBOX_ASSIGNERS +from .assign_result import AssignResult +from .base_assigner import BaseAssigner + + +def calc_region(bbox, ratio, stride, featmap_size=None): + """Calculate region of the box defined by the ratio, the ratio is from the + center of the box to every edge.""" + # project bbox on the feature + f_bbox = bbox / stride + x1 = torch.round((1 - ratio) * f_bbox[0] + ratio * f_bbox[2]) + y1 = torch.round((1 - ratio) * f_bbox[1] + ratio * f_bbox[3]) + x2 = torch.round(ratio * f_bbox[0] + (1 - ratio) * f_bbox[2]) + y2 = torch.round(ratio * f_bbox[1] + (1 - ratio) * f_bbox[3]) + if featmap_size is not None: + x1 = x1.clamp(min=0, max=featmap_size[1]) + y1 = y1.clamp(min=0, max=featmap_size[0]) + x2 = x2.clamp(min=0, max=featmap_size[1]) + y2 = y2.clamp(min=0, max=featmap_size[0]) + return (x1, y1, x2, y2) + + +def anchor_ctr_inside_region_flags(anchors, stride, region): + """Get the flag indicate whether anchor centers are inside regions.""" + x1, y1, x2, y2 = region + f_anchors = anchors / stride + x = (f_anchors[:, 0] + f_anchors[:, 2]) * 0.5 + y = (f_anchors[:, 1] + f_anchors[:, 3]) * 0.5 + flags = (x >= x1) & (x <= x2) & (y >= y1) & (y <= y2) + return flags + + +@BBOX_ASSIGNERS.register_module() +class RegionAssigner(BaseAssigner): + """Assign a corresponding gt bbox or background to each bbox. + + Each proposals will be assigned with `-1`, `0`, or a positive integer + indicating the ground truth index. + + - -1: don't care + - 0: negative sample, no assigned gt + - positive integer: positive sample, index (1-based) of assigned gt + + Args: + center_ratio: ratio of the region in the center of the bbox to + define positive sample. + ignore_ratio: ratio of the region to define ignore samples. + """ + + def __init__(self, center_ratio=0.2, ignore_ratio=0.5): + self.center_ratio = center_ratio + self.ignore_ratio = ignore_ratio + + def assign(self, + mlvl_anchors, + mlvl_valid_flags, + gt_bboxes, + img_meta, + featmap_sizes, + anchor_scale, + anchor_strides, + gt_bboxes_ignore=None, + gt_labels=None, + allowed_border=0): + """Assign gt to anchors. + + This method assign a gt bbox to every bbox (proposal/anchor), each bbox + will be assigned with -1, 0, or a positive number. -1 means don't care, + 0 means negative sample, positive number is the index (1-based) of + assigned gt. + The assignment is done in following steps, the order matters. + + 1. Assign every anchor to 0 (negative) + For each gt_bboxes: + 2. Compute ignore flags based on ignore_region then + assign -1 to anchors w.r.t. ignore flags + 3. Compute pos flags based on center_region then + assign gt_bboxes to anchors w.r.t. pos flags + 4. Compute ignore flags based on adjacent anchor lvl then + assign -1 to anchors w.r.t. ignore flags + 5. Assign anchor outside of image to -1 + + Args: + mlvl_anchors (list[Tensor]): Multi level anchors. + mlvl_valid_flags (list[Tensor]): Multi level valid flags. + gt_bboxes (Tensor): Ground truth bboxes of image + img_meta (dict): Meta info of image. + featmap_sizes (list[Tensor]): Feature mapsize each level + anchor_scale (int): Scale of the anchor. + anchor_strides (list[int]): Stride of the anchor. + gt_bboxes (Tensor): Groundtruth boxes, shape (k, 4). + gt_bboxes_ignore (Tensor, optional): Ground truth bboxes that are + labelled as `ignored`, e.g., crowd boxes in COCO. + gt_labels (Tensor, optional): Label of gt_bboxes, shape (k, ). + allowed_border (int, optional): The border to allow the valid + anchor. Defaults to 0. + + Returns: + :obj:`AssignResult`: The assign result. + """ + if gt_bboxes_ignore is not None: + raise NotImplementedError + + num_gts = gt_bboxes.shape[0] + num_bboxes = sum(x.shape[0] for x in mlvl_anchors) + + if num_gts == 0 or num_bboxes == 0: + # No ground truth or boxes, return empty assignment + max_overlaps = gt_bboxes.new_zeros((num_bboxes, )) + assigned_gt_inds = gt_bboxes.new_zeros((num_bboxes, ), + dtype=torch.long) + if gt_labels is None: + assigned_labels = None + else: + assigned_labels = gt_bboxes.new_full((num_bboxes, ), + -1, + dtype=torch.long) + return AssignResult( + num_gts, + assigned_gt_inds, + max_overlaps, + labels=assigned_labels) + + num_lvls = len(mlvl_anchors) + r1 = (1 - self.center_ratio) / 2 + r2 = (1 - self.ignore_ratio) / 2 + + scale = torch.sqrt((gt_bboxes[:, 2] - gt_bboxes[:, 0]) * + (gt_bboxes[:, 3] - gt_bboxes[:, 1])) + min_anchor_size = scale.new_full( + (1, ), float(anchor_scale * anchor_strides[0])) + target_lvls = torch.floor( + torch.log2(scale) - torch.log2(min_anchor_size) + 0.5) + target_lvls = target_lvls.clamp(min=0, max=num_lvls - 1).long() + + # 1. assign 0 (negative) by default + mlvl_assigned_gt_inds = [] + mlvl_ignore_flags = [] + for lvl in range(num_lvls): + h, w = featmap_sizes[lvl] + assert h * w == mlvl_anchors[lvl].shape[0] + assigned_gt_inds = gt_bboxes.new_full((h * w, ), + 0, + dtype=torch.long) + ignore_flags = torch.zeros_like(assigned_gt_inds) + mlvl_assigned_gt_inds.append(assigned_gt_inds) + mlvl_ignore_flags.append(ignore_flags) + + for gt_id in range(num_gts): + lvl = target_lvls[gt_id].item() + featmap_size = featmap_sizes[lvl] + stride = anchor_strides[lvl] + anchors = mlvl_anchors[lvl] + gt_bbox = gt_bboxes[gt_id, :4] + + # Compute regions + ignore_region = calc_region(gt_bbox, r2, stride, featmap_size) + ctr_region = calc_region(gt_bbox, r1, stride, featmap_size) + + # 2. Assign -1 to ignore flags + ignore_flags = anchor_ctr_inside_region_flags( + anchors, stride, ignore_region) + mlvl_assigned_gt_inds[lvl][ignore_flags] = -1 + + # 3. Assign gt_bboxes to pos flags + pos_flags = anchor_ctr_inside_region_flags(anchors, stride, + ctr_region) + mlvl_assigned_gt_inds[lvl][pos_flags] = gt_id + 1 + + # 4. Assign -1 to ignore adjacent lvl + if lvl > 0: + d_lvl = lvl - 1 + d_anchors = mlvl_anchors[d_lvl] + d_featmap_size = featmap_sizes[d_lvl] + d_stride = anchor_strides[d_lvl] + d_ignore_region = calc_region(gt_bbox, r2, d_stride, + d_featmap_size) + ignore_flags = anchor_ctr_inside_region_flags( + d_anchors, d_stride, d_ignore_region) + mlvl_ignore_flags[d_lvl][ignore_flags] = 1 + if lvl < num_lvls - 1: + u_lvl = lvl + 1 + u_anchors = mlvl_anchors[u_lvl] + u_featmap_size = featmap_sizes[u_lvl] + u_stride = anchor_strides[u_lvl] + u_ignore_region = calc_region(gt_bbox, r2, u_stride, + u_featmap_size) + ignore_flags = anchor_ctr_inside_region_flags( + u_anchors, u_stride, u_ignore_region) + mlvl_ignore_flags[u_lvl][ignore_flags] = 1 + + # 4. (cont.) Assign -1 to ignore adjacent lvl + for lvl in range(num_lvls): + ignore_flags = mlvl_ignore_flags[lvl] + mlvl_assigned_gt_inds[lvl][ignore_flags] = -1 + + # 5. Assign -1 to anchor outside of image + flat_assigned_gt_inds = torch.cat(mlvl_assigned_gt_inds) + flat_anchors = torch.cat(mlvl_anchors) + flat_valid_flags = torch.cat(mlvl_valid_flags) + assert (flat_assigned_gt_inds.shape[0] == flat_anchors.shape[0] == + flat_valid_flags.shape[0]) + inside_flags = anchor_inside_flags(flat_anchors, flat_valid_flags, + img_meta['img_shape'], + allowed_border) + outside_flags = ~inside_flags + flat_assigned_gt_inds[outside_flags] = -1 + + if gt_labels is not None: + assigned_labels = torch.zeros_like(flat_assigned_gt_inds) + pos_flags = assigned_gt_inds > 0 + assigned_labels[pos_flags] = gt_labels[ + flat_assigned_gt_inds[pos_flags] - 1] + else: + assigned_labels = None + + return AssignResult( + num_gts, flat_assigned_gt_inds, None, labels=assigned_labels) diff --git a/detection/mmdet/core/bbox/builder.py b/detection/mmdet/core/bbox/builder.py new file mode 100644 index 0000000..682683b --- /dev/null +++ b/detection/mmdet/core/bbox/builder.py @@ -0,0 +1,20 @@ +from mmcv.utils import Registry, build_from_cfg + +BBOX_ASSIGNERS = Registry('bbox_assigner') +BBOX_SAMPLERS = Registry('bbox_sampler') +BBOX_CODERS = Registry('bbox_coder') + + +def build_assigner(cfg, **default_args): + """Builder of box assigner.""" + return build_from_cfg(cfg, BBOX_ASSIGNERS, default_args) + + +def build_sampler(cfg, **default_args): + """Builder of box sampler.""" + return build_from_cfg(cfg, BBOX_SAMPLERS, default_args) + + +def build_bbox_coder(cfg, **default_args): + """Builder of box coder.""" + return build_from_cfg(cfg, BBOX_CODERS, default_args) diff --git a/detection/mmdet/core/bbox/coder/__init__.py b/detection/mmdet/core/bbox/coder/__init__.py new file mode 100644 index 0000000..ae455ba --- /dev/null +++ b/detection/mmdet/core/bbox/coder/__init__.py @@ -0,0 +1,13 @@ +from .base_bbox_coder import BaseBBoxCoder +from .bucketing_bbox_coder import BucketingBBoxCoder +from .delta_xywh_bbox_coder import DeltaXYWHBBoxCoder +from .legacy_delta_xywh_bbox_coder import LegacyDeltaXYWHBBoxCoder +from .pseudo_bbox_coder import PseudoBBoxCoder +from .tblr_bbox_coder import TBLRBBoxCoder +from .yolo_bbox_coder import YOLOBBoxCoder + +__all__ = [ + 'BaseBBoxCoder', 'PseudoBBoxCoder', 'DeltaXYWHBBoxCoder', + 'LegacyDeltaXYWHBBoxCoder', 'TBLRBBoxCoder', 'YOLOBBoxCoder', + 'BucketingBBoxCoder' +] diff --git a/detection/mmdet/core/bbox/coder/base_bbox_coder.py b/detection/mmdet/core/bbox/coder/base_bbox_coder.py new file mode 100644 index 0000000..cf0b34c --- /dev/null +++ b/detection/mmdet/core/bbox/coder/base_bbox_coder.py @@ -0,0 +1,17 @@ +from abc import ABCMeta, abstractmethod + + +class BaseBBoxCoder(metaclass=ABCMeta): + """Base bounding box coder.""" + + def __init__(self, **kwargs): + pass + + @abstractmethod + def encode(self, bboxes, gt_bboxes): + """Encode deltas between bboxes and ground truth boxes.""" + + @abstractmethod + def decode(self, bboxes, bboxes_pred): + """Decode the predicted bboxes according to prediction and base + boxes.""" diff --git a/detection/mmdet/core/bbox/coder/bucketing_bbox_coder.py b/detection/mmdet/core/bbox/coder/bucketing_bbox_coder.py new file mode 100644 index 0000000..92d24b4 --- /dev/null +++ b/detection/mmdet/core/bbox/coder/bucketing_bbox_coder.py @@ -0,0 +1,350 @@ +import mmcv +import numpy as np +import torch +import torch.nn.functional as F + +from ..builder import BBOX_CODERS +from ..transforms import bbox_rescale +from .base_bbox_coder import BaseBBoxCoder + + +@BBOX_CODERS.register_module() +class BucketingBBoxCoder(BaseBBoxCoder): + """Bucketing BBox Coder for Side-Aware Boundary Localization (SABL). + + Boundary Localization with Bucketing and Bucketing Guided Rescoring + are implemented here. + + Please refer to https://arxiv.org/abs/1912.04260 for more details. + + Args: + num_buckets (int): Number of buckets. + scale_factor (int): Scale factor of proposals to generate buckets. + offset_topk (int): Topk buckets are used to generate + bucket fine regression targets. Defaults to 2. + offset_upperbound (float): Offset upperbound to generate + bucket fine regression targets. + To avoid too large offset displacements. Defaults to 1.0. + cls_ignore_neighbor (bool): Ignore second nearest bucket or Not. + Defaults to True. + clip_border (bool, optional): Whether clip the objects outside the + border of the image. Defaults to True. + """ + + def __init__(self, + num_buckets, + scale_factor, + offset_topk=2, + offset_upperbound=1.0, + cls_ignore_neighbor=True, + clip_border=True): + super(BucketingBBoxCoder, self).__init__() + self.num_buckets = num_buckets + self.scale_factor = scale_factor + self.offset_topk = offset_topk + self.offset_upperbound = offset_upperbound + self.cls_ignore_neighbor = cls_ignore_neighbor + self.clip_border = clip_border + + def encode(self, bboxes, gt_bboxes): + """Get bucketing estimation and fine regression targets during + training. + + Args: + bboxes (torch.Tensor): source boxes, e.g., object proposals. + gt_bboxes (torch.Tensor): target of the transformation, e.g., + ground truth boxes. + + Returns: + encoded_bboxes(tuple[Tensor]): bucketing estimation + and fine regression targets and weights + """ + + assert bboxes.size(0) == gt_bboxes.size(0) + assert bboxes.size(-1) == gt_bboxes.size(-1) == 4 + encoded_bboxes = bbox2bucket(bboxes, gt_bboxes, self.num_buckets, + self.scale_factor, self.offset_topk, + self.offset_upperbound, + self.cls_ignore_neighbor) + return encoded_bboxes + + def decode(self, bboxes, pred_bboxes, max_shape=None): + """Apply transformation `pred_bboxes` to `boxes`. + Args: + boxes (torch.Tensor): Basic boxes. + pred_bboxes (torch.Tensor): Predictions for bucketing estimation + and fine regression + max_shape (tuple[int], optional): Maximum shape of boxes. + Defaults to None. + + Returns: + torch.Tensor: Decoded boxes. + """ + assert len(pred_bboxes) == 2 + cls_preds, offset_preds = pred_bboxes + assert cls_preds.size(0) == bboxes.size(0) and offset_preds.size( + 0) == bboxes.size(0) + decoded_bboxes = bucket2bbox(bboxes, cls_preds, offset_preds, + self.num_buckets, self.scale_factor, + max_shape, self.clip_border) + + return decoded_bboxes + + +@mmcv.jit(coderize=True) +def generat_buckets(proposals, num_buckets, scale_factor=1.0): + """Generate buckets w.r.t bucket number and scale factor of proposals. + + Args: + proposals (Tensor): Shape (n, 4) + num_buckets (int): Number of buckets. + scale_factor (float): Scale factor to rescale proposals. + + Returns: + tuple[Tensor]: (bucket_w, bucket_h, l_buckets, r_buckets, + t_buckets, d_buckets) + + - bucket_w: Width of buckets on x-axis. Shape (n, ). + - bucket_h: Height of buckets on y-axis. Shape (n, ). + - l_buckets: Left buckets. Shape (n, ceil(side_num/2)). + - r_buckets: Right buckets. Shape (n, ceil(side_num/2)). + - t_buckets: Top buckets. Shape (n, ceil(side_num/2)). + - d_buckets: Down buckets. Shape (n, ceil(side_num/2)). + """ + proposals = bbox_rescale(proposals, scale_factor) + + # number of buckets in each side + side_num = int(np.ceil(num_buckets / 2.0)) + pw = proposals[..., 2] - proposals[..., 0] + ph = proposals[..., 3] - proposals[..., 1] + px1 = proposals[..., 0] + py1 = proposals[..., 1] + px2 = proposals[..., 2] + py2 = proposals[..., 3] + + bucket_w = pw / num_buckets + bucket_h = ph / num_buckets + + # left buckets + l_buckets = px1[:, None] + (0.5 + torch.arange( + 0, side_num).to(proposals).float())[None, :] * bucket_w[:, None] + # right buckets + r_buckets = px2[:, None] - (0.5 + torch.arange( + 0, side_num).to(proposals).float())[None, :] * bucket_w[:, None] + # top buckets + t_buckets = py1[:, None] + (0.5 + torch.arange( + 0, side_num).to(proposals).float())[None, :] * bucket_h[:, None] + # down buckets + d_buckets = py2[:, None] - (0.5 + torch.arange( + 0, side_num).to(proposals).float())[None, :] * bucket_h[:, None] + return bucket_w, bucket_h, l_buckets, r_buckets, t_buckets, d_buckets + + +@mmcv.jit(coderize=True) +def bbox2bucket(proposals, + gt, + num_buckets, + scale_factor, + offset_topk=2, + offset_upperbound=1.0, + cls_ignore_neighbor=True): + """Generate buckets estimation and fine regression targets. + + Args: + proposals (Tensor): Shape (n, 4) + gt (Tensor): Shape (n, 4) + num_buckets (int): Number of buckets. + scale_factor (float): Scale factor to rescale proposals. + offset_topk (int): Topk buckets are used to generate + bucket fine regression targets. Defaults to 2. + offset_upperbound (float): Offset allowance to generate + bucket fine regression targets. + To avoid too large offset displacements. Defaults to 1.0. + cls_ignore_neighbor (bool): Ignore second nearest bucket or Not. + Defaults to True. + + Returns: + tuple[Tensor]: (offsets, offsets_weights, bucket_labels, cls_weights). + + - offsets: Fine regression targets. \ + Shape (n, num_buckets*2). + - offsets_weights: Fine regression weights. \ + Shape (n, num_buckets*2). + - bucket_labels: Bucketing estimation labels. \ + Shape (n, num_buckets*2). + - cls_weights: Bucketing estimation weights. \ + Shape (n, num_buckets*2). + """ + assert proposals.size() == gt.size() + + # generate buckets + proposals = proposals.float() + gt = gt.float() + (bucket_w, bucket_h, l_buckets, r_buckets, t_buckets, + d_buckets) = generat_buckets(proposals, num_buckets, scale_factor) + + gx1 = gt[..., 0] + gy1 = gt[..., 1] + gx2 = gt[..., 2] + gy2 = gt[..., 3] + + # generate offset targets and weights + # offsets from buckets to gts + l_offsets = (l_buckets - gx1[:, None]) / bucket_w[:, None] + r_offsets = (r_buckets - gx2[:, None]) / bucket_w[:, None] + t_offsets = (t_buckets - gy1[:, None]) / bucket_h[:, None] + d_offsets = (d_buckets - gy2[:, None]) / bucket_h[:, None] + + # select top-k nearset buckets + l_topk, l_label = l_offsets.abs().topk( + offset_topk, dim=1, largest=False, sorted=True) + r_topk, r_label = r_offsets.abs().topk( + offset_topk, dim=1, largest=False, sorted=True) + t_topk, t_label = t_offsets.abs().topk( + offset_topk, dim=1, largest=False, sorted=True) + d_topk, d_label = d_offsets.abs().topk( + offset_topk, dim=1, largest=False, sorted=True) + + offset_l_weights = l_offsets.new_zeros(l_offsets.size()) + offset_r_weights = r_offsets.new_zeros(r_offsets.size()) + offset_t_weights = t_offsets.new_zeros(t_offsets.size()) + offset_d_weights = d_offsets.new_zeros(d_offsets.size()) + inds = torch.arange(0, proposals.size(0)).to(proposals).long() + + # generate offset weights of top-k nearset buckets + for k in range(offset_topk): + if k >= 1: + offset_l_weights[inds, l_label[:, + k]] = (l_topk[:, k] < + offset_upperbound).float() + offset_r_weights[inds, r_label[:, + k]] = (r_topk[:, k] < + offset_upperbound).float() + offset_t_weights[inds, t_label[:, + k]] = (t_topk[:, k] < + offset_upperbound).float() + offset_d_weights[inds, d_label[:, + k]] = (d_topk[:, k] < + offset_upperbound).float() + else: + offset_l_weights[inds, l_label[:, k]] = 1.0 + offset_r_weights[inds, r_label[:, k]] = 1.0 + offset_t_weights[inds, t_label[:, k]] = 1.0 + offset_d_weights[inds, d_label[:, k]] = 1.0 + + offsets = torch.cat([l_offsets, r_offsets, t_offsets, d_offsets], dim=-1) + offsets_weights = torch.cat([ + offset_l_weights, offset_r_weights, offset_t_weights, offset_d_weights + ], + dim=-1) + + # generate bucket labels and weight + side_num = int(np.ceil(num_buckets / 2.0)) + labels = torch.stack( + [l_label[:, 0], r_label[:, 0], t_label[:, 0], d_label[:, 0]], dim=-1) + + batch_size = labels.size(0) + bucket_labels = F.one_hot(labels.view(-1), side_num).view(batch_size, + -1).float() + bucket_cls_l_weights = (l_offsets.abs() < 1).float() + bucket_cls_r_weights = (r_offsets.abs() < 1).float() + bucket_cls_t_weights = (t_offsets.abs() < 1).float() + bucket_cls_d_weights = (d_offsets.abs() < 1).float() + bucket_cls_weights = torch.cat([ + bucket_cls_l_weights, bucket_cls_r_weights, bucket_cls_t_weights, + bucket_cls_d_weights + ], + dim=-1) + # ignore second nearest buckets for cls if necessary + if cls_ignore_neighbor: + bucket_cls_weights = (~((bucket_cls_weights == 1) & + (bucket_labels == 0))).float() + else: + bucket_cls_weights[:] = 1.0 + return offsets, offsets_weights, bucket_labels, bucket_cls_weights + + +@mmcv.jit(coderize=True) +def bucket2bbox(proposals, + cls_preds, + offset_preds, + num_buckets, + scale_factor=1.0, + max_shape=None, + clip_border=True): + """Apply bucketing estimation (cls preds) and fine regression (offset + preds) to generate det bboxes. + + Args: + proposals (Tensor): Boxes to be transformed. Shape (n, 4) + cls_preds (Tensor): bucketing estimation. Shape (n, num_buckets*2). + offset_preds (Tensor): fine regression. Shape (n, num_buckets*2). + num_buckets (int): Number of buckets. + scale_factor (float): Scale factor to rescale proposals. + max_shape (tuple[int, int]): Maximum bounds for boxes. specifies (H, W) + clip_border (bool, optional): Whether clip the objects outside the + border of the image. Defaults to True. + + Returns: + tuple[Tensor]: (bboxes, loc_confidence). + + - bboxes: predicted bboxes. Shape (n, 4) + - loc_confidence: localization confidence of predicted bboxes. + Shape (n,). + """ + + side_num = int(np.ceil(num_buckets / 2.0)) + cls_preds = cls_preds.view(-1, side_num) + offset_preds = offset_preds.view(-1, side_num) + + scores = F.softmax(cls_preds, dim=1) + score_topk, score_label = scores.topk(2, dim=1, largest=True, sorted=True) + + rescaled_proposals = bbox_rescale(proposals, scale_factor) + + pw = rescaled_proposals[..., 2] - rescaled_proposals[..., 0] + ph = rescaled_proposals[..., 3] - rescaled_proposals[..., 1] + px1 = rescaled_proposals[..., 0] + py1 = rescaled_proposals[..., 1] + px2 = rescaled_proposals[..., 2] + py2 = rescaled_proposals[..., 3] + + bucket_w = pw / num_buckets + bucket_h = ph / num_buckets + + score_inds_l = score_label[0::4, 0] + score_inds_r = score_label[1::4, 0] + score_inds_t = score_label[2::4, 0] + score_inds_d = score_label[3::4, 0] + l_buckets = px1 + (0.5 + score_inds_l.float()) * bucket_w + r_buckets = px2 - (0.5 + score_inds_r.float()) * bucket_w + t_buckets = py1 + (0.5 + score_inds_t.float()) * bucket_h + d_buckets = py2 - (0.5 + score_inds_d.float()) * bucket_h + + offsets = offset_preds.view(-1, 4, side_num) + inds = torch.arange(proposals.size(0)).to(proposals).long() + l_offsets = offsets[:, 0, :][inds, score_inds_l] + r_offsets = offsets[:, 1, :][inds, score_inds_r] + t_offsets = offsets[:, 2, :][inds, score_inds_t] + d_offsets = offsets[:, 3, :][inds, score_inds_d] + + x1 = l_buckets - l_offsets * bucket_w + x2 = r_buckets - r_offsets * bucket_w + y1 = t_buckets - t_offsets * bucket_h + y2 = d_buckets - d_offsets * bucket_h + + if clip_border and max_shape is not None: + x1 = x1.clamp(min=0, max=max_shape[1] - 1) + y1 = y1.clamp(min=0, max=max_shape[0] - 1) + x2 = x2.clamp(min=0, max=max_shape[1] - 1) + y2 = y2.clamp(min=0, max=max_shape[0] - 1) + bboxes = torch.cat([x1[:, None], y1[:, None], x2[:, None], y2[:, None]], + dim=-1) + + # bucketing guided rescoring + loc_confidence = score_topk[:, 0] + top2_neighbor_inds = (score_label[:, 0] - score_label[:, 1]).abs() == 1 + loc_confidence += score_topk[:, 1] * top2_neighbor_inds.float() + loc_confidence = loc_confidence.view(-1, 4).mean(dim=1) + + return bboxes, loc_confidence diff --git a/detection/mmdet/core/bbox/coder/delta_xywh_bbox_coder.py b/detection/mmdet/core/bbox/coder/delta_xywh_bbox_coder.py new file mode 100644 index 0000000..da31718 --- /dev/null +++ b/detection/mmdet/core/bbox/coder/delta_xywh_bbox_coder.py @@ -0,0 +1,237 @@ +import mmcv +import numpy as np +import torch + +from ..builder import BBOX_CODERS +from .base_bbox_coder import BaseBBoxCoder + + +@BBOX_CODERS.register_module() +class DeltaXYWHBBoxCoder(BaseBBoxCoder): + """Delta XYWH BBox coder. + + Following the practice in `R-CNN `_, + this coder encodes bbox (x1, y1, x2, y2) into delta (dx, dy, dw, dh) and + decodes delta (dx, dy, dw, dh) back to original bbox (x1, y1, x2, y2). + + Args: + target_means (Sequence[float]): Denormalizing means of target for + delta coordinates + target_stds (Sequence[float]): Denormalizing standard deviation of + target for delta coordinates + clip_border (bool, optional): Whether clip the objects outside the + border of the image. Defaults to True. + """ + + def __init__(self, + target_means=(0., 0., 0., 0.), + target_stds=(1., 1., 1., 1.), + clip_border=True): + super(BaseBBoxCoder, self).__init__() + self.means = target_means + self.stds = target_stds + self.clip_border = clip_border + + def encode(self, bboxes, gt_bboxes): + """Get box regression transformation deltas that can be used to + transform the ``bboxes`` into the ``gt_bboxes``. + + Args: + bboxes (torch.Tensor): Source boxes, e.g., object proposals. + gt_bboxes (torch.Tensor): Target of the transformation, e.g., + ground-truth boxes. + + Returns: + torch.Tensor: Box transformation deltas + """ + + assert bboxes.size(0) == gt_bboxes.size(0) + assert bboxes.size(-1) == gt_bboxes.size(-1) == 4 + encoded_bboxes = bbox2delta(bboxes, gt_bboxes, self.means, self.stds) + return encoded_bboxes + + def decode(self, + bboxes, + pred_bboxes, + max_shape=None, + wh_ratio_clip=16 / 1000): + """Apply transformation `pred_bboxes` to `boxes`. + + Args: + bboxes (torch.Tensor): Basic boxes. Shape (B, N, 4) or (N, 4) + pred_bboxes (Tensor): Encoded offsets with respect to each roi. + Has shape (B, N, num_classes * 4) or (B, N, 4) or + (N, num_classes * 4) or (N, 4). Note N = num_anchors * W * H + when rois is a grid of anchors.Offset encoding follows [1]_. + max_shape (Sequence[int] or torch.Tensor or Sequence[ + Sequence[int]],optional): Maximum bounds for boxes, specifies + (H, W, C) or (H, W). If bboxes shape is (B, N, 4), then + the max_shape should be a Sequence[Sequence[int]] + and the length of max_shape should also be B. + wh_ratio_clip (float, optional): The allowed ratio between + width and height. + + Returns: + torch.Tensor: Decoded boxes. + """ + + assert pred_bboxes.size(0) == bboxes.size(0) + if pred_bboxes.ndim == 3: + assert pred_bboxes.size(1) == bboxes.size(1) + decoded_bboxes = delta2bbox(bboxes, pred_bboxes, self.means, self.stds, + max_shape, wh_ratio_clip, self.clip_border) + + return decoded_bboxes + + +@mmcv.jit(coderize=True) +def bbox2delta(proposals, gt, means=(0., 0., 0., 0.), stds=(1., 1., 1., 1.)): + """Compute deltas of proposals w.r.t. gt. + + We usually compute the deltas of x, y, w, h of proposals w.r.t ground + truth bboxes to get regression target. + This is the inverse function of :func:`delta2bbox`. + + Args: + proposals (Tensor): Boxes to be transformed, shape (N, ..., 4) + gt (Tensor): Gt bboxes to be used as base, shape (N, ..., 4) + means (Sequence[float]): Denormalizing means for delta coordinates + stds (Sequence[float]): Denormalizing standard deviation for delta + coordinates + + Returns: + Tensor: deltas with shape (N, 4), where columns represent dx, dy, + dw, dh. + """ + assert proposals.size() == gt.size() + + proposals = proposals.float() + gt = gt.float() + px = (proposals[..., 0] + proposals[..., 2]) * 0.5 + py = (proposals[..., 1] + proposals[..., 3]) * 0.5 + pw = proposals[..., 2] - proposals[..., 0] + ph = proposals[..., 3] - proposals[..., 1] + + gx = (gt[..., 0] + gt[..., 2]) * 0.5 + gy = (gt[..., 1] + gt[..., 3]) * 0.5 + gw = gt[..., 2] - gt[..., 0] + gh = gt[..., 3] - gt[..., 1] + + dx = (gx - px) / pw + dy = (gy - py) / ph + dw = torch.log(gw / pw) + dh = torch.log(gh / ph) + deltas = torch.stack([dx, dy, dw, dh], dim=-1) + + means = deltas.new_tensor(means).unsqueeze(0) + stds = deltas.new_tensor(stds).unsqueeze(0) + deltas = deltas.sub_(means).div_(stds) + + return deltas + + +@mmcv.jit(coderize=True) +def delta2bbox(rois, + deltas, + means=(0., 0., 0., 0.), + stds=(1., 1., 1., 1.), + max_shape=None, + wh_ratio_clip=16 / 1000, + clip_border=True): + """Apply deltas to shift/scale base boxes. + + Typically the rois are anchor or proposed bounding boxes and the deltas are + network outputs used to shift/scale those boxes. + This is the inverse function of :func:`bbox2delta`. + + Args: + rois (Tensor): Boxes to be transformed. Has shape (N, 4) or (B, N, 4) + deltas (Tensor): Encoded offsets with respect to each roi. + Has shape (B, N, num_classes * 4) or (B, N, 4) or + (N, num_classes * 4) or (N, 4). Note N = num_anchors * W * H + when rois is a grid of anchors.Offset encoding follows [1]_. + means (Sequence[float]): Denormalizing means for delta coordinates + stds (Sequence[float]): Denormalizing standard deviation for delta + coordinates + max_shape (Sequence[int] or torch.Tensor or Sequence[ + Sequence[int]],optional): Maximum bounds for boxes, specifies + (H, W, C) or (H, W). If rois shape is (B, N, 4), then + the max_shape should be a Sequence[Sequence[int]] + and the length of max_shape should also be B. + wh_ratio_clip (float): Maximum aspect ratio for boxes. + clip_border (bool, optional): Whether clip the objects outside the + border of the image. Defaults to True. + + Returns: + Tensor: Boxes with shape (B, N, num_classes * 4) or (B, N, 4) or + (N, num_classes * 4) or (N, 4), where 4 represent + tl_x, tl_y, br_x, br_y. + + References: + .. [1] https://arxiv.org/abs/1311.2524 + + Example: + >>> rois = torch.Tensor([[ 0., 0., 1., 1.], + >>> [ 0., 0., 1., 1.], + >>> [ 0., 0., 1., 1.], + >>> [ 5., 5., 5., 5.]]) + >>> deltas = torch.Tensor([[ 0., 0., 0., 0.], + >>> [ 1., 1., 1., 1.], + >>> [ 0., 0., 2., -1.], + >>> [ 0.7, -1.9, -0.5, 0.3]]) + >>> delta2bbox(rois, deltas, max_shape=(32, 32, 3)) + tensor([[0.0000, 0.0000, 1.0000, 1.0000], + [0.1409, 0.1409, 2.8591, 2.8591], + [0.0000, 0.3161, 4.1945, 0.6839], + [5.0000, 5.0000, 5.0000, 5.0000]]) + """ + means = deltas.new_tensor(means).view(1, + -1).repeat(1, + deltas.size(-1) // 4) + stds = deltas.new_tensor(stds).view(1, -1).repeat(1, deltas.size(-1) // 4) + denorm_deltas = deltas * stds + means + dx = denorm_deltas[..., 0::4] + dy = denorm_deltas[..., 1::4] + dw = denorm_deltas[..., 2::4] + dh = denorm_deltas[..., 3::4] + max_ratio = np.abs(np.log(wh_ratio_clip)) + dw = dw.clamp(min=-max_ratio, max=max_ratio) + dh = dh.clamp(min=-max_ratio, max=max_ratio) + x1, y1 = rois[..., 0], rois[..., 1] + x2, y2 = rois[..., 2], rois[..., 3] + # Compute center of each roi + px = ((x1 + x2) * 0.5).unsqueeze(-1).expand_as(dx) + py = ((y1 + y2) * 0.5).unsqueeze(-1).expand_as(dy) + # Compute width/height of each roi + pw = (x2 - x1).unsqueeze(-1).expand_as(dw) + ph = (y2 - y1).unsqueeze(-1).expand_as(dh) + # Use exp(network energy) to enlarge/shrink each roi + gw = pw * dw.exp() + gh = ph * dh.exp() + # Use network energy to shift the center of each roi + gx = px + pw * dx + gy = py + ph * dy + # Convert center-xy/width/height to top-left, bottom-right + x1 = gx - gw * 0.5 + y1 = gy - gh * 0.5 + x2 = gx + gw * 0.5 + y2 = gy + gh * 0.5 + + bboxes = torch.stack([x1, y1, x2, y2], dim=-1).view(deltas.size()) + + if clip_border and max_shape is not None: + if not isinstance(max_shape, torch.Tensor): + max_shape = x1.new_tensor(max_shape) + max_shape = max_shape[..., :2].type_as(x1) + if max_shape.ndim == 2: + assert bboxes.ndim == 3 + assert max_shape.size(0) == bboxes.size(0) + + min_xy = x1.new_tensor(0) + max_xy = torch.cat( + [max_shape] * (deltas.size(-1) // 2), + dim=-1).flip(-1).unsqueeze(-2) + bboxes = torch.where(bboxes < min_xy, min_xy, bboxes) + bboxes = torch.where(bboxes > max_xy, max_xy, bboxes) + + return bboxes diff --git a/detection/mmdet/core/bbox/coder/legacy_delta_xywh_bbox_coder.py b/detection/mmdet/core/bbox/coder/legacy_delta_xywh_bbox_coder.py new file mode 100644 index 0000000..190309f --- /dev/null +++ b/detection/mmdet/core/bbox/coder/legacy_delta_xywh_bbox_coder.py @@ -0,0 +1,215 @@ +import mmcv +import numpy as np +import torch + +from ..builder import BBOX_CODERS +from .base_bbox_coder import BaseBBoxCoder + + +@BBOX_CODERS.register_module() +class LegacyDeltaXYWHBBoxCoder(BaseBBoxCoder): + """Legacy Delta XYWH BBox coder used in MMDet V1.x. + + Following the practice in R-CNN [1]_, this coder encodes bbox (x1, y1, x2, + y2) into delta (dx, dy, dw, dh) and decodes delta (dx, dy, dw, dh) + back to original bbox (x1, y1, x2, y2). + + Note: + The main difference between :class`LegacyDeltaXYWHBBoxCoder` and + :class:`DeltaXYWHBBoxCoder` is whether ``+ 1`` is used during width and + height calculation. We suggest to only use this coder when testing with + MMDet V1.x models. + + References: + .. [1] https://arxiv.org/abs/1311.2524 + + Args: + target_means (Sequence[float]): denormalizing means of target for + delta coordinates + target_stds (Sequence[float]): denormalizing standard deviation of + target for delta coordinates + """ + + def __init__(self, + target_means=(0., 0., 0., 0.), + target_stds=(1., 1., 1., 1.)): + super(BaseBBoxCoder, self).__init__() + self.means = target_means + self.stds = target_stds + + def encode(self, bboxes, gt_bboxes): + """Get box regression transformation deltas that can be used to + transform the ``bboxes`` into the ``gt_bboxes``. + + Args: + bboxes (torch.Tensor): source boxes, e.g., object proposals. + gt_bboxes (torch.Tensor): target of the transformation, e.g., + ground-truth boxes. + + Returns: + torch.Tensor: Box transformation deltas + """ + assert bboxes.size(0) == gt_bboxes.size(0) + assert bboxes.size(-1) == gt_bboxes.size(-1) == 4 + encoded_bboxes = legacy_bbox2delta(bboxes, gt_bboxes, self.means, + self.stds) + return encoded_bboxes + + def decode(self, + bboxes, + pred_bboxes, + max_shape=None, + wh_ratio_clip=16 / 1000): + """Apply transformation `pred_bboxes` to `boxes`. + + Args: + boxes (torch.Tensor): Basic boxes. + pred_bboxes (torch.Tensor): Encoded boxes with shape + max_shape (tuple[int], optional): Maximum shape of boxes. + Defaults to None. + wh_ratio_clip (float, optional): The allowed ratio between + width and height. + + Returns: + torch.Tensor: Decoded boxes. + """ + assert pred_bboxes.size(0) == bboxes.size(0) + decoded_bboxes = legacy_delta2bbox(bboxes, pred_bboxes, self.means, + self.stds, max_shape, wh_ratio_clip) + + return decoded_bboxes + + +@mmcv.jit(coderize=True) +def legacy_bbox2delta(proposals, + gt, + means=(0., 0., 0., 0.), + stds=(1., 1., 1., 1.)): + """Compute deltas of proposals w.r.t. gt in the MMDet V1.x manner. + + We usually compute the deltas of x, y, w, h of proposals w.r.t ground + truth bboxes to get regression target. + This is the inverse function of `delta2bbox()` + + Args: + proposals (Tensor): Boxes to be transformed, shape (N, ..., 4) + gt (Tensor): Gt bboxes to be used as base, shape (N, ..., 4) + means (Sequence[float]): Denormalizing means for delta coordinates + stds (Sequence[float]): Denormalizing standard deviation for delta + coordinates + + Returns: + Tensor: deltas with shape (N, 4), where columns represent dx, dy, + dw, dh. + """ + assert proposals.size() == gt.size() + + proposals = proposals.float() + gt = gt.float() + px = (proposals[..., 0] + proposals[..., 2]) * 0.5 + py = (proposals[..., 1] + proposals[..., 3]) * 0.5 + pw = proposals[..., 2] - proposals[..., 0] + 1.0 + ph = proposals[..., 3] - proposals[..., 1] + 1.0 + + gx = (gt[..., 0] + gt[..., 2]) * 0.5 + gy = (gt[..., 1] + gt[..., 3]) * 0.5 + gw = gt[..., 2] - gt[..., 0] + 1.0 + gh = gt[..., 3] - gt[..., 1] + 1.0 + + dx = (gx - px) / pw + dy = (gy - py) / ph + dw = torch.log(gw / pw) + dh = torch.log(gh / ph) + deltas = torch.stack([dx, dy, dw, dh], dim=-1) + + means = deltas.new_tensor(means).unsqueeze(0) + stds = deltas.new_tensor(stds).unsqueeze(0) + deltas = deltas.sub_(means).div_(stds) + + return deltas + + +@mmcv.jit(coderize=True) +def legacy_delta2bbox(rois, + deltas, + means=(0., 0., 0., 0.), + stds=(1., 1., 1., 1.), + max_shape=None, + wh_ratio_clip=16 / 1000): + """Apply deltas to shift/scale base boxes in the MMDet V1.x manner. + + Typically the rois are anchor or proposed bounding boxes and the deltas are + network outputs used to shift/scale those boxes. + This is the inverse function of `bbox2delta()` + + Args: + rois (Tensor): Boxes to be transformed. Has shape (N, 4) + deltas (Tensor): Encoded offsets with respect to each roi. + Has shape (N, 4 * num_classes). Note N = num_anchors * W * H when + rois is a grid of anchors. Offset encoding follows [1]_. + means (Sequence[float]): Denormalizing means for delta coordinates + stds (Sequence[float]): Denormalizing standard deviation for delta + coordinates + max_shape (tuple[int, int]): Maximum bounds for boxes. specifies (H, W) + wh_ratio_clip (float): Maximum aspect ratio for boxes. + + Returns: + Tensor: Boxes with shape (N, 4), where columns represent + tl_x, tl_y, br_x, br_y. + + References: + .. [1] https://arxiv.org/abs/1311.2524 + + Example: + >>> rois = torch.Tensor([[ 0., 0., 1., 1.], + >>> [ 0., 0., 1., 1.], + >>> [ 0., 0., 1., 1.], + >>> [ 5., 5., 5., 5.]]) + >>> deltas = torch.Tensor([[ 0., 0., 0., 0.], + >>> [ 1., 1., 1., 1.], + >>> [ 0., 0., 2., -1.], + >>> [ 0.7, -1.9, -0.5, 0.3]]) + >>> legacy_delta2bbox(rois, deltas, max_shape=(32, 32)) + tensor([[0.0000, 0.0000, 1.5000, 1.5000], + [0.0000, 0.0000, 5.2183, 5.2183], + [0.0000, 0.1321, 7.8891, 0.8679], + [5.3967, 2.4251, 6.0033, 3.7749]]) + """ + means = deltas.new_tensor(means).repeat(1, deltas.size(1) // 4) + stds = deltas.new_tensor(stds).repeat(1, deltas.size(1) // 4) + denorm_deltas = deltas * stds + means + dx = denorm_deltas[:, 0::4] + dy = denorm_deltas[:, 1::4] + dw = denorm_deltas[:, 2::4] + dh = denorm_deltas[:, 3::4] + max_ratio = np.abs(np.log(wh_ratio_clip)) + dw = dw.clamp(min=-max_ratio, max=max_ratio) + dh = dh.clamp(min=-max_ratio, max=max_ratio) + # Compute center of each roi + px = ((rois[:, 0] + rois[:, 2]) * 0.5).unsqueeze(1).expand_as(dx) + py = ((rois[:, 1] + rois[:, 3]) * 0.5).unsqueeze(1).expand_as(dy) + # Compute width/height of each roi + pw = (rois[:, 2] - rois[:, 0] + 1.0).unsqueeze(1).expand_as(dw) + ph = (rois[:, 3] - rois[:, 1] + 1.0).unsqueeze(1).expand_as(dh) + # Use exp(network energy) to enlarge/shrink each roi + gw = pw * dw.exp() + gh = ph * dh.exp() + # Use network energy to shift the center of each roi + gx = px + pw * dx + gy = py + ph * dy + # Convert center-xy/width/height to top-left, bottom-right + + # The true legacy box coder should +- 0.5 here. + # However, current implementation improves the performance when testing + # the models trained in MMDetection 1.X (~0.5 bbox AP, 0.2 mask AP) + x1 = gx - gw * 0.5 + y1 = gy - gh * 0.5 + x2 = gx + gw * 0.5 + y2 = gy + gh * 0.5 + if max_shape is not None: + x1 = x1.clamp(min=0, max=max_shape[1] - 1) + y1 = y1.clamp(min=0, max=max_shape[0] - 1) + x2 = x2.clamp(min=0, max=max_shape[1] - 1) + y2 = y2.clamp(min=0, max=max_shape[0] - 1) + bboxes = torch.stack([x1, y1, x2, y2], dim=-1).view_as(deltas) + return bboxes diff --git a/detection/mmdet/core/bbox/coder/pseudo_bbox_coder.py b/detection/mmdet/core/bbox/coder/pseudo_bbox_coder.py new file mode 100644 index 0000000..1c8346f --- /dev/null +++ b/detection/mmdet/core/bbox/coder/pseudo_bbox_coder.py @@ -0,0 +1,18 @@ +from ..builder import BBOX_CODERS +from .base_bbox_coder import BaseBBoxCoder + + +@BBOX_CODERS.register_module() +class PseudoBBoxCoder(BaseBBoxCoder): + """Pseudo bounding box coder.""" + + def __init__(self, **kwargs): + super(BaseBBoxCoder, self).__init__(**kwargs) + + def encode(self, bboxes, gt_bboxes): + """torch.Tensor: return the given ``bboxes``""" + return gt_bboxes + + def decode(self, bboxes, pred_bboxes): + """torch.Tensor: return the given ``pred_bboxes``""" + return pred_bboxes diff --git a/detection/mmdet/core/bbox/coder/tblr_bbox_coder.py b/detection/mmdet/core/bbox/coder/tblr_bbox_coder.py new file mode 100644 index 0000000..edaffaf --- /dev/null +++ b/detection/mmdet/core/bbox/coder/tblr_bbox_coder.py @@ -0,0 +1,198 @@ +import mmcv +import torch + +from ..builder import BBOX_CODERS +from .base_bbox_coder import BaseBBoxCoder + + +@BBOX_CODERS.register_module() +class TBLRBBoxCoder(BaseBBoxCoder): + """TBLR BBox coder. + + Following the practice in `FSAF `_, + this coder encodes gt bboxes (x1, y1, x2, y2) into (top, bottom, left, + right) and decode it back to the original. + + Args: + normalizer (list | float): Normalization factor to be + divided with when coding the coordinates. If it is a list, it should + have length of 4 indicating normalization factor in tblr dims. + Otherwise it is a unified float factor for all dims. Default: 4.0 + clip_border (bool, optional): Whether clip the objects outside the + border of the image. Defaults to True. + """ + + def __init__(self, normalizer=4.0, clip_border=True): + super(BaseBBoxCoder, self).__init__() + self.normalizer = normalizer + self.clip_border = clip_border + + def encode(self, bboxes, gt_bboxes): + """Get box regression transformation deltas that can be used to + transform the ``bboxes`` into the ``gt_bboxes`` in the (top, left, + bottom, right) order. + + Args: + bboxes (torch.Tensor): source boxes, e.g., object proposals. + gt_bboxes (torch.Tensor): target of the transformation, e.g., + ground truth boxes. + + Returns: + torch.Tensor: Box transformation deltas + """ + assert bboxes.size(0) == gt_bboxes.size(0) + assert bboxes.size(-1) == gt_bboxes.size(-1) == 4 + encoded_bboxes = bboxes2tblr( + bboxes, gt_bboxes, normalizer=self.normalizer) + return encoded_bboxes + + def decode(self, bboxes, pred_bboxes, max_shape=None): + """Apply transformation `pred_bboxes` to `boxes`. + + Args: + bboxes (torch.Tensor): Basic boxes.Shape (B, N, 4) or (N, 4) + pred_bboxes (torch.Tensor): Encoded boxes with shape + (B, N, 4) or (N, 4) + max_shape (Sequence[int] or torch.Tensor or Sequence[ + Sequence[int]],optional): Maximum bounds for boxes, specifies + (H, W, C) or (H, W). If bboxes shape is (B, N, 4), then + the max_shape should be a Sequence[Sequence[int]] + and the length of max_shape should also be B. + + Returns: + torch.Tensor: Decoded boxes. + """ + decoded_bboxes = tblr2bboxes( + bboxes, + pred_bboxes, + normalizer=self.normalizer, + max_shape=max_shape, + clip_border=self.clip_border) + + return decoded_bboxes + + +@mmcv.jit(coderize=True) +def bboxes2tblr(priors, gts, normalizer=4.0, normalize_by_wh=True): + """Encode ground truth boxes to tblr coordinate. + + It first convert the gt coordinate to tblr format, + (top, bottom, left, right), relative to prior box centers. + The tblr coordinate may be normalized by the side length of prior bboxes + if `normalize_by_wh` is specified as True, and it is then normalized by + the `normalizer` factor. + + Args: + priors (Tensor): Prior boxes in point form + Shape: (num_proposals,4). + gts (Tensor): Coords of ground truth for each prior in point-form + Shape: (num_proposals, 4). + normalizer (Sequence[float] | float): normalization parameter of + encoded boxes. If it is a list, it has to have length = 4. + Default: 4.0 + normalize_by_wh (bool): Whether to normalize tblr coordinate by the + side length (wh) of prior bboxes. + + Return: + encoded boxes (Tensor), Shape: (num_proposals, 4) + """ + + # dist b/t match center and prior's center + if not isinstance(normalizer, float): + normalizer = torch.tensor(normalizer, device=priors.device) + assert len(normalizer) == 4, 'Normalizer must have length = 4' + assert priors.size(0) == gts.size(0) + prior_centers = (priors[:, 0:2] + priors[:, 2:4]) / 2 + xmin, ymin, xmax, ymax = gts.split(1, dim=1) + top = prior_centers[:, 1].unsqueeze(1) - ymin + bottom = ymax - prior_centers[:, 1].unsqueeze(1) + left = prior_centers[:, 0].unsqueeze(1) - xmin + right = xmax - prior_centers[:, 0].unsqueeze(1) + loc = torch.cat((top, bottom, left, right), dim=1) + if normalize_by_wh: + # Normalize tblr by anchor width and height + wh = priors[:, 2:4] - priors[:, 0:2] + w, h = torch.split(wh, 1, dim=1) + loc[:, :2] /= h # tb is normalized by h + loc[:, 2:] /= w # lr is normalized by w + # Normalize tblr by the given normalization factor + return loc / normalizer + + +@mmcv.jit(coderize=True) +def tblr2bboxes(priors, + tblr, + normalizer=4.0, + normalize_by_wh=True, + max_shape=None, + clip_border=True): + """Decode tblr outputs to prediction boxes. + + The process includes 3 steps: 1) De-normalize tblr coordinates by + multiplying it with `normalizer`; 2) De-normalize tblr coordinates by the + prior bbox width and height if `normalize_by_wh` is `True`; 3) Convert + tblr (top, bottom, left, right) pair relative to the center of priors back + to (xmin, ymin, xmax, ymax) coordinate. + + Args: + priors (Tensor): Prior boxes in point form (x0, y0, x1, y1) + Shape: (N,4) or (B, N, 4). + tblr (Tensor): Coords of network output in tblr form + Shape: (N, 4) or (B, N, 4). + normalizer (Sequence[float] | float): Normalization parameter of + encoded boxes. By list, it represents the normalization factors at + tblr dims. By float, it is the unified normalization factor at all + dims. Default: 4.0 + normalize_by_wh (bool): Whether the tblr coordinates have been + normalized by the side length (wh) of prior bboxes. + max_shape (Sequence[int] or torch.Tensor or Sequence[ + Sequence[int]],optional): Maximum bounds for boxes, specifies + (H, W, C) or (H, W). If priors shape is (B, N, 4), then + the max_shape should be a Sequence[Sequence[int]] + and the length of max_shape should also be B. + clip_border (bool, optional): Whether clip the objects outside the + border of the image. Defaults to True. + + Return: + encoded boxes (Tensor): Boxes with shape (N, 4) or (B, N, 4) + """ + if not isinstance(normalizer, float): + normalizer = torch.tensor(normalizer, device=priors.device) + assert len(normalizer) == 4, 'Normalizer must have length = 4' + assert priors.size(0) == tblr.size(0) + if priors.ndim == 3: + assert priors.size(1) == tblr.size(1) + + loc_decode = tblr * normalizer + prior_centers = (priors[..., 0:2] + priors[..., 2:4]) / 2 + if normalize_by_wh: + wh = priors[..., 2:4] - priors[..., 0:2] + w, h = torch.split(wh, 1, dim=-1) + # Inplace operation with slice would failed for exporting to ONNX + th = h * loc_decode[..., :2] # tb + tw = w * loc_decode[..., 2:] # lr + loc_decode = torch.cat([th, tw], dim=-1) + # Cannot be exported using onnx when loc_decode.split(1, dim=-1) + top, bottom, left, right = loc_decode.split((1, 1, 1, 1), dim=-1) + xmin = prior_centers[..., 0].unsqueeze(-1) - left + xmax = prior_centers[..., 0].unsqueeze(-1) + right + ymin = prior_centers[..., 1].unsqueeze(-1) - top + ymax = prior_centers[..., 1].unsqueeze(-1) + bottom + + bboxes = torch.cat((xmin, ymin, xmax, ymax), dim=-1) + + if clip_border and max_shape is not None: + if not isinstance(max_shape, torch.Tensor): + max_shape = priors.new_tensor(max_shape) + max_shape = max_shape[..., :2].type_as(priors) + if max_shape.ndim == 2: + assert bboxes.ndim == 3 + assert max_shape.size(0) == bboxes.size(0) + + min_xy = priors.new_tensor(0) + max_xy = torch.cat([max_shape, max_shape], + dim=-1).flip(-1).unsqueeze(-2) + bboxes = torch.where(bboxes < min_xy, min_xy, bboxes) + bboxes = torch.where(bboxes > max_xy, max_xy, bboxes) + + return bboxes diff --git a/detection/mmdet/core/bbox/coder/yolo_bbox_coder.py b/detection/mmdet/core/bbox/coder/yolo_bbox_coder.py new file mode 100644 index 0000000..d6d0e82 --- /dev/null +++ b/detection/mmdet/core/bbox/coder/yolo_bbox_coder.py @@ -0,0 +1,89 @@ +import mmcv +import torch + +from ..builder import BBOX_CODERS +from .base_bbox_coder import BaseBBoxCoder + + +@BBOX_CODERS.register_module() +class YOLOBBoxCoder(BaseBBoxCoder): + """YOLO BBox coder. + + Following `YOLO `_, this coder divide + image into grids, and encode bbox (x1, y1, x2, y2) into (cx, cy, dw, dh). + cx, cy in [0., 1.], denotes relative center position w.r.t the center of + bboxes. dw, dh are the same as :obj:`DeltaXYWHBBoxCoder`. + + Args: + eps (float): Min value of cx, cy when encoding. + """ + + def __init__(self, eps=1e-6): + super(BaseBBoxCoder, self).__init__() + self.eps = eps + + @mmcv.jit(coderize=True) + def encode(self, bboxes, gt_bboxes, stride): + """Get box regression transformation deltas that can be used to + transform the ``bboxes`` into the ``gt_bboxes``. + + Args: + bboxes (torch.Tensor): Source boxes, e.g., anchors. + gt_bboxes (torch.Tensor): Target of the transformation, e.g., + ground-truth boxes. + stride (torch.Tensor | int): Stride of bboxes. + + Returns: + torch.Tensor: Box transformation deltas + """ + + assert bboxes.size(0) == gt_bboxes.size(0) + assert bboxes.size(-1) == gt_bboxes.size(-1) == 4 + x_center_gt = (gt_bboxes[..., 0] + gt_bboxes[..., 2]) * 0.5 + y_center_gt = (gt_bboxes[..., 1] + gt_bboxes[..., 3]) * 0.5 + w_gt = gt_bboxes[..., 2] - gt_bboxes[..., 0] + h_gt = gt_bboxes[..., 3] - gt_bboxes[..., 1] + x_center = (bboxes[..., 0] + bboxes[..., 2]) * 0.5 + y_center = (bboxes[..., 1] + bboxes[..., 3]) * 0.5 + w = bboxes[..., 2] - bboxes[..., 0] + h = bboxes[..., 3] - bboxes[..., 1] + w_target = torch.log((w_gt / w).clamp(min=self.eps)) + h_target = torch.log((h_gt / h).clamp(min=self.eps)) + x_center_target = ((x_center_gt - x_center) / stride + 0.5).clamp( + self.eps, 1 - self.eps) + y_center_target = ((y_center_gt - y_center) / stride + 0.5).clamp( + self.eps, 1 - self.eps) + encoded_bboxes = torch.stack( + [x_center_target, y_center_target, w_target, h_target], dim=-1) + return encoded_bboxes + + @mmcv.jit(coderize=True) + def decode(self, bboxes, pred_bboxes, stride): + """Apply transformation `pred_bboxes` to `boxes`. + + Args: + boxes (torch.Tensor): Basic boxes, e.g. anchors. + pred_bboxes (torch.Tensor): Encoded boxes with shape + stride (torch.Tensor | int): Strides of bboxes. + + Returns: + torch.Tensor: Decoded boxes. + """ + assert pred_bboxes.size(0) == bboxes.size(0) + assert pred_bboxes.size(-1) == bboxes.size(-1) == 4 + x_center = (bboxes[..., 0] + bboxes[..., 2]) * 0.5 + y_center = (bboxes[..., 1] + bboxes[..., 3]) * 0.5 + w = bboxes[..., 2] - bboxes[..., 0] + h = bboxes[..., 3] - bboxes[..., 1] + # Get outputs x, y + x_center_pred = (pred_bboxes[..., 0] - 0.5) * stride + x_center + y_center_pred = (pred_bboxes[..., 1] - 0.5) * stride + y_center + w_pred = torch.exp(pred_bboxes[..., 2]) * w + h_pred = torch.exp(pred_bboxes[..., 3]) * h + + decoded_bboxes = torch.stack( + (x_center_pred - w_pred / 2, y_center_pred - h_pred / 2, + x_center_pred + w_pred / 2, y_center_pred + h_pred / 2), + dim=-1) + + return decoded_bboxes diff --git a/detection/mmdet/core/bbox/demodata.py b/detection/mmdet/core/bbox/demodata.py new file mode 100644 index 0000000..feecb69 --- /dev/null +++ b/detection/mmdet/core/bbox/demodata.py @@ -0,0 +1,41 @@ +import numpy as np +import torch + +from mmdet.utils.util_random import ensure_rng + + +def random_boxes(num=1, scale=1, rng=None): + """Simple version of ``kwimage.Boxes.random`` + + Returns: + Tensor: shape (n, 4) in x1, y1, x2, y2 format. + + References: + https://gitlab.kitware.com/computer-vision/kwimage/blob/master/kwimage/structs/boxes.py#L1390 + + Example: + >>> num = 3 + >>> scale = 512 + >>> rng = 0 + >>> boxes = random_boxes(num, scale, rng) + >>> print(boxes) + tensor([[280.9925, 278.9802, 308.6148, 366.1769], + [216.9113, 330.6978, 224.0446, 456.5878], + [405.3632, 196.3221, 493.3953, 270.7942]]) + """ + rng = ensure_rng(rng) + + tlbr = rng.rand(num, 4).astype(np.float32) + + tl_x = np.minimum(tlbr[:, 0], tlbr[:, 2]) + tl_y = np.minimum(tlbr[:, 1], tlbr[:, 3]) + br_x = np.maximum(tlbr[:, 0], tlbr[:, 2]) + br_y = np.maximum(tlbr[:, 1], tlbr[:, 3]) + + tlbr[:, 0] = tl_x * scale + tlbr[:, 1] = tl_y * scale + tlbr[:, 2] = br_x * scale + tlbr[:, 3] = br_y * scale + + boxes = torch.from_numpy(tlbr) + return boxes diff --git a/detection/mmdet/core/bbox/iou_calculators/__init__.py b/detection/mmdet/core/bbox/iou_calculators/__init__.py new file mode 100644 index 0000000..e71369a --- /dev/null +++ b/detection/mmdet/core/bbox/iou_calculators/__init__.py @@ -0,0 +1,4 @@ +from .builder import build_iou_calculator +from .iou2d_calculator import BboxOverlaps2D, bbox_overlaps + +__all__ = ['build_iou_calculator', 'BboxOverlaps2D', 'bbox_overlaps'] diff --git a/detection/mmdet/core/bbox/iou_calculators/builder.py b/detection/mmdet/core/bbox/iou_calculators/builder.py new file mode 100644 index 0000000..09094d7 --- /dev/null +++ b/detection/mmdet/core/bbox/iou_calculators/builder.py @@ -0,0 +1,8 @@ +from mmcv.utils import Registry, build_from_cfg + +IOU_CALCULATORS = Registry('IoU calculator') + + +def build_iou_calculator(cfg, default_args=None): + """Builder of IoU calculator.""" + return build_from_cfg(cfg, IOU_CALCULATORS, default_args) diff --git a/detection/mmdet/core/bbox/iou_calculators/iou2d_calculator.py b/detection/mmdet/core/bbox/iou_calculators/iou2d_calculator.py new file mode 100644 index 0000000..158b702 --- /dev/null +++ b/detection/mmdet/core/bbox/iou_calculators/iou2d_calculator.py @@ -0,0 +1,159 @@ +import torch + +from .builder import IOU_CALCULATORS + + +@IOU_CALCULATORS.register_module() +class BboxOverlaps2D(object): + """2D Overlaps (e.g. IoUs, GIoUs) Calculator.""" + + def __call__(self, bboxes1, bboxes2, mode='iou', is_aligned=False): + """Calculate IoU between 2D bboxes. + + Args: + bboxes1 (Tensor): bboxes have shape (m, 4) in + format, or shape (m, 5) in format. + bboxes2 (Tensor): bboxes have shape (m, 4) in + format, shape (m, 5) in format, or be + empty. If ``is_aligned `` is ``True``, then m and n must be + equal. + mode (str): "iou" (intersection over union), "iof" (intersection + over foreground), or "giou" (generalized intersection over + union). + is_aligned (bool, optional): If True, then m and n must be equal. + Default False. + + Returns: + Tensor: shape (m, n) if ``is_aligned `` is False else shape (m,) + """ + assert bboxes1.size(-1) in [0, 4, 5] + assert bboxes2.size(-1) in [0, 4, 5] + if bboxes2.size(-1) == 5: + bboxes2 = bboxes2[..., :4] + if bboxes1.size(-1) == 5: + bboxes1 = bboxes1[..., :4] + return bbox_overlaps(bboxes1, bboxes2, mode, is_aligned) + + def __repr__(self): + """str: a string describing the module""" + repr_str = self.__class__.__name__ + '()' + return repr_str + + +def bbox_overlaps(bboxes1, bboxes2, mode='iou', is_aligned=False, eps=1e-6): + """Calculate overlap between two set of bboxes. + + If ``is_aligned `` is ``False``, then calculate the overlaps between each + bbox of bboxes1 and bboxes2, otherwise the overlaps between each aligned + pair of bboxes1 and bboxes2. + + Args: + bboxes1 (Tensor): shape (B, m, 4) in format or empty. + bboxes2 (Tensor): shape (B, n, 4) in format or empty. + B indicates the batch dim, in shape (B1, B2, ..., Bn). + If ``is_aligned `` is ``True``, then m and n must be equal. + mode (str): "iou" (intersection over union), "iof" (intersection over + foreground) or "giou" (generalized intersection over union). + Default "iou". + is_aligned (bool, optional): If True, then m and n must be equal. + Default False. + eps (float, optional): A value added to the denominator for numerical + stability. Default 1e-6. + + Returns: + Tensor: shape (m, n) if ``is_aligned `` is False else shape (m,) + + Example: + >>> bboxes1 = torch.FloatTensor([ + >>> [0, 0, 10, 10], + >>> [10, 10, 20, 20], + >>> [32, 32, 38, 42], + >>> ]) + >>> bboxes2 = torch.FloatTensor([ + >>> [0, 0, 10, 20], + >>> [0, 10, 10, 19], + >>> [10, 10, 20, 20], + >>> ]) + >>> overlaps = bbox_overlaps(bboxes1, bboxes2) + >>> assert overlaps.shape == (3, 3) + >>> overlaps = bbox_overlaps(bboxes1, bboxes2, is_aligned=True) + >>> assert overlaps.shape == (3, ) + + Example: + >>> empty = torch.empty(0, 4) + >>> nonempty = torch.FloatTensor([[0, 0, 10, 9]]) + >>> assert tuple(bbox_overlaps(empty, nonempty).shape) == (0, 1) + >>> assert tuple(bbox_overlaps(nonempty, empty).shape) == (1, 0) + >>> assert tuple(bbox_overlaps(empty, empty).shape) == (0, 0) + """ + + assert mode in ['iou', 'iof', 'giou'], f'Unsupported mode {mode}' + # Either the boxes are empty or the length of boxes' last dimension is 4 + assert (bboxes1.size(-1) == 4 or bboxes1.size(0) == 0) + assert (bboxes2.size(-1) == 4 or bboxes2.size(0) == 0) + + # Batch dim must be the same + # Batch dim: (B1, B2, ... Bn) + assert bboxes1.shape[:-2] == bboxes2.shape[:-2] + batch_shape = bboxes1.shape[:-2] + + rows = bboxes1.size(-2) + cols = bboxes2.size(-2) + if is_aligned: + assert rows == cols + + if rows * cols == 0: + if is_aligned: + return bboxes1.new(batch_shape + (rows, )) + else: + return bboxes1.new(batch_shape + (rows, cols)) + + area1 = (bboxes1[..., 2] - bboxes1[..., 0]) * ( + bboxes1[..., 3] - bboxes1[..., 1]) + area2 = (bboxes2[..., 2] - bboxes2[..., 0]) * ( + bboxes2[..., 3] - bboxes2[..., 1]) + + if is_aligned: + lt = torch.max(bboxes1[..., :2], bboxes2[..., :2]) # [B, rows, 2] + rb = torch.min(bboxes1[..., 2:], bboxes2[..., 2:]) # [B, rows, 2] + + wh = (rb - lt).clamp(min=0) # [B, rows, 2] + overlap = wh[..., 0] * wh[..., 1] + + if mode in ['iou', 'giou']: + union = area1 + area2 - overlap + else: + union = area1 + if mode == 'giou': + enclosed_lt = torch.min(bboxes1[..., :2], bboxes2[..., :2]) + enclosed_rb = torch.max(bboxes1[..., 2:], bboxes2[..., 2:]) + else: + lt = torch.max(bboxes1[..., :, None, :2], + bboxes2[..., None, :, :2]) # [B, rows, cols, 2] + rb = torch.min(bboxes1[..., :, None, 2:], + bboxes2[..., None, :, 2:]) # [B, rows, cols, 2] + + wh = (rb - lt).clamp(min=0) # [B, rows, cols, 2] + overlap = wh[..., 0] * wh[..., 1] + + if mode in ['iou', 'giou']: + union = area1[..., None] + area2[..., None, :] - overlap + else: + union = area1[..., None] + if mode == 'giou': + enclosed_lt = torch.min(bboxes1[..., :, None, :2], + bboxes2[..., None, :, :2]) + enclosed_rb = torch.max(bboxes1[..., :, None, 2:], + bboxes2[..., None, :, 2:]) + + eps = union.new_tensor([eps]) + union = torch.max(union, eps) + ious = overlap / union + if mode in ['iou', 'iof']: + return ious + # calculate gious + enclose_wh = (enclosed_rb - enclosed_lt).clamp(min=0) + enclose_area = enclose_wh[..., 0] * enclose_wh[..., 1] + enclose_area = torch.max(enclose_area, eps) + gious = ious - (enclose_area - union) / enclose_area + return gious diff --git a/detection/mmdet/core/bbox/match_costs/__init__.py b/detection/mmdet/core/bbox/match_costs/__init__.py new file mode 100644 index 0000000..add5e0d --- /dev/null +++ b/detection/mmdet/core/bbox/match_costs/__init__.py @@ -0,0 +1,7 @@ +from .builder import build_match_cost +from .match_cost import BBoxL1Cost, ClassificationCost, FocalLossCost, IoUCost + +__all__ = [ + 'build_match_cost', 'ClassificationCost', 'BBoxL1Cost', 'IoUCost', + 'FocalLossCost' +] diff --git a/detection/mmdet/core/bbox/match_costs/builder.py b/detection/mmdet/core/bbox/match_costs/builder.py new file mode 100644 index 0000000..6894017 --- /dev/null +++ b/detection/mmdet/core/bbox/match_costs/builder.py @@ -0,0 +1,8 @@ +from mmcv.utils import Registry, build_from_cfg + +MATCH_COST = Registry('Match Cost') + + +def build_match_cost(cfg, default_args=None): + """Builder of IoU calculator.""" + return build_from_cfg(cfg, MATCH_COST, default_args) diff --git a/detection/mmdet/core/bbox/match_costs/match_cost.py b/detection/mmdet/core/bbox/match_costs/match_cost.py new file mode 100644 index 0000000..3886973 --- /dev/null +++ b/detection/mmdet/core/bbox/match_costs/match_cost.py @@ -0,0 +1,184 @@ +import torch + +from mmdet.core.bbox.iou_calculators import bbox_overlaps +from mmdet.core.bbox.transforms import bbox_cxcywh_to_xyxy, bbox_xyxy_to_cxcywh +from .builder import MATCH_COST + + +@MATCH_COST.register_module() +class BBoxL1Cost(object): + """BBoxL1Cost. + + Args: + weight (int | float, optional): loss_weight + box_format (str, optional): 'xyxy' for DETR, 'xywh' for Sparse_RCNN + + Examples: + >>> from mmdet.core.bbox.match_costs.match_cost import BBoxL1Cost + >>> import torch + >>> self = BBoxL1Cost() + >>> bbox_pred = torch.rand(1, 4) + >>> gt_bboxes= torch.FloatTensor([[0, 0, 2, 4], [1, 2, 3, 4]]) + >>> factor = torch.tensor([10, 8, 10, 8]) + >>> self(bbox_pred, gt_bboxes, factor) + tensor([[1.6172, 1.6422]]) + """ + + def __init__(self, weight=1., box_format='xyxy'): + self.weight = weight + assert box_format in ['xyxy', 'xywh'] + self.box_format = box_format + + def __call__(self, bbox_pred, gt_bboxes): + """ + Args: + bbox_pred (Tensor): Predicted boxes with normalized coordinates + (cx, cy, w, h), which are all in range [0, 1]. Shape + [num_query, 4]. + gt_bboxes (Tensor): Ground truth boxes with normalized + coordinates (x1, y1, x2, y2). Shape [num_gt, 4]. + + Returns: + torch.Tensor: bbox_cost value with weight + """ + if self.box_format == 'xywh': + gt_bboxes = bbox_xyxy_to_cxcywh(gt_bboxes) + elif self.box_format == 'xyxy': + bbox_pred = bbox_cxcywh_to_xyxy(bbox_pred) + bbox_cost = torch.cdist(bbox_pred, gt_bboxes, p=1) + return bbox_cost * self.weight + + +@MATCH_COST.register_module() +class FocalLossCost(object): + """FocalLossCost. + + Args: + weight (int | float, optional): loss_weight + alpha (int | float, optional): focal_loss alpha + gamma (int | float, optional): focal_loss gamma + eps (float, optional): default 1e-12 + + Examples: + >>> from mmdet.core.bbox.match_costs.match_cost import FocalLossCost + >>> import torch + >>> self = FocalLossCost() + >>> cls_pred = torch.rand(4, 3) + >>> gt_labels = torch.tensor([0, 1, 2]) + >>> factor = torch.tensor([10, 8, 10, 8]) + >>> self(cls_pred, gt_labels) + tensor([[-0.3236, -0.3364, -0.2699], + [-0.3439, -0.3209, -0.4807], + [-0.4099, -0.3795, -0.2929], + [-0.1950, -0.1207, -0.2626]]) + """ + + def __init__(self, weight=1., alpha=0.25, gamma=2, eps=1e-12): + self.weight = weight + self.alpha = alpha + self.gamma = gamma + self.eps = eps + + def __call__(self, cls_pred, gt_labels): + """ + Args: + cls_pred (Tensor): Predicted classification logits, shape + [num_query, num_class]. + gt_labels (Tensor): Label of `gt_bboxes`, shape (num_gt,). + + Returns: + torch.Tensor: cls_cost value with weight + """ + cls_pred = cls_pred.sigmoid() + neg_cost = -(1 - cls_pred + self.eps).log() * ( + 1 - self.alpha) * cls_pred.pow(self.gamma) + pos_cost = -(cls_pred + self.eps).log() * self.alpha * ( + 1 - cls_pred).pow(self.gamma) + cls_cost = pos_cost[:, gt_labels] - neg_cost[:, gt_labels] + return cls_cost * self.weight + + +@MATCH_COST.register_module() +class ClassificationCost(object): + """ClsSoftmaxCost. + + Args: + weight (int | float, optional): loss_weight + + Examples: + >>> from mmdet.core.bbox.match_costs.match_cost import \ + ... ClassificationCost + >>> import torch + >>> self = ClassificationCost() + >>> cls_pred = torch.rand(4, 3) + >>> gt_labels = torch.tensor([0, 1, 2]) + >>> factor = torch.tensor([10, 8, 10, 8]) + >>> self(cls_pred, gt_labels) + tensor([[-0.3430, -0.3525, -0.3045], + [-0.3077, -0.2931, -0.3992], + [-0.3664, -0.3455, -0.2881], + [-0.3343, -0.2701, -0.3956]]) + """ + + def __init__(self, weight=1.): + self.weight = weight + + def __call__(self, cls_pred, gt_labels): + """ + Args: + cls_pred (Tensor): Predicted classification logits, shape + [num_query, num_class]. + gt_labels (Tensor): Label of `gt_bboxes`, shape (num_gt,). + + Returns: + torch.Tensor: cls_cost value with weight + """ + # Following the official DETR repo, contrary to the loss that + # NLL is used, we approximate it in 1 - cls_score[gt_label]. + # The 1 is a constant that doesn't change the matching, + # so it can be omitted. + cls_score = cls_pred.softmax(-1) + cls_cost = -cls_score[:, gt_labels] + return cls_cost * self.weight + + +@MATCH_COST.register_module() +class IoUCost(object): + """IoUCost. + + Args: + iou_mode (str, optional): iou mode such as 'iou' | 'giou' + weight (int | float, optional): loss weight + + Examples: + >>> from mmdet.core.bbox.match_costs.match_cost import IoUCost + >>> import torch + >>> self = IoUCost() + >>> bboxes = torch.FloatTensor([[1,1, 2, 2], [2, 2, 3, 4]]) + >>> gt_bboxes = torch.FloatTensor([[0, 0, 2, 4], [1, 2, 3, 4]]) + >>> self(bboxes, gt_bboxes) + tensor([[-0.1250, 0.1667], + [ 0.1667, -0.5000]]) + """ + + def __init__(self, iou_mode='giou', weight=1.): + self.weight = weight + self.iou_mode = iou_mode + + def __call__(self, bboxes, gt_bboxes): + """ + Args: + bboxes (Tensor): Predicted boxes with unnormalized coordinates + (x1, y1, x2, y2). Shape [num_query, 4]. + gt_bboxes (Tensor): Ground truth boxes with unnormalized + coordinates (x1, y1, x2, y2). Shape [num_gt, 4]. + + Returns: + torch.Tensor: iou_cost value with weight + """ + # overlaps: [num_bboxes, num_gt] + overlaps = bbox_overlaps( + bboxes, gt_bboxes, mode=self.iou_mode, is_aligned=False) + # The 1 is a constant that doesn't change the matching, so omitted. + iou_cost = -overlaps + return iou_cost * self.weight diff --git a/detection/mmdet/core/bbox/samplers/__init__.py b/detection/mmdet/core/bbox/samplers/__init__.py new file mode 100644 index 0000000..0b06303 --- /dev/null +++ b/detection/mmdet/core/bbox/samplers/__init__.py @@ -0,0 +1,15 @@ +from .base_sampler import BaseSampler +from .combined_sampler import CombinedSampler +from .instance_balanced_pos_sampler import InstanceBalancedPosSampler +from .iou_balanced_neg_sampler import IoUBalancedNegSampler +from .ohem_sampler import OHEMSampler +from .pseudo_sampler import PseudoSampler +from .random_sampler import RandomSampler +from .sampling_result import SamplingResult +from .score_hlr_sampler import ScoreHLRSampler + +__all__ = [ + 'BaseSampler', 'PseudoSampler', 'RandomSampler', + 'InstanceBalancedPosSampler', 'IoUBalancedNegSampler', 'CombinedSampler', + 'OHEMSampler', 'SamplingResult', 'ScoreHLRSampler' +] diff --git a/detection/mmdet/core/bbox/samplers/base_sampler.py b/detection/mmdet/core/bbox/samplers/base_sampler.py new file mode 100644 index 0000000..9ea35de --- /dev/null +++ b/detection/mmdet/core/bbox/samplers/base_sampler.py @@ -0,0 +1,101 @@ +from abc import ABCMeta, abstractmethod + +import torch + +from .sampling_result import SamplingResult + + +class BaseSampler(metaclass=ABCMeta): + """Base class of samplers.""" + + def __init__(self, + num, + pos_fraction, + neg_pos_ub=-1, + add_gt_as_proposals=True, + **kwargs): + self.num = num + self.pos_fraction = pos_fraction + self.neg_pos_ub = neg_pos_ub + self.add_gt_as_proposals = add_gt_as_proposals + self.pos_sampler = self + self.neg_sampler = self + + @abstractmethod + def _sample_pos(self, assign_result, num_expected, **kwargs): + """Sample positive samples.""" + pass + + @abstractmethod + def _sample_neg(self, assign_result, num_expected, **kwargs): + """Sample negative samples.""" + pass + + def sample(self, + assign_result, + bboxes, + gt_bboxes, + gt_labels=None, + **kwargs): + """Sample positive and negative bboxes. + + This is a simple implementation of bbox sampling given candidates, + assigning results and ground truth bboxes. + + Args: + assign_result (:obj:`AssignResult`): Bbox assigning results. + bboxes (Tensor): Boxes to be sampled from. + gt_bboxes (Tensor): Ground truth bboxes. + gt_labels (Tensor, optional): Class labels of ground truth bboxes. + + Returns: + :obj:`SamplingResult`: Sampling result. + + Example: + >>> from mmdet.core.bbox import RandomSampler + >>> from mmdet.core.bbox import AssignResult + >>> from mmdet.core.bbox.demodata import ensure_rng, random_boxes + >>> rng = ensure_rng(None) + >>> assign_result = AssignResult.random(rng=rng) + >>> bboxes = random_boxes(assign_result.num_preds, rng=rng) + >>> gt_bboxes = random_boxes(assign_result.num_gts, rng=rng) + >>> gt_labels = None + >>> self = RandomSampler(num=32, pos_fraction=0.5, neg_pos_ub=-1, + >>> add_gt_as_proposals=False) + >>> self = self.sample(assign_result, bboxes, gt_bboxes, gt_labels) + """ + if len(bboxes.shape) < 2: + bboxes = bboxes[None, :] + + bboxes = bboxes[:, :4] + + gt_flags = bboxes.new_zeros((bboxes.shape[0], ), dtype=torch.uint8) + if self.add_gt_as_proposals and len(gt_bboxes) > 0: + if gt_labels is None: + raise ValueError( + 'gt_labels must be given when add_gt_as_proposals is True') + bboxes = torch.cat([gt_bboxes, bboxes], dim=0) + assign_result.add_gt_(gt_labels) + gt_ones = bboxes.new_ones(gt_bboxes.shape[0], dtype=torch.uint8) + gt_flags = torch.cat([gt_ones, gt_flags]) + + num_expected_pos = int(self.num * self.pos_fraction) + pos_inds = self.pos_sampler._sample_pos( + assign_result, num_expected_pos, bboxes=bboxes, **kwargs) + # We found that sampled indices have duplicated items occasionally. + # (may be a bug of PyTorch) + pos_inds = pos_inds.unique() + num_sampled_pos = pos_inds.numel() + num_expected_neg = self.num - num_sampled_pos + if self.neg_pos_ub >= 0: + _pos = max(1, num_sampled_pos) + neg_upper_bound = int(self.neg_pos_ub * _pos) + if num_expected_neg > neg_upper_bound: + num_expected_neg = neg_upper_bound + neg_inds = self.neg_sampler._sample_neg( + assign_result, num_expected_neg, bboxes=bboxes, **kwargs) + neg_inds = neg_inds.unique() + + sampling_result = SamplingResult(pos_inds, neg_inds, bboxes, gt_bboxes, + assign_result, gt_flags) + return sampling_result diff --git a/detection/mmdet/core/bbox/samplers/combined_sampler.py b/detection/mmdet/core/bbox/samplers/combined_sampler.py new file mode 100644 index 0000000..564729f --- /dev/null +++ b/detection/mmdet/core/bbox/samplers/combined_sampler.py @@ -0,0 +1,20 @@ +from ..builder import BBOX_SAMPLERS, build_sampler +from .base_sampler import BaseSampler + + +@BBOX_SAMPLERS.register_module() +class CombinedSampler(BaseSampler): + """A sampler that combines positive sampler and negative sampler.""" + + def __init__(self, pos_sampler, neg_sampler, **kwargs): + super(CombinedSampler, self).__init__(**kwargs) + self.pos_sampler = build_sampler(pos_sampler, **kwargs) + self.neg_sampler = build_sampler(neg_sampler, **kwargs) + + def _sample_pos(self, **kwargs): + """Sample positive samples.""" + raise NotImplementedError + + def _sample_neg(self, **kwargs): + """Sample negative samples.""" + raise NotImplementedError diff --git a/detection/mmdet/core/bbox/samplers/instance_balanced_pos_sampler.py b/detection/mmdet/core/bbox/samplers/instance_balanced_pos_sampler.py new file mode 100644 index 0000000..c735298 --- /dev/null +++ b/detection/mmdet/core/bbox/samplers/instance_balanced_pos_sampler.py @@ -0,0 +1,55 @@ +import numpy as np +import torch + +from ..builder import BBOX_SAMPLERS +from .random_sampler import RandomSampler + + +@BBOX_SAMPLERS.register_module() +class InstanceBalancedPosSampler(RandomSampler): + """Instance balanced sampler that samples equal number of positive samples + for each instance.""" + + def _sample_pos(self, assign_result, num_expected, **kwargs): + """Sample positive boxes. + + Args: + assign_result (:obj:`AssignResult`): The assigned results of boxes. + num_expected (int): The number of expected positive samples + + Returns: + Tensor or ndarray: sampled indices. + """ + pos_inds = torch.nonzero(assign_result.gt_inds > 0, as_tuple=False) + if pos_inds.numel() != 0: + pos_inds = pos_inds.squeeze(1) + if pos_inds.numel() <= num_expected: + return pos_inds + else: + unique_gt_inds = assign_result.gt_inds[pos_inds].unique() + num_gts = len(unique_gt_inds) + num_per_gt = int(round(num_expected / float(num_gts)) + 1) + sampled_inds = [] + for i in unique_gt_inds: + inds = torch.nonzero( + assign_result.gt_inds == i.item(), as_tuple=False) + if inds.numel() != 0: + inds = inds.squeeze(1) + else: + continue + if len(inds) > num_per_gt: + inds = self.random_choice(inds, num_per_gt) + sampled_inds.append(inds) + sampled_inds = torch.cat(sampled_inds) + if len(sampled_inds) < num_expected: + num_extra = num_expected - len(sampled_inds) + extra_inds = np.array( + list(set(pos_inds.cpu()) - set(sampled_inds.cpu()))) + if len(extra_inds) > num_extra: + extra_inds = self.random_choice(extra_inds, num_extra) + extra_inds = torch.from_numpy(extra_inds).to( + assign_result.gt_inds.device).long() + sampled_inds = torch.cat([sampled_inds, extra_inds]) + elif len(sampled_inds) > num_expected: + sampled_inds = self.random_choice(sampled_inds, num_expected) + return sampled_inds diff --git a/detection/mmdet/core/bbox/samplers/iou_balanced_neg_sampler.py b/detection/mmdet/core/bbox/samplers/iou_balanced_neg_sampler.py new file mode 100644 index 0000000..f275e43 --- /dev/null +++ b/detection/mmdet/core/bbox/samplers/iou_balanced_neg_sampler.py @@ -0,0 +1,157 @@ +import numpy as np +import torch + +from ..builder import BBOX_SAMPLERS +from .random_sampler import RandomSampler + + +@BBOX_SAMPLERS.register_module() +class IoUBalancedNegSampler(RandomSampler): + """IoU Balanced Sampling. + + arXiv: https://arxiv.org/pdf/1904.02701.pdf (CVPR 2019) + + Sampling proposals according to their IoU. `floor_fraction` of needed RoIs + are sampled from proposals whose IoU are lower than `floor_thr` randomly. + The others are sampled from proposals whose IoU are higher than + `floor_thr`. These proposals are sampled from some bins evenly, which are + split by `num_bins` via IoU evenly. + + Args: + num (int): number of proposals. + pos_fraction (float): fraction of positive proposals. + floor_thr (float): threshold (minimum) IoU for IoU balanced sampling, + set to -1 if all using IoU balanced sampling. + floor_fraction (float): sampling fraction of proposals under floor_thr. + num_bins (int): number of bins in IoU balanced sampling. + """ + + def __init__(self, + num, + pos_fraction, + floor_thr=-1, + floor_fraction=0, + num_bins=3, + **kwargs): + super(IoUBalancedNegSampler, self).__init__(num, pos_fraction, + **kwargs) + assert floor_thr >= 0 or floor_thr == -1 + assert 0 <= floor_fraction <= 1 + assert num_bins >= 1 + + self.floor_thr = floor_thr + self.floor_fraction = floor_fraction + self.num_bins = num_bins + + def sample_via_interval(self, max_overlaps, full_set, num_expected): + """Sample according to the iou interval. + + Args: + max_overlaps (torch.Tensor): IoU between bounding boxes and ground + truth boxes. + full_set (set(int)): A full set of indices of boxes。 + num_expected (int): Number of expected samples。 + + Returns: + np.ndarray: Indices of samples + """ + max_iou = max_overlaps.max() + iou_interval = (max_iou - self.floor_thr) / self.num_bins + per_num_expected = int(num_expected / self.num_bins) + + sampled_inds = [] + for i in range(self.num_bins): + start_iou = self.floor_thr + i * iou_interval + end_iou = self.floor_thr + (i + 1) * iou_interval + tmp_set = set( + np.where( + np.logical_and(max_overlaps >= start_iou, + max_overlaps < end_iou))[0]) + tmp_inds = list(tmp_set & full_set) + if len(tmp_inds) > per_num_expected: + tmp_sampled_set = self.random_choice(tmp_inds, + per_num_expected) + else: + tmp_sampled_set = np.array(tmp_inds, dtype=np.int) + sampled_inds.append(tmp_sampled_set) + + sampled_inds = np.concatenate(sampled_inds) + if len(sampled_inds) < num_expected: + num_extra = num_expected - len(sampled_inds) + extra_inds = np.array(list(full_set - set(sampled_inds))) + if len(extra_inds) > num_extra: + extra_inds = self.random_choice(extra_inds, num_extra) + sampled_inds = np.concatenate([sampled_inds, extra_inds]) + + return sampled_inds + + def _sample_neg(self, assign_result, num_expected, **kwargs): + """Sample negative boxes. + + Args: + assign_result (:obj:`AssignResult`): The assigned results of boxes. + num_expected (int): The number of expected negative samples + + Returns: + Tensor or ndarray: sampled indices. + """ + neg_inds = torch.nonzero(assign_result.gt_inds == 0, as_tuple=False) + if neg_inds.numel() != 0: + neg_inds = neg_inds.squeeze(1) + if len(neg_inds) <= num_expected: + return neg_inds + else: + max_overlaps = assign_result.max_overlaps.cpu().numpy() + # balance sampling for negative samples + neg_set = set(neg_inds.cpu().numpy()) + + if self.floor_thr > 0: + floor_set = set( + np.where( + np.logical_and(max_overlaps >= 0, + max_overlaps < self.floor_thr))[0]) + iou_sampling_set = set( + np.where(max_overlaps >= self.floor_thr)[0]) + elif self.floor_thr == 0: + floor_set = set(np.where(max_overlaps == 0)[0]) + iou_sampling_set = set( + np.where(max_overlaps > self.floor_thr)[0]) + else: + floor_set = set() + iou_sampling_set = set( + np.where(max_overlaps > self.floor_thr)[0]) + # for sampling interval calculation + self.floor_thr = 0 + + floor_neg_inds = list(floor_set & neg_set) + iou_sampling_neg_inds = list(iou_sampling_set & neg_set) + num_expected_iou_sampling = int(num_expected * + (1 - self.floor_fraction)) + if len(iou_sampling_neg_inds) > num_expected_iou_sampling: + if self.num_bins >= 2: + iou_sampled_inds = self.sample_via_interval( + max_overlaps, set(iou_sampling_neg_inds), + num_expected_iou_sampling) + else: + iou_sampled_inds = self.random_choice( + iou_sampling_neg_inds, num_expected_iou_sampling) + else: + iou_sampled_inds = np.array( + iou_sampling_neg_inds, dtype=np.int) + num_expected_floor = num_expected - len(iou_sampled_inds) + if len(floor_neg_inds) > num_expected_floor: + sampled_floor_inds = self.random_choice( + floor_neg_inds, num_expected_floor) + else: + sampled_floor_inds = np.array(floor_neg_inds, dtype=np.int) + sampled_inds = np.concatenate( + (sampled_floor_inds, iou_sampled_inds)) + if len(sampled_inds) < num_expected: + num_extra = num_expected - len(sampled_inds) + extra_inds = np.array(list(neg_set - set(sampled_inds))) + if len(extra_inds) > num_extra: + extra_inds = self.random_choice(extra_inds, num_extra) + sampled_inds = np.concatenate((sampled_inds, extra_inds)) + sampled_inds = torch.from_numpy(sampled_inds).long().to( + assign_result.gt_inds.device) + return sampled_inds diff --git a/detection/mmdet/core/bbox/samplers/ohem_sampler.py b/detection/mmdet/core/bbox/samplers/ohem_sampler.py new file mode 100644 index 0000000..8b99f60 --- /dev/null +++ b/detection/mmdet/core/bbox/samplers/ohem_sampler.py @@ -0,0 +1,107 @@ +import torch + +from ..builder import BBOX_SAMPLERS +from ..transforms import bbox2roi +from .base_sampler import BaseSampler + + +@BBOX_SAMPLERS.register_module() +class OHEMSampler(BaseSampler): + r"""Online Hard Example Mining Sampler described in `Training Region-based + Object Detectors with Online Hard Example Mining + `_. + """ + + def __init__(self, + num, + pos_fraction, + context, + neg_pos_ub=-1, + add_gt_as_proposals=True, + **kwargs): + super(OHEMSampler, self).__init__(num, pos_fraction, neg_pos_ub, + add_gt_as_proposals) + self.context = context + if not hasattr(self.context, 'num_stages'): + self.bbox_head = self.context.bbox_head + else: + self.bbox_head = self.context.bbox_head[self.context.current_stage] + + def hard_mining(self, inds, num_expected, bboxes, labels, feats): + with torch.no_grad(): + rois = bbox2roi([bboxes]) + if not hasattr(self.context, 'num_stages'): + bbox_results = self.context._bbox_forward(feats, rois) + else: + bbox_results = self.context._bbox_forward( + self.context.current_stage, feats, rois) + cls_score = bbox_results['cls_score'] + loss = self.bbox_head.loss( + cls_score=cls_score, + bbox_pred=None, + rois=rois, + labels=labels, + label_weights=cls_score.new_ones(cls_score.size(0)), + bbox_targets=None, + bbox_weights=None, + reduction_override='none')['loss_cls'] + _, topk_loss_inds = loss.topk(num_expected) + return inds[topk_loss_inds] + + def _sample_pos(self, + assign_result, + num_expected, + bboxes=None, + feats=None, + **kwargs): + """Sample positive boxes. + + Args: + assign_result (:obj:`AssignResult`): Assigned results + num_expected (int): Number of expected positive samples + bboxes (torch.Tensor, optional): Boxes. Defaults to None. + feats (list[torch.Tensor], optional): Multi-level features. + Defaults to None. + + Returns: + torch.Tensor: Indices of positive samples + """ + # Sample some hard positive samples + pos_inds = torch.nonzero(assign_result.gt_inds > 0, as_tuple=False) + if pos_inds.numel() != 0: + pos_inds = pos_inds.squeeze(1) + if pos_inds.numel() <= num_expected: + return pos_inds + else: + return self.hard_mining(pos_inds, num_expected, bboxes[pos_inds], + assign_result.labels[pos_inds], feats) + + def _sample_neg(self, + assign_result, + num_expected, + bboxes=None, + feats=None, + **kwargs): + """Sample negative boxes. + + Args: + assign_result (:obj:`AssignResult`): Assigned results + num_expected (int): Number of expected negative samples + bboxes (torch.Tensor, optional): Boxes. Defaults to None. + feats (list[torch.Tensor], optional): Multi-level features. + Defaults to None. + + Returns: + torch.Tensor: Indices of negative samples + """ + # Sample some hard negative samples + neg_inds = torch.nonzero(assign_result.gt_inds == 0, as_tuple=False) + if neg_inds.numel() != 0: + neg_inds = neg_inds.squeeze(1) + if len(neg_inds) <= num_expected: + return neg_inds + else: + neg_labels = assign_result.labels.new_empty( + neg_inds.size(0)).fill_(self.bbox_head.num_classes) + return self.hard_mining(neg_inds, num_expected, bboxes[neg_inds], + neg_labels, feats) diff --git a/detection/mmdet/core/bbox/samplers/pseudo_sampler.py b/detection/mmdet/core/bbox/samplers/pseudo_sampler.py new file mode 100644 index 0000000..2bd81ab --- /dev/null +++ b/detection/mmdet/core/bbox/samplers/pseudo_sampler.py @@ -0,0 +1,41 @@ +import torch + +from ..builder import BBOX_SAMPLERS +from .base_sampler import BaseSampler +from .sampling_result import SamplingResult + + +@BBOX_SAMPLERS.register_module() +class PseudoSampler(BaseSampler): + """A pseudo sampler that does not do sampling actually.""" + + def __init__(self, **kwargs): + pass + + def _sample_pos(self, **kwargs): + """Sample positive samples.""" + raise NotImplementedError + + def _sample_neg(self, **kwargs): + """Sample negative samples.""" + raise NotImplementedError + + def sample(self, assign_result, bboxes, gt_bboxes, **kwargs): + """Directly returns the positive and negative indices of samples. + + Args: + assign_result (:obj:`AssignResult`): Assigned results + bboxes (torch.Tensor): Bounding boxes + gt_bboxes (torch.Tensor): Ground truth boxes + + Returns: + :obj:`SamplingResult`: sampler results + """ + pos_inds = torch.nonzero( + assign_result.gt_inds > 0, as_tuple=False).squeeze(-1).unique() + neg_inds = torch.nonzero( + assign_result.gt_inds == 0, as_tuple=False).squeeze(-1).unique() + gt_flags = bboxes.new_zeros(bboxes.shape[0], dtype=torch.uint8) + sampling_result = SamplingResult(pos_inds, neg_inds, bboxes, gt_bboxes, + assign_result, gt_flags) + return sampling_result diff --git a/detection/mmdet/core/bbox/samplers/random_sampler.py b/detection/mmdet/core/bbox/samplers/random_sampler.py new file mode 100644 index 0000000..0d2681b --- /dev/null +++ b/detection/mmdet/core/bbox/samplers/random_sampler.py @@ -0,0 +1,78 @@ +import torch + +from ..builder import BBOX_SAMPLERS +from .base_sampler import BaseSampler + + +@BBOX_SAMPLERS.register_module() +class RandomSampler(BaseSampler): + """Random sampler. + + Args: + num (int): Number of samples + pos_fraction (float): Fraction of positive samples + neg_pos_up (int, optional): Upper bound number of negative and + positive samples. Defaults to -1. + add_gt_as_proposals (bool, optional): Whether to add ground truth + boxes as proposals. Defaults to True. + """ + + def __init__(self, + num, + pos_fraction, + neg_pos_ub=-1, + add_gt_as_proposals=True, + **kwargs): + from mmdet.core.bbox import demodata + super(RandomSampler, self).__init__(num, pos_fraction, neg_pos_ub, + add_gt_as_proposals) + self.rng = demodata.ensure_rng(kwargs.get('rng', None)) + + def random_choice(self, gallery, num): + """Random select some elements from the gallery. + + If `gallery` is a Tensor, the returned indices will be a Tensor; + If `gallery` is a ndarray or list, the returned indices will be a + ndarray. + + Args: + gallery (Tensor | ndarray | list): indices pool. + num (int): expected sample num. + + Returns: + Tensor or ndarray: sampled indices. + """ + assert len(gallery) >= num + + is_tensor = isinstance(gallery, torch.Tensor) + if not is_tensor: + if torch.cuda.is_available(): + device = torch.cuda.current_device() + else: + device = 'cpu' + gallery = torch.tensor(gallery, dtype=torch.long, device=device) + perm = torch.randperm(gallery.numel())[:num].to(device=gallery.device) + rand_inds = gallery[perm] + if not is_tensor: + rand_inds = rand_inds.cpu().numpy() + return rand_inds + + def _sample_pos(self, assign_result, num_expected, **kwargs): + """Randomly sample some positive samples.""" + pos_inds = torch.nonzero(assign_result.gt_inds > 0, as_tuple=False) + if pos_inds.numel() != 0: + pos_inds = pos_inds.squeeze(1) + if pos_inds.numel() <= num_expected: + return pos_inds + else: + return self.random_choice(pos_inds, num_expected) + + def _sample_neg(self, assign_result, num_expected, **kwargs): + """Randomly sample some negative samples.""" + neg_inds = torch.nonzero(assign_result.gt_inds == 0, as_tuple=False) + if neg_inds.numel() != 0: + neg_inds = neg_inds.squeeze(1) + if len(neg_inds) <= num_expected: + return neg_inds + else: + return self.random_choice(neg_inds, num_expected) diff --git a/detection/mmdet/core/bbox/samplers/sampling_result.py b/detection/mmdet/core/bbox/samplers/sampling_result.py new file mode 100644 index 0000000..419a8e3 --- /dev/null +++ b/detection/mmdet/core/bbox/samplers/sampling_result.py @@ -0,0 +1,152 @@ +import torch + +from mmdet.utils import util_mixins + + +class SamplingResult(util_mixins.NiceRepr): + """Bbox sampling result. + + Example: + >>> # xdoctest: +IGNORE_WANT + >>> from mmdet.core.bbox.samplers.sampling_result import * # NOQA + >>> self = SamplingResult.random(rng=10) + >>> print(f'self = {self}') + self = + """ + + def __init__(self, pos_inds, neg_inds, bboxes, gt_bboxes, assign_result, + gt_flags): + self.pos_inds = pos_inds + self.neg_inds = neg_inds + self.pos_bboxes = bboxes[pos_inds] + self.neg_bboxes = bboxes[neg_inds] + self.pos_is_gt = gt_flags[pos_inds] + + self.num_gts = gt_bboxes.shape[0] + self.pos_assigned_gt_inds = assign_result.gt_inds[pos_inds] - 1 + + if gt_bboxes.numel() == 0: + # hack for index error case + assert self.pos_assigned_gt_inds.numel() == 0 + self.pos_gt_bboxes = torch.empty_like(gt_bboxes).view(-1, 4) + else: + if len(gt_bboxes.shape) < 2: + gt_bboxes = gt_bboxes.view(-1, 4) + + self.pos_gt_bboxes = gt_bboxes[self.pos_assigned_gt_inds, :] + + if assign_result.labels is not None: + self.pos_gt_labels = assign_result.labels[pos_inds] + else: + self.pos_gt_labels = None + + @property + def bboxes(self): + """torch.Tensor: concatenated positive and negative boxes""" + return torch.cat([self.pos_bboxes, self.neg_bboxes]) + + def to(self, device): + """Change the device of the data inplace. + + Example: + >>> self = SamplingResult.random() + >>> print(f'self = {self.to(None)}') + >>> # xdoctest: +REQUIRES(--gpu) + >>> print(f'self = {self.to(0)}') + """ + _dict = self.__dict__ + for key, value in _dict.items(): + if isinstance(value, torch.Tensor): + _dict[key] = value.to(device) + return self + + def __nice__(self): + data = self.info.copy() + data['pos_bboxes'] = data.pop('pos_bboxes').shape + data['neg_bboxes'] = data.pop('neg_bboxes').shape + parts = [f"'{k}': {v!r}" for k, v in sorted(data.items())] + body = ' ' + ',\n '.join(parts) + return '{\n' + body + '\n}' + + @property + def info(self): + """Returns a dictionary of info about the object.""" + return { + 'pos_inds': self.pos_inds, + 'neg_inds': self.neg_inds, + 'pos_bboxes': self.pos_bboxes, + 'neg_bboxes': self.neg_bboxes, + 'pos_is_gt': self.pos_is_gt, + 'num_gts': self.num_gts, + 'pos_assigned_gt_inds': self.pos_assigned_gt_inds, + } + + @classmethod + def random(cls, rng=None, **kwargs): + """ + Args: + rng (None | int | numpy.random.RandomState): seed or state. + kwargs (keyword arguments): + - num_preds: number of predicted boxes + - num_gts: number of true boxes + - p_ignore (float): probability of a predicted box assinged to \ + an ignored truth. + - p_assigned (float): probability of a predicted box not being \ + assigned. + - p_use_label (float | bool): with labels or not. + + Returns: + :obj:`SamplingResult`: Randomly generated sampling result. + + Example: + >>> from mmdet.core.bbox.samplers.sampling_result import * # NOQA + >>> self = SamplingResult.random() + >>> print(self.__dict__) + """ + from mmdet.core.bbox.samplers.random_sampler import RandomSampler + from mmdet.core.bbox.assigners.assign_result import AssignResult + from mmdet.core.bbox import demodata + rng = demodata.ensure_rng(rng) + + # make probabalistic? + num = 32 + pos_fraction = 0.5 + neg_pos_ub = -1 + + assign_result = AssignResult.random(rng=rng, **kwargs) + + # Note we could just compute an assignment + bboxes = demodata.random_boxes(assign_result.num_preds, rng=rng) + gt_bboxes = demodata.random_boxes(assign_result.num_gts, rng=rng) + + if rng.rand() > 0.2: + # sometimes algorithms squeeze their data, be robust to that + gt_bboxes = gt_bboxes.squeeze() + bboxes = bboxes.squeeze() + + if assign_result.labels is None: + gt_labels = None + else: + gt_labels = None # todo + + if gt_labels is None: + add_gt_as_proposals = False + else: + add_gt_as_proposals = True # make probabalistic? + + sampler = RandomSampler( + num, + pos_fraction, + neg_pos_ub=neg_pos_ub, + add_gt_as_proposals=add_gt_as_proposals, + rng=rng) + self = sampler.sample(assign_result, bboxes, gt_bboxes, gt_labels) + return self diff --git a/detection/mmdet/core/bbox/samplers/score_hlr_sampler.py b/detection/mmdet/core/bbox/samplers/score_hlr_sampler.py new file mode 100644 index 0000000..11d46b9 --- /dev/null +++ b/detection/mmdet/core/bbox/samplers/score_hlr_sampler.py @@ -0,0 +1,264 @@ +import torch +from mmcv.ops import nms_match + +from ..builder import BBOX_SAMPLERS +from ..transforms import bbox2roi +from .base_sampler import BaseSampler +from .sampling_result import SamplingResult + + +@BBOX_SAMPLERS.register_module() +class ScoreHLRSampler(BaseSampler): + r"""Importance-based Sample Reweighting (ISR_N), described in `Prime Sample + Attention in Object Detection `_. + + Score hierarchical local rank (HLR) differentiates with RandomSampler in + negative part. It firstly computes Score-HLR in a two-step way, + then linearly maps score hlr to the loss weights. + + Args: + num (int): Total number of sampled RoIs. + pos_fraction (float): Fraction of positive samples. + context (:class:`BaseRoIHead`): RoI head that the sampler belongs to. + neg_pos_ub (int): Upper bound of the ratio of num negative to num + positive, -1 means no upper bound. + add_gt_as_proposals (bool): Whether to add ground truth as proposals. + k (float): Power of the non-linear mapping. + bias (float): Shift of the non-linear mapping. + score_thr (float): Minimum score that a negative sample is to be + considered as valid bbox. + """ + + def __init__(self, + num, + pos_fraction, + context, + neg_pos_ub=-1, + add_gt_as_proposals=True, + k=0.5, + bias=0, + score_thr=0.05, + iou_thr=0.5, + **kwargs): + super().__init__(num, pos_fraction, neg_pos_ub, add_gt_as_proposals) + self.k = k + self.bias = bias + self.score_thr = score_thr + self.iou_thr = iou_thr + self.context = context + # context of cascade detectors is a list, so distinguish them here. + if not hasattr(context, 'num_stages'): + self.bbox_roi_extractor = context.bbox_roi_extractor + self.bbox_head = context.bbox_head + self.with_shared_head = context.with_shared_head + if self.with_shared_head: + self.shared_head = context.shared_head + else: + self.bbox_roi_extractor = context.bbox_roi_extractor[ + context.current_stage] + self.bbox_head = context.bbox_head[context.current_stage] + + @staticmethod + def random_choice(gallery, num): + """Randomly select some elements from the gallery. + + If `gallery` is a Tensor, the returned indices will be a Tensor; + If `gallery` is a ndarray or list, the returned indices will be a + ndarray. + + Args: + gallery (Tensor | ndarray | list): indices pool. + num (int): expected sample num. + + Returns: + Tensor or ndarray: sampled indices. + """ + assert len(gallery) >= num + + is_tensor = isinstance(gallery, torch.Tensor) + if not is_tensor: + if torch.cuda.is_available(): + device = torch.cuda.current_device() + else: + device = 'cpu' + gallery = torch.tensor(gallery, dtype=torch.long, device=device) + perm = torch.randperm(gallery.numel(), device=gallery.device)[:num] + rand_inds = gallery[perm] + if not is_tensor: + rand_inds = rand_inds.cpu().numpy() + return rand_inds + + def _sample_pos(self, assign_result, num_expected, **kwargs): + """Randomly sample some positive samples.""" + pos_inds = torch.nonzero(assign_result.gt_inds > 0).flatten() + if pos_inds.numel() <= num_expected: + return pos_inds + else: + return self.random_choice(pos_inds, num_expected) + + def _sample_neg(self, + assign_result, + num_expected, + bboxes, + feats=None, + img_meta=None, + **kwargs): + """Sample negative samples. + + Score-HLR sampler is done in the following steps: + 1. Take the maximum positive score prediction of each negative samples + as s_i. + 2. Filter out negative samples whose s_i <= score_thr, the left samples + are called valid samples. + 3. Use NMS-Match to divide valid samples into different groups, + samples in the same group will greatly overlap with each other + 4. Rank the matched samples in two-steps to get Score-HLR. + (1) In the same group, rank samples with their scores. + (2) In the same score rank across different groups, + rank samples with their scores again. + 5. Linearly map Score-HLR to the final label weights. + + Args: + assign_result (:obj:`AssignResult`): result of assigner. + num_expected (int): Expected number of samples. + bboxes (Tensor): bbox to be sampled. + feats (Tensor): Features come from FPN. + img_meta (dict): Meta information dictionary. + """ + neg_inds = torch.nonzero(assign_result.gt_inds == 0).flatten() + num_neg = neg_inds.size(0) + if num_neg == 0: + return neg_inds, None + with torch.no_grad(): + neg_bboxes = bboxes[neg_inds] + neg_rois = bbox2roi([neg_bboxes]) + bbox_result = self.context._bbox_forward(feats, neg_rois) + cls_score, bbox_pred = bbox_result['cls_score'], bbox_result[ + 'bbox_pred'] + + ori_loss = self.bbox_head.loss( + cls_score=cls_score, + bbox_pred=None, + rois=None, + labels=neg_inds.new_full((num_neg, ), + self.bbox_head.num_classes), + label_weights=cls_score.new_ones(num_neg), + bbox_targets=None, + bbox_weights=None, + reduction_override='none')['loss_cls'] + + # filter out samples with the max score lower than score_thr + max_score, argmax_score = cls_score.softmax(-1)[:, :-1].max(-1) + valid_inds = (max_score > self.score_thr).nonzero().view(-1) + invalid_inds = (max_score <= self.score_thr).nonzero().view(-1) + num_valid = valid_inds.size(0) + num_invalid = invalid_inds.size(0) + + num_expected = min(num_neg, num_expected) + num_hlr = min(num_valid, num_expected) + num_rand = num_expected - num_hlr + if num_valid > 0: + valid_rois = neg_rois[valid_inds] + valid_max_score = max_score[valid_inds] + valid_argmax_score = argmax_score[valid_inds] + valid_bbox_pred = bbox_pred[valid_inds] + + # valid_bbox_pred shape: [num_valid, #num_classes, 4] + valid_bbox_pred = valid_bbox_pred.view( + valid_bbox_pred.size(0), -1, 4) + selected_bbox_pred = valid_bbox_pred[range(num_valid), + valid_argmax_score] + pred_bboxes = self.bbox_head.bbox_coder.decode( + valid_rois[:, 1:], selected_bbox_pred) + pred_bboxes_with_score = torch.cat( + [pred_bboxes, valid_max_score[:, None]], -1) + group = nms_match(pred_bboxes_with_score, self.iou_thr) + + # imp: importance + imp = cls_score.new_zeros(num_valid) + for g in group: + g_score = valid_max_score[g] + # g_score has already sorted + rank = g_score.new_tensor(range(g_score.size(0))) + imp[g] = num_valid - rank + g_score + _, imp_rank_inds = imp.sort(descending=True) + _, imp_rank = imp_rank_inds.sort() + hlr_inds = imp_rank_inds[:num_expected] + + if num_rand > 0: + rand_inds = torch.randperm(num_invalid)[:num_rand] + select_inds = torch.cat( + [valid_inds[hlr_inds], invalid_inds[rand_inds]]) + else: + select_inds = valid_inds[hlr_inds] + + neg_label_weights = cls_score.new_ones(num_expected) + + up_bound = max(num_expected, num_valid) + imp_weights = (up_bound - + imp_rank[hlr_inds].float()) / up_bound + neg_label_weights[:num_hlr] = imp_weights + neg_label_weights[num_hlr:] = imp_weights.min() + neg_label_weights = (self.bias + + (1 - self.bias) * neg_label_weights).pow( + self.k) + ori_selected_loss = ori_loss[select_inds] + new_loss = ori_selected_loss * neg_label_weights + norm_ratio = ori_selected_loss.sum() / new_loss.sum() + neg_label_weights *= norm_ratio + else: + neg_label_weights = cls_score.new_ones(num_expected) + select_inds = torch.randperm(num_neg)[:num_expected] + + return neg_inds[select_inds], neg_label_weights + + def sample(self, + assign_result, + bboxes, + gt_bboxes, + gt_labels=None, + img_meta=None, + **kwargs): + """Sample positive and negative bboxes. + + This is a simple implementation of bbox sampling given candidates, + assigning results and ground truth bboxes. + + Args: + assign_result (:obj:`AssignResult`): Bbox assigning results. + bboxes (Tensor): Boxes to be sampled from. + gt_bboxes (Tensor): Ground truth bboxes. + gt_labels (Tensor, optional): Class labels of ground truth bboxes. + + Returns: + tuple[:obj:`SamplingResult`, Tensor]: Sampling result and negetive + label weights. + """ + bboxes = bboxes[:, :4] + + gt_flags = bboxes.new_zeros((bboxes.shape[0], ), dtype=torch.uint8) + if self.add_gt_as_proposals: + bboxes = torch.cat([gt_bboxes, bboxes], dim=0) + assign_result.add_gt_(gt_labels) + gt_ones = bboxes.new_ones(gt_bboxes.shape[0], dtype=torch.uint8) + gt_flags = torch.cat([gt_ones, gt_flags]) + + num_expected_pos = int(self.num * self.pos_fraction) + pos_inds = self.pos_sampler._sample_pos( + assign_result, num_expected_pos, bboxes=bboxes, **kwargs) + num_sampled_pos = pos_inds.numel() + num_expected_neg = self.num - num_sampled_pos + if self.neg_pos_ub >= 0: + _pos = max(1, num_sampled_pos) + neg_upper_bound = int(self.neg_pos_ub * _pos) + if num_expected_neg > neg_upper_bound: + num_expected_neg = neg_upper_bound + neg_inds, neg_label_weights = self.neg_sampler._sample_neg( + assign_result, + num_expected_neg, + bboxes, + img_meta=img_meta, + **kwargs) + + return SamplingResult(pos_inds, neg_inds, bboxes, gt_bboxes, + assign_result, gt_flags), neg_label_weights diff --git a/detection/mmdet/core/bbox/transforms.py b/detection/mmdet/core/bbox/transforms.py new file mode 100644 index 0000000..df55b0a --- /dev/null +++ b/detection/mmdet/core/bbox/transforms.py @@ -0,0 +1,240 @@ +import numpy as np +import torch + + +def bbox_flip(bboxes, img_shape, direction='horizontal'): + """Flip bboxes horizontally or vertically. + + Args: + bboxes (Tensor): Shape (..., 4*k) + img_shape (tuple): Image shape. + direction (str): Flip direction, options are "horizontal", "vertical", + "diagonal". Default: "horizontal" + + Returns: + Tensor: Flipped bboxes. + """ + assert bboxes.shape[-1] % 4 == 0 + assert direction in ['horizontal', 'vertical', 'diagonal'] + flipped = bboxes.clone() + if direction == 'horizontal': + flipped[..., 0::4] = img_shape[1] - bboxes[..., 2::4] + flipped[..., 2::4] = img_shape[1] - bboxes[..., 0::4] + elif direction == 'vertical': + flipped[..., 1::4] = img_shape[0] - bboxes[..., 3::4] + flipped[..., 3::4] = img_shape[0] - bboxes[..., 1::4] + else: + flipped[..., 0::4] = img_shape[1] - bboxes[..., 2::4] + flipped[..., 1::4] = img_shape[0] - bboxes[..., 3::4] + flipped[..., 2::4] = img_shape[1] - bboxes[..., 0::4] + flipped[..., 3::4] = img_shape[0] - bboxes[..., 1::4] + return flipped + + +def bbox_mapping(bboxes, + img_shape, + scale_factor, + flip, + flip_direction='horizontal'): + """Map bboxes from the original image scale to testing scale.""" + new_bboxes = bboxes * bboxes.new_tensor(scale_factor) + if flip: + new_bboxes = bbox_flip(new_bboxes, img_shape, flip_direction) + return new_bboxes + + +def bbox_mapping_back(bboxes, + img_shape, + scale_factor, + flip, + flip_direction='horizontal'): + """Map bboxes from testing scale to original image scale.""" + new_bboxes = bbox_flip(bboxes, img_shape, + flip_direction) if flip else bboxes + new_bboxes = new_bboxes.view(-1, 4) / new_bboxes.new_tensor(scale_factor) + return new_bboxes.view(bboxes.shape) + + +def bbox2roi(bbox_list): + """Convert a list of bboxes to roi format. + + Args: + bbox_list (list[Tensor]): a list of bboxes corresponding to a batch + of images. + + Returns: + Tensor: shape (n, 5), [batch_ind, x1, y1, x2, y2] + """ + rois_list = [] + for img_id, bboxes in enumerate(bbox_list): + if bboxes.size(0) > 0: + img_inds = bboxes.new_full((bboxes.size(0), 1), img_id) + rois = torch.cat([img_inds, bboxes[:, :4]], dim=-1) + else: + rois = bboxes.new_zeros((0, 5)) + rois_list.append(rois) + rois = torch.cat(rois_list, 0) + return rois + + +def roi2bbox(rois): + """Convert rois to bounding box format. + + Args: + rois (torch.Tensor): RoIs with the shape (n, 5) where the first + column indicates batch id of each RoI. + + Returns: + list[torch.Tensor]: Converted boxes of corresponding rois. + """ + bbox_list = [] + img_ids = torch.unique(rois[:, 0].cpu(), sorted=True) + for img_id in img_ids: + inds = (rois[:, 0] == img_id.item()) + bbox = rois[inds, 1:] + bbox_list.append(bbox) + return bbox_list + + +def bbox2result(bboxes, labels, num_classes): + """Convert detection results to a list of numpy arrays. + + Args: + bboxes (torch.Tensor | np.ndarray): shape (n, 5) + labels (torch.Tensor | np.ndarray): shape (n, ) + num_classes (int): class number, including background class + + Returns: + list(ndarray): bbox results of each class + """ + if bboxes.shape[0] == 0: + return [np.zeros((0, 5), dtype=np.float32) for i in range(num_classes)] + else: + if isinstance(bboxes, torch.Tensor): + bboxes = bboxes.detach().cpu().numpy() + labels = labels.detach().cpu().numpy() + return [bboxes[labels == i, :] for i in range(num_classes)] + + +def distance2bbox(points, distance, max_shape=None): + """Decode distance prediction to bounding box. + + Args: + points (Tensor): Shape (B, N, 2) or (N, 2). + distance (Tensor): Distance from the given point to 4 + boundaries (left, top, right, bottom). Shape (B, N, 4) or (N, 4) + max_shape (Sequence[int] or torch.Tensor or Sequence[ + Sequence[int]],optional): Maximum bounds for boxes, specifies + (H, W, C) or (H, W). If priors shape is (B, N, 4), then + the max_shape should be a Sequence[Sequence[int]] + and the length of max_shape should also be B. + + Returns: + Tensor: Boxes with shape (N, 4) or (B, N, 4) + """ + x1 = points[..., 0] - distance[..., 0] + y1 = points[..., 1] - distance[..., 1] + x2 = points[..., 0] + distance[..., 2] + y2 = points[..., 1] + distance[..., 3] + + bboxes = torch.stack([x1, y1, x2, y2], -1) + + if max_shape is not None: + if not isinstance(max_shape, torch.Tensor): + max_shape = x1.new_tensor(max_shape) + max_shape = max_shape[..., :2].type_as(x1) + if max_shape.ndim == 2: + assert bboxes.ndim == 3 + assert max_shape.size(0) == bboxes.size(0) + + min_xy = x1.new_tensor(0) + max_xy = torch.cat([max_shape, max_shape], + dim=-1).flip(-1).unsqueeze(-2) + bboxes = torch.where(bboxes < min_xy, min_xy, bboxes) + bboxes = torch.where(bboxes > max_xy, max_xy, bboxes) + + return bboxes + + +def bbox2distance(points, bbox, max_dis=None, eps=0.1): + """Decode bounding box based on distances. + + Args: + points (Tensor): Shape (n, 2), [x, y]. + bbox (Tensor): Shape (n, 4), "xyxy" format + max_dis (float): Upper bound of the distance. + eps (float): a small value to ensure target < max_dis, instead <= + + Returns: + Tensor: Decoded distances. + """ + left = points[:, 0] - bbox[:, 0] + top = points[:, 1] - bbox[:, 1] + right = bbox[:, 2] - points[:, 0] + bottom = bbox[:, 3] - points[:, 1] + if max_dis is not None: + left = left.clamp(min=0, max=max_dis - eps) + top = top.clamp(min=0, max=max_dis - eps) + right = right.clamp(min=0, max=max_dis - eps) + bottom = bottom.clamp(min=0, max=max_dis - eps) + return torch.stack([left, top, right, bottom], -1) + + +def bbox_rescale(bboxes, scale_factor=1.0): + """Rescale bounding box w.r.t. scale_factor. + + Args: + bboxes (Tensor): Shape (n, 4) for bboxes or (n, 5) for rois + scale_factor (float): rescale factor + + Returns: + Tensor: Rescaled bboxes. + """ + if bboxes.size(1) == 5: + bboxes_ = bboxes[:, 1:] + inds_ = bboxes[:, 0] + else: + bboxes_ = bboxes + cx = (bboxes_[:, 0] + bboxes_[:, 2]) * 0.5 + cy = (bboxes_[:, 1] + bboxes_[:, 3]) * 0.5 + w = bboxes_[:, 2] - bboxes_[:, 0] + h = bboxes_[:, 3] - bboxes_[:, 1] + w = w * scale_factor + h = h * scale_factor + x1 = cx - 0.5 * w + x2 = cx + 0.5 * w + y1 = cy - 0.5 * h + y2 = cy + 0.5 * h + if bboxes.size(1) == 5: + rescaled_bboxes = torch.stack([inds_, x1, y1, x2, y2], dim=-1) + else: + rescaled_bboxes = torch.stack([x1, y1, x2, y2], dim=-1) + return rescaled_bboxes + + +def bbox_cxcywh_to_xyxy(bbox): + """Convert bbox coordinates from (cx, cy, w, h) to (x1, y1, x2, y2). + + Args: + bbox (Tensor): Shape (n, 4) for bboxes. + + Returns: + Tensor: Converted bboxes. + """ + cx, cy, w, h = bbox.split((1, 1, 1, 1), dim=-1) + bbox_new = [(cx - 0.5 * w), (cy - 0.5 * h), (cx + 0.5 * w), (cy + 0.5 * h)] + return torch.cat(bbox_new, dim=-1) + + +def bbox_xyxy_to_cxcywh(bbox): + """Convert bbox coordinates from (x1, y1, x2, y2) to (cx, cy, w, h). + + Args: + bbox (Tensor): Shape (n, 4) for bboxes. + + Returns: + Tensor: Converted bboxes. + """ + x1, y1, x2, y2 = bbox.split((1, 1, 1, 1), dim=-1) + bbox_new = [(x1 + x2) / 2, (y1 + y2) / 2, (x2 - x1), (y2 - y1)] + return torch.cat(bbox_new, dim=-1) diff --git a/detection/mmdet/core/evaluation/__init__.py b/detection/mmdet/core/evaluation/__init__.py new file mode 100644 index 0000000..d11ef15 --- /dev/null +++ b/detection/mmdet/core/evaluation/__init__.py @@ -0,0 +1,15 @@ +from .class_names import (cityscapes_classes, coco_classes, dataset_aliases, + get_classes, imagenet_det_classes, + imagenet_vid_classes, voc_classes) +from .eval_hooks import DistEvalHook, EvalHook +from .mean_ap import average_precision, eval_map, print_map_summary +from .recall import (eval_recalls, plot_iou_recall, plot_num_recall, + print_recall_summary) + +__all__ = [ + 'voc_classes', 'imagenet_det_classes', 'imagenet_vid_classes', + 'coco_classes', 'cityscapes_classes', 'dataset_aliases', 'get_classes', + 'DistEvalHook', 'EvalHook', 'average_precision', 'eval_map', + 'print_map_summary', 'eval_recalls', 'print_recall_summary', + 'plot_num_recall', 'plot_iou_recall' +] diff --git a/detection/mmdet/core/evaluation/bbox_overlaps.py b/detection/mmdet/core/evaluation/bbox_overlaps.py new file mode 100644 index 0000000..93559ea --- /dev/null +++ b/detection/mmdet/core/evaluation/bbox_overlaps.py @@ -0,0 +1,48 @@ +import numpy as np + + +def bbox_overlaps(bboxes1, bboxes2, mode='iou', eps=1e-6): + """Calculate the ious between each bbox of bboxes1 and bboxes2. + + Args: + bboxes1(ndarray): shape (n, 4) + bboxes2(ndarray): shape (k, 4) + mode(str): iou (intersection over union) or iof (intersection + over foreground) + + Returns: + ious(ndarray): shape (n, k) + """ + + assert mode in ['iou', 'iof'] + + bboxes1 = bboxes1.astype(np.float32) + bboxes2 = bboxes2.astype(np.float32) + rows = bboxes1.shape[0] + cols = bboxes2.shape[0] + ious = np.zeros((rows, cols), dtype=np.float32) + if rows * cols == 0: + return ious + exchange = False + if bboxes1.shape[0] > bboxes2.shape[0]: + bboxes1, bboxes2 = bboxes2, bboxes1 + ious = np.zeros((cols, rows), dtype=np.float32) + exchange = True + area1 = (bboxes1[:, 2] - bboxes1[:, 0]) * (bboxes1[:, 3] - bboxes1[:, 1]) + area2 = (bboxes2[:, 2] - bboxes2[:, 0]) * (bboxes2[:, 3] - bboxes2[:, 1]) + for i in range(bboxes1.shape[0]): + x_start = np.maximum(bboxes1[i, 0], bboxes2[:, 0]) + y_start = np.maximum(bboxes1[i, 1], bboxes2[:, 1]) + x_end = np.minimum(bboxes1[i, 2], bboxes2[:, 2]) + y_end = np.minimum(bboxes1[i, 3], bboxes2[:, 3]) + overlap = np.maximum(x_end - x_start, 0) * np.maximum( + y_end - y_start, 0) + if mode == 'iou': + union = area1[i] + area2 - overlap + else: + union = area1[i] if not exchange else area2 + union = np.maximum(union, eps) + ious[i, :] = overlap / union + if exchange: + ious = ious.T + return ious diff --git a/detection/mmdet/core/evaluation/class_names.py b/detection/mmdet/core/evaluation/class_names.py new file mode 100644 index 0000000..c2487c2 --- /dev/null +++ b/detection/mmdet/core/evaluation/class_names.py @@ -0,0 +1,116 @@ +import mmcv + + +def wider_face_classes(): + return ['face'] + + +def voc_classes(): + return [ + 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', + 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person', + 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor' + ] + + +def imagenet_det_classes(): + return [ + 'accordion', 'airplane', 'ant', 'antelope', 'apple', 'armadillo', + 'artichoke', 'axe', 'baby_bed', 'backpack', 'bagel', 'balance_beam', + 'banana', 'band_aid', 'banjo', 'baseball', 'basketball', 'bathing_cap', + 'beaker', 'bear', 'bee', 'bell_pepper', 'bench', 'bicycle', 'binder', + 'bird', 'bookshelf', 'bow_tie', 'bow', 'bowl', 'brassiere', 'burrito', + 'bus', 'butterfly', 'camel', 'can_opener', 'car', 'cart', 'cattle', + 'cello', 'centipede', 'chain_saw', 'chair', 'chime', 'cocktail_shaker', + 'coffee_maker', 'computer_keyboard', 'computer_mouse', 'corkscrew', + 'cream', 'croquet_ball', 'crutch', 'cucumber', 'cup_or_mug', 'diaper', + 'digital_clock', 'dishwasher', 'dog', 'domestic_cat', 'dragonfly', + 'drum', 'dumbbell', 'electric_fan', 'elephant', 'face_powder', 'fig', + 'filing_cabinet', 'flower_pot', 'flute', 'fox', 'french_horn', 'frog', + 'frying_pan', 'giant_panda', 'goldfish', 'golf_ball', 'golfcart', + 'guacamole', 'guitar', 'hair_dryer', 'hair_spray', 'hamburger', + 'hammer', 'hamster', 'harmonica', 'harp', 'hat_with_a_wide_brim', + 'head_cabbage', 'helmet', 'hippopotamus', 'horizontal_bar', 'horse', + 'hotdog', 'iPod', 'isopod', 'jellyfish', 'koala_bear', 'ladle', + 'ladybug', 'lamp', 'laptop', 'lemon', 'lion', 'lipstick', 'lizard', + 'lobster', 'maillot', 'maraca', 'microphone', 'microwave', 'milk_can', + 'miniskirt', 'monkey', 'motorcycle', 'mushroom', 'nail', 'neck_brace', + 'oboe', 'orange', 'otter', 'pencil_box', 'pencil_sharpener', 'perfume', + 'person', 'piano', 'pineapple', 'ping-pong_ball', 'pitcher', 'pizza', + 'plastic_bag', 'plate_rack', 'pomegranate', 'popsicle', 'porcupine', + 'power_drill', 'pretzel', 'printer', 'puck', 'punching_bag', 'purse', + 'rabbit', 'racket', 'ray', 'red_panda', 'refrigerator', + 'remote_control', 'rubber_eraser', 'rugby_ball', 'ruler', + 'salt_or_pepper_shaker', 'saxophone', 'scorpion', 'screwdriver', + 'seal', 'sheep', 'ski', 'skunk', 'snail', 'snake', 'snowmobile', + 'snowplow', 'soap_dispenser', 'soccer_ball', 'sofa', 'spatula', + 'squirrel', 'starfish', 'stethoscope', 'stove', 'strainer', + 'strawberry', 'stretcher', 'sunglasses', 'swimming_trunks', 'swine', + 'syringe', 'table', 'tape_player', 'tennis_ball', 'tick', 'tie', + 'tiger', 'toaster', 'traffic_light', 'train', 'trombone', 'trumpet', + 'turtle', 'tv_or_monitor', 'unicycle', 'vacuum', 'violin', + 'volleyball', 'waffle_iron', 'washer', 'water_bottle', 'watercraft', + 'whale', 'wine_bottle', 'zebra' + ] + + +def imagenet_vid_classes(): + return [ + 'airplane', 'antelope', 'bear', 'bicycle', 'bird', 'bus', 'car', + 'cattle', 'dog', 'domestic_cat', 'elephant', 'fox', 'giant_panda', + 'hamster', 'horse', 'lion', 'lizard', 'monkey', 'motorcycle', 'rabbit', + 'red_panda', 'sheep', 'snake', 'squirrel', 'tiger', 'train', 'turtle', + 'watercraft', 'whale', 'zebra' + ] + + +def coco_classes(): + return [ + 'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', + 'truck', 'boat', 'traffic_light', 'fire_hydrant', 'stop_sign', + 'parking_meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', + 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', + 'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard', + 'sports_ball', 'kite', 'baseball_bat', 'baseball_glove', 'skateboard', + 'surfboard', 'tennis_racket', 'bottle', 'wine_glass', 'cup', 'fork', + 'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange', + 'broccoli', 'carrot', 'hot_dog', 'pizza', 'donut', 'cake', 'chair', + 'couch', 'potted_plant', 'bed', 'dining_table', 'toilet', 'tv', + 'laptop', 'mouse', 'remote', 'keyboard', 'cell_phone', 'microwave', + 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', + 'scissors', 'teddy_bear', 'hair_drier', 'toothbrush' + ] + + +def cityscapes_classes(): + return [ + 'person', 'rider', 'car', 'truck', 'bus', 'train', 'motorcycle', + 'bicycle' + ] + + +dataset_aliases = { + 'voc': ['voc', 'pascal_voc', 'voc07', 'voc12'], + 'imagenet_det': ['det', 'imagenet_det', 'ilsvrc_det'], + 'imagenet_vid': ['vid', 'imagenet_vid', 'ilsvrc_vid'], + 'coco': ['coco', 'mscoco', 'ms_coco'], + 'wider_face': ['WIDERFaceDataset', 'wider_face', 'WIDERFace'], + 'cityscapes': ['cityscapes'] +} + + +def get_classes(dataset): + """Get class names of a dataset.""" + alias2name = {} + for name, aliases in dataset_aliases.items(): + for alias in aliases: + alias2name[alias] = name + + if mmcv.is_str(dataset): + if dataset in alias2name: + labels = eval(alias2name[dataset] + '_classes()') + else: + raise ValueError(f'Unrecognized dataset: {dataset}') + else: + raise TypeError(f'dataset must a str, but got {type(dataset)}') + return labels diff --git a/detection/mmdet/core/evaluation/eval_hooks.py b/detection/mmdet/core/evaluation/eval_hooks.py new file mode 100644 index 0000000..6fb932e --- /dev/null +++ b/detection/mmdet/core/evaluation/eval_hooks.py @@ -0,0 +1,303 @@ +import os.path as osp +import warnings +from math import inf + +import mmcv +import torch.distributed as dist +from mmcv.runner import Hook +from torch.nn.modules.batchnorm import _BatchNorm +from torch.utils.data import DataLoader + +from mmdet.utils import get_root_logger + + +class EvalHook(Hook): + """Evaluation hook. + + Notes: + If new arguments are added for EvalHook, tools/test.py, + tools/analysis_tools/eval_metric.py may be effected. + + Attributes: + dataloader (DataLoader): A PyTorch dataloader. + start (int, optional): Evaluation starting epoch. It enables evaluation + before the training starts if ``start`` <= the resuming epoch. + If None, whether to evaluate is merely decided by ``interval``. + Default: None. + interval (int): Evaluation interval (by epochs). Default: 1. + save_best (str, optional): If a metric is specified, it would measure + the best checkpoint during evaluation. The information about best + checkpoint would be save in best.json. + Options are the evaluation metrics to the test dataset. e.g., + ``bbox_mAP``, ``segm_mAP`` for bbox detection and instance + segmentation. ``AR@100`` for proposal recall. If ``save_best`` is + ``auto``, the first key will be used. The interval of + ``CheckpointHook`` should device EvalHook. Default: None. + rule (str, optional): Comparison rule for best score. If set to None, + it will infer a reasonable rule. Keys such as 'mAP' or 'AR' will + be inferred by 'greater' rule. Keys contain 'loss' will be inferred + by 'less' rule. Options are 'greater', 'less'. Default: None. + **eval_kwargs: Evaluation arguments fed into the evaluate function of + the dataset. + """ + + rule_map = {'greater': lambda x, y: x > y, 'less': lambda x, y: x < y} + init_value_map = {'greater': -inf, 'less': inf} + greater_keys = ['mAP', 'AR'] + less_keys = ['loss'] + + def __init__(self, + dataloader, + start=None, + interval=1, + by_epoch=True, + save_best=None, + rule=None, + **eval_kwargs): + if not isinstance(dataloader, DataLoader): + raise TypeError('dataloader must be a pytorch DataLoader, but got' + f' {type(dataloader)}') + if not interval > 0: + raise ValueError(f'interval must be positive, but got {interval}') + if start is not None and start < 0: + warnings.warn( + f'The evaluation start epoch {start} is smaller than 0, ' + f'use 0 instead', UserWarning) + start = 0 + self.dataloader = dataloader + self.interval = interval + self.by_epoch = by_epoch + self.start = start + assert isinstance(save_best, str) or save_best is None + self.save_best = save_best + self.eval_kwargs = eval_kwargs + self.initial_epoch_flag = True + + self.logger = get_root_logger() + + if self.save_best is not None: + self._init_rule(rule, self.save_best) + + def _init_rule(self, rule, key_indicator): + """Initialize rule, key_indicator, comparison_func, and best score. + + Args: + rule (str | None): Comparison rule for best score. + key_indicator (str | None): Key indicator to determine the + comparison rule. + """ + if rule not in self.rule_map and rule is not None: + raise KeyError(f'rule must be greater, less or None, ' + f'but got {rule}.') + + if rule is None: + if key_indicator != 'auto': + if any(key in key_indicator for key in self.greater_keys): + rule = 'greater' + elif any(key in key_indicator for key in self.less_keys): + rule = 'less' + else: + raise ValueError(f'Cannot infer the rule for key ' + f'{key_indicator}, thus a specific rule ' + f'must be specified.') + self.rule = rule + self.key_indicator = key_indicator + if self.rule is not None: + self.compare_func = self.rule_map[self.rule] + + def before_run(self, runner): + if self.save_best is not None: + if runner.meta is None: + warnings.warn('runner.meta is None. Creating a empty one.') + runner.meta = dict() + runner.meta.setdefault('hook_msgs', dict()) + + def before_train_epoch(self, runner): + """Evaluate the model only at the start of training.""" + if not self.initial_epoch_flag: + return + if self.start is not None and runner.epoch >= self.start: + self.after_train_epoch(runner) + self.initial_epoch_flag = False + + def evaluation_flag(self, runner): + """Judge whether to perform_evaluation after this epoch. + + Returns: + bool: The flag indicating whether to perform evaluation. + """ + if self.start is None: + if not self.every_n_epochs(runner, self.interval): + # No evaluation during the interval epochs. + return False + elif (runner.epoch + 1) < self.start: + # No evaluation if start is larger than the current epoch. + return False + else: + # Evaluation only at epochs 3, 5, 7... if start==3 and interval==2 + if (runner.epoch + 1 - self.start) % self.interval: + return False + return True + + def after_train_epoch(self, runner): + if not self.by_epoch or not self.evaluation_flag(runner): + return + from mmdet.apis import single_gpu_test + results = single_gpu_test(runner.model, self.dataloader, show=False) + key_score = self.evaluate(runner, results) + if self.save_best: + self.save_best_checkpoint(runner, key_score) + + def after_train_iter(self, runner): + if self.by_epoch or not self.every_n_iters(runner, self.interval): + return + from mmdet.apis import single_gpu_test + results = single_gpu_test(runner.model, self.dataloader, show=False) + key_score = self.evaluate(runner, results) + if self.save_best: + self.save_best_checkpoint(runner, key_score) + + def save_best_checkpoint(self, runner, key_score): + best_score = runner.meta['hook_msgs'].get( + 'best_score', self.init_value_map[self.rule]) + if self.compare_func(key_score, best_score): + best_score = key_score + runner.meta['hook_msgs']['best_score'] = best_score + last_ckpt = runner.meta['hook_msgs']['last_ckpt'] + runner.meta['hook_msgs']['best_ckpt'] = last_ckpt + mmcv.symlink( + last_ckpt, + osp.join(runner.work_dir, f'best_{self.key_indicator}.pth')) + time_stamp = runner.epoch + 1 if self.by_epoch else runner.iter + 1 + self.logger.info(f'Now best checkpoint is epoch_{time_stamp}.pth.' + f'Best {self.key_indicator} is {best_score:0.4f}') + + def evaluate(self, runner, results): + eval_res = self.dataloader.dataset.evaluate( + results, logger=runner.logger, **self.eval_kwargs) + for name, val in eval_res.items(): + runner.log_buffer.output[name] = val + runner.log_buffer.ready = True + if self.save_best is not None: + if self.key_indicator == 'auto': + # infer from eval_results + self._init_rule(self.rule, list(eval_res.keys())[0]) + return eval_res[self.key_indicator] + else: + return None + + +class DistEvalHook(EvalHook): + """Distributed evaluation hook. + + Notes: + If new arguments are added, tools/test.py may be effected. + + Attributes: + dataloader (DataLoader): A PyTorch dataloader. + start (int, optional): Evaluation starting epoch. It enables evaluation + before the training starts if ``start`` <= the resuming epoch. + If None, whether to evaluate is merely decided by ``interval``. + Default: None. + interval (int): Evaluation interval (by epochs). Default: 1. + tmpdir (str | None): Temporary directory to save the results of all + processes. Default: None. + gpu_collect (bool): Whether to use gpu or cpu to collect results. + Default: False. + save_best (str, optional): If a metric is specified, it would measure + the best checkpoint during evaluation. The information about best + checkpoint would be save in best.json. + Options are the evaluation metrics to the test dataset. e.g., + ``bbox_mAP``, ``segm_mAP`` for bbox detection and instance + segmentation. ``AR@100`` for proposal recall. If ``save_best`` is + ``auto``, the first key will be used. The interval of + ``CheckpointHook`` should device EvalHook. Default: None. + rule (str | None): Comparison rule for best score. If set to None, + it will infer a reasonable rule. Default: 'None'. + broadcast_bn_buffer (bool): Whether to broadcast the + buffer(running_mean and running_var) of rank 0 to other rank + before evaluation. Default: True. + **eval_kwargs: Evaluation arguments fed into the evaluate function of + the dataset. + """ + + def __init__(self, + dataloader, + start=None, + interval=1, + by_epoch=True, + tmpdir=None, + gpu_collect=False, + save_best=None, + rule=None, + broadcast_bn_buffer=True, + **eval_kwargs): + super().__init__( + dataloader, + start=start, + interval=interval, + by_epoch=by_epoch, + save_best=save_best, + rule=rule, + **eval_kwargs) + self.broadcast_bn_buffer = broadcast_bn_buffer + self.tmpdir = tmpdir + self.gpu_collect = gpu_collect + + def _broadcast_bn_buffer(self, runner): + # Synchronization of BatchNorm's buffer (running_mean + # and running_var) is not supported in the DDP of pytorch, + # which may cause the inconsistent performance of models in + # different ranks, so we broadcast BatchNorm's buffers + # of rank 0 to other ranks to avoid this. + if self.broadcast_bn_buffer: + model = runner.model + for name, module in model.named_modules(): + if isinstance(module, + _BatchNorm) and module.track_running_stats: + dist.broadcast(module.running_var, 0) + dist.broadcast(module.running_mean, 0) + + def after_train_epoch(self, runner): + if not self.by_epoch or not self.evaluation_flag(runner): + return + + if self.broadcast_bn_buffer: + self._broadcast_bn_buffer(runner) + + from mmdet.apis import multi_gpu_test + tmpdir = self.tmpdir + if tmpdir is None: + tmpdir = osp.join(runner.work_dir, '.eval_hook') + results = multi_gpu_test( + runner.model, + self.dataloader, + tmpdir=tmpdir, + gpu_collect=self.gpu_collect) + if runner.rank == 0: + print('\n') + key_score = self.evaluate(runner, results) + if self.save_best: + self.save_best_checkpoint(runner, key_score) + + def after_train_iter(self, runner): + if self.by_epoch or not self.every_n_iters(runner, self.interval): + return + + if self.broadcast_bn_buffer: + self._broadcast_bn_buffer(runner) + + from mmdet.apis import multi_gpu_test + tmpdir = self.tmpdir + if tmpdir is None: + tmpdir = osp.join(runner.work_dir, '.eval_hook') + results = multi_gpu_test( + runner.model, + self.dataloader, + tmpdir=tmpdir, + gpu_collect=self.gpu_collect) + if runner.rank == 0: + print('\n') + key_score = self.evaluate(runner, results) + if self.save_best: + self.save_best_checkpoint(runner, key_score) diff --git a/detection/mmdet/core/evaluation/mean_ap.py b/detection/mmdet/core/evaluation/mean_ap.py new file mode 100644 index 0000000..1d653a3 --- /dev/null +++ b/detection/mmdet/core/evaluation/mean_ap.py @@ -0,0 +1,469 @@ +from multiprocessing import Pool + +import mmcv +import numpy as np +from mmcv.utils import print_log +from terminaltables import AsciiTable + +from .bbox_overlaps import bbox_overlaps +from .class_names import get_classes + + +def average_precision(recalls, precisions, mode='area'): + """Calculate average precision (for single or multiple scales). + + Args: + recalls (ndarray): shape (num_scales, num_dets) or (num_dets, ) + precisions (ndarray): shape (num_scales, num_dets) or (num_dets, ) + mode (str): 'area' or '11points', 'area' means calculating the area + under precision-recall curve, '11points' means calculating + the average precision of recalls at [0, 0.1, ..., 1] + + Returns: + float or ndarray: calculated average precision + """ + no_scale = False + if recalls.ndim == 1: + no_scale = True + recalls = recalls[np.newaxis, :] + precisions = precisions[np.newaxis, :] + assert recalls.shape == precisions.shape and recalls.ndim == 2 + num_scales = recalls.shape[0] + ap = np.zeros(num_scales, dtype=np.float32) + if mode == 'area': + zeros = np.zeros((num_scales, 1), dtype=recalls.dtype) + ones = np.ones((num_scales, 1), dtype=recalls.dtype) + mrec = np.hstack((zeros, recalls, ones)) + mpre = np.hstack((zeros, precisions, zeros)) + for i in range(mpre.shape[1] - 1, 0, -1): + mpre[:, i - 1] = np.maximum(mpre[:, i - 1], mpre[:, i]) + for i in range(num_scales): + ind = np.where(mrec[i, 1:] != mrec[i, :-1])[0] + ap[i] = np.sum( + (mrec[i, ind + 1] - mrec[i, ind]) * mpre[i, ind + 1]) + elif mode == '11points': + for i in range(num_scales): + for thr in np.arange(0, 1 + 1e-3, 0.1): + precs = precisions[i, recalls[i, :] >= thr] + prec = precs.max() if precs.size > 0 else 0 + ap[i] += prec + ap /= 11 + else: + raise ValueError( + 'Unrecognized mode, only "area" and "11points" are supported') + if no_scale: + ap = ap[0] + return ap + + +def tpfp_imagenet(det_bboxes, + gt_bboxes, + gt_bboxes_ignore=None, + default_iou_thr=0.5, + area_ranges=None): + """Check if detected bboxes are true positive or false positive. + + Args: + det_bbox (ndarray): Detected bboxes of this image, of shape (m, 5). + gt_bboxes (ndarray): GT bboxes of this image, of shape (n, 4). + gt_bboxes_ignore (ndarray): Ignored gt bboxes of this image, + of shape (k, 4). Default: None + default_iou_thr (float): IoU threshold to be considered as matched for + medium and large bboxes (small ones have special rules). + Default: 0.5. + area_ranges (list[tuple] | None): Range of bbox areas to be evaluated, + in the format [(min1, max1), (min2, max2), ...]. Default: None. + + Returns: + tuple[np.ndarray]: (tp, fp) whose elements are 0 and 1. The shape of + each array is (num_scales, m). + """ + # an indicator of ignored gts + gt_ignore_inds = np.concatenate( + (np.zeros(gt_bboxes.shape[0], dtype=np.bool), + np.ones(gt_bboxes_ignore.shape[0], dtype=np.bool))) + # stack gt_bboxes and gt_bboxes_ignore for convenience + gt_bboxes = np.vstack((gt_bboxes, gt_bboxes_ignore)) + + num_dets = det_bboxes.shape[0] + num_gts = gt_bboxes.shape[0] + if area_ranges is None: + area_ranges = [(None, None)] + num_scales = len(area_ranges) + # tp and fp are of shape (num_scales, num_gts), each row is tp or fp + # of a certain scale. + tp = np.zeros((num_scales, num_dets), dtype=np.float32) + fp = np.zeros((num_scales, num_dets), dtype=np.float32) + if gt_bboxes.shape[0] == 0: + if area_ranges == [(None, None)]: + fp[...] = 1 + else: + det_areas = (det_bboxes[:, 2] - det_bboxes[:, 0]) * ( + det_bboxes[:, 3] - det_bboxes[:, 1]) + for i, (min_area, max_area) in enumerate(area_ranges): + fp[i, (det_areas >= min_area) & (det_areas < max_area)] = 1 + return tp, fp + ious = bbox_overlaps(det_bboxes, gt_bboxes - 1) + gt_w = gt_bboxes[:, 2] - gt_bboxes[:, 0] + gt_h = gt_bboxes[:, 3] - gt_bboxes[:, 1] + iou_thrs = np.minimum((gt_w * gt_h) / ((gt_w + 10.0) * (gt_h + 10.0)), + default_iou_thr) + # sort all detections by scores in descending order + sort_inds = np.argsort(-det_bboxes[:, -1]) + for k, (min_area, max_area) in enumerate(area_ranges): + gt_covered = np.zeros(num_gts, dtype=bool) + # if no area range is specified, gt_area_ignore is all False + if min_area is None: + gt_area_ignore = np.zeros_like(gt_ignore_inds, dtype=bool) + else: + gt_areas = gt_w * gt_h + gt_area_ignore = (gt_areas < min_area) | (gt_areas >= max_area) + for i in sort_inds: + max_iou = -1 + matched_gt = -1 + # find best overlapped available gt + for j in range(num_gts): + # different from PASCAL VOC: allow finding other gts if the + # best overlapped ones are already matched by other det bboxes + if gt_covered[j]: + continue + elif ious[i, j] >= iou_thrs[j] and ious[i, j] > max_iou: + max_iou = ious[i, j] + matched_gt = j + # there are 4 cases for a det bbox: + # 1. it matches a gt, tp = 1, fp = 0 + # 2. it matches an ignored gt, tp = 0, fp = 0 + # 3. it matches no gt and within area range, tp = 0, fp = 1 + # 4. it matches no gt but is beyond area range, tp = 0, fp = 0 + if matched_gt >= 0: + gt_covered[matched_gt] = 1 + if not (gt_ignore_inds[matched_gt] + or gt_area_ignore[matched_gt]): + tp[k, i] = 1 + elif min_area is None: + fp[k, i] = 1 + else: + bbox = det_bboxes[i, :4] + area = (bbox[2] - bbox[0]) * (bbox[3] - bbox[1]) + if area >= min_area and area < max_area: + fp[k, i] = 1 + return tp, fp + + +def tpfp_default(det_bboxes, + gt_bboxes, + gt_bboxes_ignore=None, + iou_thr=0.5, + area_ranges=None): + """Check if detected bboxes are true positive or false positive. + + Args: + det_bbox (ndarray): Detected bboxes of this image, of shape (m, 5). + gt_bboxes (ndarray): GT bboxes of this image, of shape (n, 4). + gt_bboxes_ignore (ndarray): Ignored gt bboxes of this image, + of shape (k, 4). Default: None + iou_thr (float): IoU threshold to be considered as matched. + Default: 0.5. + area_ranges (list[tuple] | None): Range of bbox areas to be evaluated, + in the format [(min1, max1), (min2, max2), ...]. Default: None. + + Returns: + tuple[np.ndarray]: (tp, fp) whose elements are 0 and 1. The shape of + each array is (num_scales, m). + """ + # an indicator of ignored gts + gt_ignore_inds = np.concatenate( + (np.zeros(gt_bboxes.shape[0], dtype=np.bool), + np.ones(gt_bboxes_ignore.shape[0], dtype=np.bool))) + # stack gt_bboxes and gt_bboxes_ignore for convenience + gt_bboxes = np.vstack((gt_bboxes, gt_bboxes_ignore)) + + num_dets = det_bboxes.shape[0] + num_gts = gt_bboxes.shape[0] + if area_ranges is None: + area_ranges = [(None, None)] + num_scales = len(area_ranges) + # tp and fp are of shape (num_scales, num_gts), each row is tp or fp of + # a certain scale + tp = np.zeros((num_scales, num_dets), dtype=np.float32) + fp = np.zeros((num_scales, num_dets), dtype=np.float32) + + # if there is no gt bboxes in this image, then all det bboxes + # within area range are false positives + if gt_bboxes.shape[0] == 0: + if area_ranges == [(None, None)]: + fp[...] = 1 + else: + det_areas = (det_bboxes[:, 2] - det_bboxes[:, 0]) * ( + det_bboxes[:, 3] - det_bboxes[:, 1]) + for i, (min_area, max_area) in enumerate(area_ranges): + fp[i, (det_areas >= min_area) & (det_areas < max_area)] = 1 + return tp, fp + + ious = bbox_overlaps(det_bboxes, gt_bboxes) + # for each det, the max iou with all gts + ious_max = ious.max(axis=1) + # for each det, which gt overlaps most with it + ious_argmax = ious.argmax(axis=1) + # sort all dets in descending order by scores + sort_inds = np.argsort(-det_bboxes[:, -1]) + for k, (min_area, max_area) in enumerate(area_ranges): + gt_covered = np.zeros(num_gts, dtype=bool) + # if no area range is specified, gt_area_ignore is all False + if min_area is None: + gt_area_ignore = np.zeros_like(gt_ignore_inds, dtype=bool) + else: + gt_areas = (gt_bboxes[:, 2] - gt_bboxes[:, 0]) * ( + gt_bboxes[:, 3] - gt_bboxes[:, 1]) + gt_area_ignore = (gt_areas < min_area) | (gt_areas >= max_area) + for i in sort_inds: + if ious_max[i] >= iou_thr: + matched_gt = ious_argmax[i] + if not (gt_ignore_inds[matched_gt] + or gt_area_ignore[matched_gt]): + if not gt_covered[matched_gt]: + gt_covered[matched_gt] = True + tp[k, i] = 1 + else: + fp[k, i] = 1 + # otherwise ignore this detected bbox, tp = 0, fp = 0 + elif min_area is None: + fp[k, i] = 1 + else: + bbox = det_bboxes[i, :4] + area = (bbox[2] - bbox[0]) * (bbox[3] - bbox[1]) + if area >= min_area and area < max_area: + fp[k, i] = 1 + return tp, fp + + +def get_cls_results(det_results, annotations, class_id): + """Get det results and gt information of a certain class. + + Args: + det_results (list[list]): Same as `eval_map()`. + annotations (list[dict]): Same as `eval_map()`. + class_id (int): ID of a specific class. + + Returns: + tuple[list[np.ndarray]]: detected bboxes, gt bboxes, ignored gt bboxes + """ + cls_dets = [img_res[class_id] for img_res in det_results] + cls_gts = [] + cls_gts_ignore = [] + for ann in annotations: + gt_inds = ann['labels'] == class_id + cls_gts.append(ann['bboxes'][gt_inds, :]) + + if ann.get('labels_ignore', None) is not None: + ignore_inds = ann['labels_ignore'] == class_id + cls_gts_ignore.append(ann['bboxes_ignore'][ignore_inds, :]) + else: + cls_gts_ignore.append(np.empty((0, 4), dtype=np.float32)) + + return cls_dets, cls_gts, cls_gts_ignore + + +def eval_map(det_results, + annotations, + scale_ranges=None, + iou_thr=0.5, + dataset=None, + logger=None, + tpfp_fn=None, + nproc=4): + """Evaluate mAP of a dataset. + + Args: + det_results (list[list]): [[cls1_det, cls2_det, ...], ...]. + The outer list indicates images, and the inner list indicates + per-class detected bboxes. + annotations (list[dict]): Ground truth annotations where each item of + the list indicates an image. Keys of annotations are: + + - `bboxes`: numpy array of shape (n, 4) + - `labels`: numpy array of shape (n, ) + - `bboxes_ignore` (optional): numpy array of shape (k, 4) + - `labels_ignore` (optional): numpy array of shape (k, ) + scale_ranges (list[tuple] | None): Range of scales to be evaluated, + in the format [(min1, max1), (min2, max2), ...]. A range of + (32, 64) means the area range between (32**2, 64**2). + Default: None. + iou_thr (float): IoU threshold to be considered as matched. + Default: 0.5. + dataset (list[str] | str | None): Dataset name or dataset classes, + there are minor differences in metrics for different datsets, e.g. + "voc07", "imagenet_det", etc. Default: None. + logger (logging.Logger | str | None): The way to print the mAP + summary. See `mmcv.utils.print_log()` for details. Default: None. + tpfp_fn (callable | None): The function used to determine true/ + false positives. If None, :func:`tpfp_default` is used as default + unless dataset is 'det' or 'vid' (:func:`tpfp_imagenet` in this + case). If it is given as a function, then this function is used + to evaluate tp & fp. Default None. + nproc (int): Processes used for computing TP and FP. + Default: 4. + + Returns: + tuple: (mAP, [dict, dict, ...]) + """ + assert len(det_results) == len(annotations) + + num_imgs = len(det_results) + num_scales = len(scale_ranges) if scale_ranges is not None else 1 + num_classes = len(det_results[0]) # positive class num + area_ranges = ([(rg[0]**2, rg[1]**2) for rg in scale_ranges] + if scale_ranges is not None else None) + + pool = Pool(nproc) + eval_results = [] + for i in range(num_classes): + # get gt and det bboxes of this class + cls_dets, cls_gts, cls_gts_ignore = get_cls_results( + det_results, annotations, i) + # choose proper function according to datasets to compute tp and fp + if tpfp_fn is None: + if dataset in ['det', 'vid']: + tpfp_fn = tpfp_imagenet + else: + tpfp_fn = tpfp_default + if not callable(tpfp_fn): + raise ValueError( + f'tpfp_fn has to be a function or None, but got {tpfp_fn}') + + # compute tp and fp for each image with multiple processes + tpfp = pool.starmap( + tpfp_fn, + zip(cls_dets, cls_gts, cls_gts_ignore, + [iou_thr for _ in range(num_imgs)], + [area_ranges for _ in range(num_imgs)])) + tp, fp = tuple(zip(*tpfp)) + # calculate gt number of each scale + # ignored gts or gts beyond the specific scale are not counted + num_gts = np.zeros(num_scales, dtype=int) + for j, bbox in enumerate(cls_gts): + if area_ranges is None: + num_gts[0] += bbox.shape[0] + else: + gt_areas = (bbox[:, 2] - bbox[:, 0]) * ( + bbox[:, 3] - bbox[:, 1]) + for k, (min_area, max_area) in enumerate(area_ranges): + num_gts[k] += np.sum((gt_areas >= min_area) + & (gt_areas < max_area)) + # sort all det bboxes by score, also sort tp and fp + cls_dets = np.vstack(cls_dets) + num_dets = cls_dets.shape[0] + sort_inds = np.argsort(-cls_dets[:, -1]) + tp = np.hstack(tp)[:, sort_inds] + fp = np.hstack(fp)[:, sort_inds] + # calculate recall and precision with tp and fp + tp = np.cumsum(tp, axis=1) + fp = np.cumsum(fp, axis=1) + eps = np.finfo(np.float32).eps + recalls = tp / np.maximum(num_gts[:, np.newaxis], eps) + precisions = tp / np.maximum((tp + fp), eps) + # calculate AP + if scale_ranges is None: + recalls = recalls[0, :] + precisions = precisions[0, :] + num_gts = num_gts.item() + mode = 'area' if dataset != 'voc07' else '11points' + ap = average_precision(recalls, precisions, mode) + eval_results.append({ + 'num_gts': num_gts, + 'num_dets': num_dets, + 'recall': recalls, + 'precision': precisions, + 'ap': ap + }) + pool.close() + if scale_ranges is not None: + # shape (num_classes, num_scales) + all_ap = np.vstack([cls_result['ap'] for cls_result in eval_results]) + all_num_gts = np.vstack( + [cls_result['num_gts'] for cls_result in eval_results]) + mean_ap = [] + for i in range(num_scales): + if np.any(all_num_gts[:, i] > 0): + mean_ap.append(all_ap[all_num_gts[:, i] > 0, i].mean()) + else: + mean_ap.append(0.0) + else: + aps = [] + for cls_result in eval_results: + if cls_result['num_gts'] > 0: + aps.append(cls_result['ap']) + mean_ap = np.array(aps).mean().item() if aps else 0.0 + + print_map_summary( + mean_ap, eval_results, dataset, area_ranges, logger=logger) + + return mean_ap, eval_results + + +def print_map_summary(mean_ap, + results, + dataset=None, + scale_ranges=None, + logger=None): + """Print mAP and results of each class. + + A table will be printed to show the gts/dets/recall/AP of each class and + the mAP. + + Args: + mean_ap (float): Calculated from `eval_map()`. + results (list[dict]): Calculated from `eval_map()`. + dataset (list[str] | str | None): Dataset name or dataset classes. + scale_ranges (list[tuple] | None): Range of scales to be evaluated. + logger (logging.Logger | str | None): The way to print the mAP + summary. See `mmcv.utils.print_log()` for details. Default: None. + """ + + if logger == 'silent': + return + + if isinstance(results[0]['ap'], np.ndarray): + num_scales = len(results[0]['ap']) + else: + num_scales = 1 + + if scale_ranges is not None: + assert len(scale_ranges) == num_scales + + num_classes = len(results) + + recalls = np.zeros((num_scales, num_classes), dtype=np.float32) + aps = np.zeros((num_scales, num_classes), dtype=np.float32) + num_gts = np.zeros((num_scales, num_classes), dtype=int) + for i, cls_result in enumerate(results): + if cls_result['recall'].size > 0: + recalls[:, i] = np.array(cls_result['recall'], ndmin=2)[:, -1] + aps[:, i] = cls_result['ap'] + num_gts[:, i] = cls_result['num_gts'] + + if dataset is None: + label_names = [str(i) for i in range(num_classes)] + elif mmcv.is_str(dataset): + label_names = get_classes(dataset) + else: + label_names = dataset + + if not isinstance(mean_ap, list): + mean_ap = [mean_ap] + + header = ['class', 'gts', 'dets', 'recall', 'ap'] + for i in range(num_scales): + if scale_ranges is not None: + print_log(f'Scale range {scale_ranges[i]}', logger=logger) + table_data = [header] + for j in range(num_classes): + row_data = [ + label_names[j], num_gts[i, j], results[j]['num_dets'], + f'{recalls[i, j]:.3f}', f'{aps[i, j]:.3f}' + ] + table_data.append(row_data) + table_data.append(['mAP', '', '', '', f'{mean_ap[i]:.3f}']) + table = AsciiTable(table_data) + table.inner_footing_row_border = True + print_log('\n' + table.table, logger=logger) diff --git a/detection/mmdet/core/evaluation/recall.py b/detection/mmdet/core/evaluation/recall.py new file mode 100644 index 0000000..23ec744 --- /dev/null +++ b/detection/mmdet/core/evaluation/recall.py @@ -0,0 +1,189 @@ +from collections.abc import Sequence + +import numpy as np +from mmcv.utils import print_log +from terminaltables import AsciiTable + +from .bbox_overlaps import bbox_overlaps + + +def _recalls(all_ious, proposal_nums, thrs): + + img_num = all_ious.shape[0] + total_gt_num = sum([ious.shape[0] for ious in all_ious]) + + _ious = np.zeros((proposal_nums.size, total_gt_num), dtype=np.float32) + for k, proposal_num in enumerate(proposal_nums): + tmp_ious = np.zeros(0) + for i in range(img_num): + ious = all_ious[i][:, :proposal_num].copy() + gt_ious = np.zeros((ious.shape[0])) + if ious.size == 0: + tmp_ious = np.hstack((tmp_ious, gt_ious)) + continue + for j in range(ious.shape[0]): + gt_max_overlaps = ious.argmax(axis=1) + max_ious = ious[np.arange(0, ious.shape[0]), gt_max_overlaps] + gt_idx = max_ious.argmax() + gt_ious[j] = max_ious[gt_idx] + box_idx = gt_max_overlaps[gt_idx] + ious[gt_idx, :] = -1 + ious[:, box_idx] = -1 + tmp_ious = np.hstack((tmp_ious, gt_ious)) + _ious[k, :] = tmp_ious + + _ious = np.fliplr(np.sort(_ious, axis=1)) + recalls = np.zeros((proposal_nums.size, thrs.size)) + for i, thr in enumerate(thrs): + recalls[:, i] = (_ious >= thr).sum(axis=1) / float(total_gt_num) + + return recalls + + +def set_recall_param(proposal_nums, iou_thrs): + """Check proposal_nums and iou_thrs and set correct format.""" + if isinstance(proposal_nums, Sequence): + _proposal_nums = np.array(proposal_nums) + elif isinstance(proposal_nums, int): + _proposal_nums = np.array([proposal_nums]) + else: + _proposal_nums = proposal_nums + + if iou_thrs is None: + _iou_thrs = np.array([0.5]) + elif isinstance(iou_thrs, Sequence): + _iou_thrs = np.array(iou_thrs) + elif isinstance(iou_thrs, float): + _iou_thrs = np.array([iou_thrs]) + else: + _iou_thrs = iou_thrs + + return _proposal_nums, _iou_thrs + + +def eval_recalls(gts, + proposals, + proposal_nums=None, + iou_thrs=0.5, + logger=None): + """Calculate recalls. + + Args: + gts (list[ndarray]): a list of arrays of shape (n, 4) + proposals (list[ndarray]): a list of arrays of shape (k, 4) or (k, 5) + proposal_nums (int | Sequence[int]): Top N proposals to be evaluated. + iou_thrs (float | Sequence[float]): IoU thresholds. Default: 0.5. + logger (logging.Logger | str | None): The way to print the recall + summary. See `mmcv.utils.print_log()` for details. Default: None. + + Returns: + ndarray: recalls of different ious and proposal nums + """ + + img_num = len(gts) + assert img_num == len(proposals) + + proposal_nums, iou_thrs = set_recall_param(proposal_nums, iou_thrs) + + all_ious = [] + for i in range(img_num): + if proposals[i].ndim == 2 and proposals[i].shape[1] == 5: + scores = proposals[i][:, 4] + sort_idx = np.argsort(scores)[::-1] + img_proposal = proposals[i][sort_idx, :] + else: + img_proposal = proposals[i] + prop_num = min(img_proposal.shape[0], proposal_nums[-1]) + if gts[i] is None or gts[i].shape[0] == 0: + ious = np.zeros((0, img_proposal.shape[0]), dtype=np.float32) + else: + ious = bbox_overlaps(gts[i], img_proposal[:prop_num, :4]) + all_ious.append(ious) + all_ious = np.array(all_ious) + recalls = _recalls(all_ious, proposal_nums, iou_thrs) + + print_recall_summary(recalls, proposal_nums, iou_thrs, logger=logger) + return recalls + + +def print_recall_summary(recalls, + proposal_nums, + iou_thrs, + row_idxs=None, + col_idxs=None, + logger=None): + """Print recalls in a table. + + Args: + recalls (ndarray): calculated from `bbox_recalls` + proposal_nums (ndarray or list): top N proposals + iou_thrs (ndarray or list): iou thresholds + row_idxs (ndarray): which rows(proposal nums) to print + col_idxs (ndarray): which cols(iou thresholds) to print + logger (logging.Logger | str | None): The way to print the recall + summary. See `mmcv.utils.print_log()` for details. Default: None. + """ + proposal_nums = np.array(proposal_nums, dtype=np.int32) + iou_thrs = np.array(iou_thrs) + if row_idxs is None: + row_idxs = np.arange(proposal_nums.size) + if col_idxs is None: + col_idxs = np.arange(iou_thrs.size) + row_header = [''] + iou_thrs[col_idxs].tolist() + table_data = [row_header] + for i, num in enumerate(proposal_nums[row_idxs]): + row = [f'{val:.3f}' for val in recalls[row_idxs[i], col_idxs].tolist()] + row.insert(0, num) + table_data.append(row) + table = AsciiTable(table_data) + print_log('\n' + table.table, logger=logger) + + +def plot_num_recall(recalls, proposal_nums): + """Plot Proposal_num-Recalls curve. + + Args: + recalls(ndarray or list): shape (k,) + proposal_nums(ndarray or list): same shape as `recalls` + """ + if isinstance(proposal_nums, np.ndarray): + _proposal_nums = proposal_nums.tolist() + else: + _proposal_nums = proposal_nums + if isinstance(recalls, np.ndarray): + _recalls = recalls.tolist() + else: + _recalls = recalls + + import matplotlib.pyplot as plt + f = plt.figure() + plt.plot([0] + _proposal_nums, [0] + _recalls) + plt.xlabel('Proposal num') + plt.ylabel('Recall') + plt.axis([0, proposal_nums.max(), 0, 1]) + f.show() + + +def plot_iou_recall(recalls, iou_thrs): + """Plot IoU-Recalls curve. + + Args: + recalls(ndarray or list): shape (k,) + iou_thrs(ndarray or list): same shape as `recalls` + """ + if isinstance(iou_thrs, np.ndarray): + _iou_thrs = iou_thrs.tolist() + else: + _iou_thrs = iou_thrs + if isinstance(recalls, np.ndarray): + _recalls = recalls.tolist() + else: + _recalls = recalls + + import matplotlib.pyplot as plt + f = plt.figure() + plt.plot(_iou_thrs + [1.0], _recalls + [0.]) + plt.xlabel('IoU') + plt.ylabel('Recall') + plt.axis([iou_thrs.min(), 1, 0, 1]) + f.show() diff --git a/detection/mmdet/core/export/__init__.py b/detection/mmdet/core/export/__init__.py new file mode 100644 index 0000000..76589b1 --- /dev/null +++ b/detection/mmdet/core/export/__init__.py @@ -0,0 +1,8 @@ +from .pytorch2onnx import (build_model_from_cfg, + generate_inputs_and_wrap_model, + preprocess_example_input) + +__all__ = [ + 'build_model_from_cfg', 'generate_inputs_and_wrap_model', + 'preprocess_example_input' +] diff --git a/detection/mmdet/core/export/pytorch2onnx.py b/detection/mmdet/core/export/pytorch2onnx.py new file mode 100644 index 0000000..809a817 --- /dev/null +++ b/detection/mmdet/core/export/pytorch2onnx.py @@ -0,0 +1,154 @@ +from functools import partial + +import mmcv +import numpy as np +import torch +from mmcv.runner import load_checkpoint + + +def generate_inputs_and_wrap_model(config_path, + checkpoint_path, + input_config, + cfg_options=None): + """Prepare sample input and wrap model for ONNX export. + + The ONNX export API only accept args, and all inputs should be + torch.Tensor or corresponding types (such as tuple of tensor). + So we should call this function before exporting. This function will: + + 1. generate corresponding inputs which are used to execute the model. + 2. Wrap the model's forward function. + + For example, the MMDet models' forward function has a parameter + ``return_loss:bool``. As we want to set it as False while export API + supports neither bool type or kwargs. So we have to replace the forward + like: ``model.forward = partial(model.forward, return_loss=False)`` + + Args: + config_path (str): the OpenMMLab config for the model we want to + export to ONNX + checkpoint_path (str): Path to the corresponding checkpoint + input_config (dict): the exactly data in this dict depends on the + framework. For MMSeg, we can just declare the input shape, + and generate the dummy data accordingly. However, for MMDet, + we may pass the real img path, or the NMS will return None + as there is no legal bbox. + + Returns: + tuple: (model, tensor_data) wrapped model which can be called by \ + model(*tensor_data) and a list of inputs which are used to execute \ + the model while exporting. + """ + + model = build_model_from_cfg( + config_path, checkpoint_path, cfg_options=cfg_options) + one_img, one_meta = preprocess_example_input(input_config) + tensor_data = [one_img] + model.forward = partial( + model.forward, img_metas=[[one_meta]], return_loss=False) + + # pytorch has some bug in pytorch1.3, we have to fix it + # by replacing these existing op + opset_version = 11 + # put the import within the function thus it will not cause import error + # when not using this function + try: + from mmcv.onnx.symbolic import register_extra_symbolics + except ModuleNotFoundError: + raise NotImplementedError('please update mmcv to version>=v1.0.4') + register_extra_symbolics(opset_version) + + return model, tensor_data + + +def build_model_from_cfg(config_path, checkpoint_path, cfg_options=None): + """Build a model from config and load the given checkpoint. + + Args: + config_path (str): the OpenMMLab config for the model we want to + export to ONNX + checkpoint_path (str): Path to the corresponding checkpoint + + Returns: + torch.nn.Module: the built model + """ + from mmdet.models import build_detector + + cfg = mmcv.Config.fromfile(config_path) + if cfg_options is not None: + cfg.merge_from_dict(cfg_options) + # import modules from string list. + if cfg.get('custom_imports', None): + from mmcv.utils import import_modules_from_strings + import_modules_from_strings(**cfg['custom_imports']) + # set cudnn_benchmark + if cfg.get('cudnn_benchmark', False): + torch.backends.cudnn.benchmark = True + cfg.model.pretrained = None + cfg.data.test.test_mode = True + + # build the model + cfg.model.train_cfg = None + model = build_detector(cfg.model, test_cfg=cfg.get('test_cfg')) + load_checkpoint(model, checkpoint_path, map_location='cpu') + model.cpu().eval() + return model + + +def preprocess_example_input(input_config): + """Prepare an example input image for ``generate_inputs_and_wrap_model``. + + Args: + input_config (dict): customized config describing the example input. + + Returns: + tuple: (one_img, one_meta), tensor of the example input image and \ + meta information for the example input image. + + Examples: + >>> from mmdet.core.export import preprocess_example_input + >>> input_config = { + >>> 'input_shape': (1,3,224,224), + >>> 'input_path': 'demo/demo.jpg', + >>> 'normalize_cfg': { + >>> 'mean': (123.675, 116.28, 103.53), + >>> 'std': (58.395, 57.12, 57.375) + >>> } + >>> } + >>> one_img, one_meta = preprocess_example_input(input_config) + >>> print(one_img.shape) + torch.Size([1, 3, 224, 224]) + >>> print(one_meta) + {'img_shape': (224, 224, 3), + 'ori_shape': (224, 224, 3), + 'pad_shape': (224, 224, 3), + 'filename': '.png', + 'scale_factor': 1.0, + 'flip': False} + """ + input_path = input_config['input_path'] + input_shape = input_config['input_shape'] + one_img = mmcv.imread(input_path) + one_img = mmcv.imresize(one_img, input_shape[2:][::-1]) + show_img = one_img.copy() + if 'normalize_cfg' in input_config.keys(): + normalize_cfg = input_config['normalize_cfg'] + mean = np.array(normalize_cfg['mean'], dtype=np.float32) + std = np.array(normalize_cfg['std'], dtype=np.float32) + to_rgb = normalize_cfg.get('to_rgb', True) + one_img = mmcv.imnormalize(one_img, mean, std, to_rgb=to_rgb) + one_img = one_img.transpose(2, 0, 1) + one_img = torch.from_numpy(one_img).unsqueeze(0).float().requires_grad_( + True) + (_, C, H, W) = input_shape + one_meta = { + 'img_shape': (H, W, C), + 'ori_shape': (H, W, C), + 'pad_shape': (H, W, C), + 'filename': '.png', + 'scale_factor': 1.0, + 'flip': False, + 'show_img': show_img, + } + + return one_img, one_meta diff --git a/detection/mmdet/core/mask/__init__.py b/detection/mmdet/core/mask/__init__.py new file mode 100644 index 0000000..ab1e88b --- /dev/null +++ b/detection/mmdet/core/mask/__init__.py @@ -0,0 +1,8 @@ +from .mask_target import mask_target +from .structures import BaseInstanceMasks, BitmapMasks, PolygonMasks +from .utils import encode_mask_results, split_combined_polys + +__all__ = [ + 'split_combined_polys', 'mask_target', 'BaseInstanceMasks', 'BitmapMasks', + 'PolygonMasks', 'encode_mask_results' +] diff --git a/detection/mmdet/core/mask/mask_target.py b/detection/mmdet/core/mask/mask_target.py new file mode 100644 index 0000000..15d26a8 --- /dev/null +++ b/detection/mmdet/core/mask/mask_target.py @@ -0,0 +1,122 @@ +import numpy as np +import torch +from torch.nn.modules.utils import _pair + + +def mask_target(pos_proposals_list, pos_assigned_gt_inds_list, gt_masks_list, + cfg): + """Compute mask target for positive proposals in multiple images. + + Args: + pos_proposals_list (list[Tensor]): Positive proposals in multiple + images. + pos_assigned_gt_inds_list (list[Tensor]): Assigned GT indices for each + positive proposals. + gt_masks_list (list[:obj:`BaseInstanceMasks`]): Ground truth masks of + each image. + cfg (dict): Config dict that specifies the mask size. + + Returns: + list[Tensor]: Mask target of each image. + + Example: + >>> import mmcv + >>> import mmdet + >>> from mmdet.core.mask import BitmapMasks + >>> from mmdet.core.mask.mask_target import * + >>> H, W = 17, 18 + >>> cfg = mmcv.Config({'mask_size': (13, 14)}) + >>> rng = np.random.RandomState(0) + >>> # Positive proposals (tl_x, tl_y, br_x, br_y) for each image + >>> pos_proposals_list = [ + >>> torch.Tensor([ + >>> [ 7.2425, 5.5929, 13.9414, 14.9541], + >>> [ 7.3241, 3.6170, 16.3850, 15.3102], + >>> ]), + >>> torch.Tensor([ + >>> [ 4.8448, 6.4010, 7.0314, 9.7681], + >>> [ 5.9790, 2.6989, 7.4416, 4.8580], + >>> [ 0.0000, 0.0000, 0.1398, 9.8232], + >>> ]), + >>> ] + >>> # Corresponding class index for each proposal for each image + >>> pos_assigned_gt_inds_list = [ + >>> torch.LongTensor([7, 0]), + >>> torch.LongTensor([5, 4, 1]), + >>> ] + >>> # Ground truth mask for each true object for each image + >>> gt_masks_list = [ + >>> BitmapMasks(rng.rand(8, H, W), height=H, width=W), + >>> BitmapMasks(rng.rand(6, H, W), height=H, width=W), + >>> ] + >>> mask_targets = mask_target( + >>> pos_proposals_list, pos_assigned_gt_inds_list, + >>> gt_masks_list, cfg) + >>> assert mask_targets.shape == (5,) + cfg['mask_size'] + """ + cfg_list = [cfg for _ in range(len(pos_proposals_list))] + mask_targets = map(mask_target_single, pos_proposals_list, + pos_assigned_gt_inds_list, gt_masks_list, cfg_list) + mask_targets = list(mask_targets) + if len(mask_targets) > 0: + mask_targets = torch.cat(mask_targets) + return mask_targets + + +def mask_target_single(pos_proposals, pos_assigned_gt_inds, gt_masks, cfg): + """Compute mask target for each positive proposal in the image. + + Args: + pos_proposals (Tensor): Positive proposals. + pos_assigned_gt_inds (Tensor): Assigned GT inds of positive proposals. + gt_masks (:obj:`BaseInstanceMasks`): GT masks in the format of Bitmap + or Polygon. + cfg (dict): Config dict that indicate the mask size. + + Returns: + Tensor: Mask target of each positive proposals in the image. + + Example: + >>> import mmcv + >>> import mmdet + >>> from mmdet.core.mask import BitmapMasks + >>> from mmdet.core.mask.mask_target import * # NOQA + >>> H, W = 32, 32 + >>> cfg = mmcv.Config({'mask_size': (7, 11)}) + >>> rng = np.random.RandomState(0) + >>> # Masks for each ground truth box (relative to the image) + >>> gt_masks_data = rng.rand(3, H, W) + >>> gt_masks = BitmapMasks(gt_masks_data, height=H, width=W) + >>> # Predicted positive boxes in one image + >>> pos_proposals = torch.FloatTensor([ + >>> [ 16.2, 5.5, 19.9, 20.9], + >>> [ 17.3, 13.6, 19.3, 19.3], + >>> [ 14.8, 16.4, 17.0, 23.7], + >>> [ 0.0, 0.0, 16.0, 16.0], + >>> [ 4.0, 0.0, 20.0, 16.0], + >>> ]) + >>> # For each predicted proposal, its assignment to a gt mask + >>> pos_assigned_gt_inds = torch.LongTensor([0, 1, 2, 1, 1]) + >>> mask_targets = mask_target_single( + >>> pos_proposals, pos_assigned_gt_inds, gt_masks, cfg) + >>> assert mask_targets.shape == (5,) + cfg['mask_size'] + """ + device = pos_proposals.device + mask_size = _pair(cfg.mask_size) + num_pos = pos_proposals.size(0) + if num_pos > 0: + proposals_np = pos_proposals.cpu().numpy() + maxh, maxw = gt_masks.height, gt_masks.width + proposals_np[:, [0, 2]] = np.clip(proposals_np[:, [0, 2]], 0, maxw) + proposals_np[:, [1, 3]] = np.clip(proposals_np[:, [1, 3]], 0, maxh) + pos_assigned_gt_inds = pos_assigned_gt_inds.cpu().numpy() + + mask_targets = gt_masks.crop_and_resize( + proposals_np, mask_size, device=device, + inds=pos_assigned_gt_inds).to_ndarray() + + mask_targets = torch.from_numpy(mask_targets).float().to(device) + else: + mask_targets = pos_proposals.new_zeros((0, ) + mask_size) + + return mask_targets diff --git a/detection/mmdet/core/mask/structures.py b/detection/mmdet/core/mask/structures.py new file mode 100644 index 0000000..d9ec577 --- /dev/null +++ b/detection/mmdet/core/mask/structures.py @@ -0,0 +1,1024 @@ +from abc import ABCMeta, abstractmethod + +import cv2 +import mmcv +import numpy as np +import pycocotools.mask as maskUtils +import torch +from mmcv.ops.roi_align import roi_align + + +class BaseInstanceMasks(metaclass=ABCMeta): + """Base class for instance masks.""" + + @abstractmethod + def rescale(self, scale, interpolation='nearest'): + """Rescale masks as large as possible while keeping the aspect ratio. + For details can refer to `mmcv.imrescale`. + + Args: + scale (tuple[int]): The maximum size (h, w) of rescaled mask. + interpolation (str): Same as :func:`mmcv.imrescale`. + + Returns: + BaseInstanceMasks: The rescaled masks. + """ + + @abstractmethod + def resize(self, out_shape, interpolation='nearest'): + """Resize masks to the given out_shape. + + Args: + out_shape: Target (h, w) of resized mask. + interpolation (str): See :func:`mmcv.imresize`. + + Returns: + BaseInstanceMasks: The resized masks. + """ + + @abstractmethod + def flip(self, flip_direction='horizontal'): + """Flip masks alone the given direction. + + Args: + flip_direction (str): Either 'horizontal' or 'vertical'. + + Returns: + BaseInstanceMasks: The flipped masks. + """ + + @abstractmethod + def pad(self, out_shape, pad_val): + """Pad masks to the given size of (h, w). + + Args: + out_shape (tuple[int]): Target (h, w) of padded mask. + pad_val (int): The padded value. + + Returns: + BaseInstanceMasks: The padded masks. + """ + + @abstractmethod + def crop(self, bbox): + """Crop each mask by the given bbox. + + Args: + bbox (ndarray): Bbox in format [x1, y1, x2, y2], shape (4, ). + + Return: + BaseInstanceMasks: The cropped masks. + """ + + @abstractmethod + def crop_and_resize(self, + bboxes, + out_shape, + inds, + device, + interpolation='bilinear'): + """Crop and resize masks by the given bboxes. + + This function is mainly used in mask targets computation. + It firstly align mask to bboxes by assigned_inds, then crop mask by the + assigned bbox and resize to the size of (mask_h, mask_w) + + Args: + bboxes (Tensor): Bboxes in format [x1, y1, x2, y2], shape (N, 4) + out_shape (tuple[int]): Target (h, w) of resized mask + inds (ndarray): Indexes to assign masks to each bbox, + shape (N,) and values should be between [0, num_masks - 1]. + device (str): Device of bboxes + interpolation (str): See `mmcv.imresize` + + Return: + BaseInstanceMasks: the cropped and resized masks. + """ + + @abstractmethod + def expand(self, expanded_h, expanded_w, top, left): + """see :class:`Expand`.""" + + @property + @abstractmethod + def areas(self): + """ndarray: areas of each instance.""" + + @abstractmethod + def to_ndarray(self): + """Convert masks to the format of ndarray. + + Return: + ndarray: Converted masks in the format of ndarray. + """ + + @abstractmethod + def to_tensor(self, dtype, device): + """Convert masks to the format of Tensor. + + Args: + dtype (str): Dtype of converted mask. + device (torch.device): Device of converted masks. + + Returns: + Tensor: Converted masks in the format of Tensor. + """ + + @abstractmethod + def translate(self, + out_shape, + offset, + direction='horizontal', + fill_val=0, + interpolation='bilinear'): + """Translate the masks. + + Args: + out_shape (tuple[int]): Shape for output mask, format (h, w). + offset (int | float): The offset for translate. + direction (str): The translate direction, either "horizontal" + or "vertical". + fill_val (int | float): Border value. Default 0. + interpolation (str): Same as :func:`mmcv.imtranslate`. + + Returns: + Translated masks. + """ + + def shear(self, + out_shape, + magnitude, + direction='horizontal', + border_value=0, + interpolation='bilinear'): + """Shear the masks. + + Args: + out_shape (tuple[int]): Shape for output mask, format (h, w). + magnitude (int | float): The magnitude used for shear. + direction (str): The shear direction, either "horizontal" + or "vertical". + border_value (int | tuple[int]): Value used in case of a + constant border. Default 0. + interpolation (str): Same as in :func:`mmcv.imshear`. + + Returns: + ndarray: Sheared masks. + """ + + @abstractmethod + def rotate(self, out_shape, angle, center=None, scale=1.0, fill_val=0): + """Rotate the masks. + + Args: + out_shape (tuple[int]): Shape for output mask, format (h, w). + angle (int | float): Rotation angle in degrees. Positive values + mean counter-clockwise rotation. + center (tuple[float], optional): Center point (w, h) of the + rotation in source image. If not specified, the center of + the image will be used. + scale (int | float): Isotropic scale factor. + fill_val (int | float): Border value. Default 0 for masks. + + Returns: + Rotated masks. + """ + + +class BitmapMasks(BaseInstanceMasks): + """This class represents masks in the form of bitmaps. + + Args: + masks (ndarray): ndarray of masks in shape (N, H, W), where N is + the number of objects. + height (int): height of masks + width (int): width of masks + + Example: + >>> from mmdet.core.mask.structures import * # NOQA + >>> num_masks, H, W = 3, 32, 32 + >>> rng = np.random.RandomState(0) + >>> masks = (rng.rand(num_masks, H, W) > 0.1).astype(np.int) + >>> self = BitmapMasks(masks, height=H, width=W) + + >>> # demo crop_and_resize + >>> num_boxes = 5 + >>> bboxes = np.array([[0, 0, 30, 10.0]] * num_boxes) + >>> out_shape = (14, 14) + >>> inds = torch.randint(0, len(self), size=(num_boxes,)) + >>> device = 'cpu' + >>> interpolation = 'bilinear' + >>> new = self.crop_and_resize( + ... bboxes, out_shape, inds, device, interpolation) + >>> assert len(new) == num_boxes + >>> assert new.height, new.width == out_shape + """ + + def __init__(self, masks, height, width): + self.height = height + self.width = width + if len(masks) == 0: + self.masks = np.empty((0, self.height, self.width), dtype=np.uint8) + else: + assert isinstance(masks, (list, np.ndarray)) + if isinstance(masks, list): + assert isinstance(masks[0], np.ndarray) + assert masks[0].ndim == 2 # (H, W) + else: + assert masks.ndim == 3 # (N, H, W) + + self.masks = np.stack(masks).reshape(-1, height, width) + assert self.masks.shape[1] == self.height + assert self.masks.shape[2] == self.width + + def __getitem__(self, index): + """Index the BitmapMask. + + Args: + index (int | ndarray): Indices in the format of integer or ndarray. + + Returns: + :obj:`BitmapMasks`: Indexed bitmap masks. + """ + masks = self.masks[index].reshape(-1, self.height, self.width) + return BitmapMasks(masks, self.height, self.width) + + def __iter__(self): + return iter(self.masks) + + def __repr__(self): + s = self.__class__.__name__ + '(' + s += f'num_masks={len(self.masks)}, ' + s += f'height={self.height}, ' + s += f'width={self.width})' + return s + + def __len__(self): + """Number of masks.""" + return len(self.masks) + + def rescale(self, scale, interpolation='nearest'): + """See :func:`BaseInstanceMasks.rescale`.""" + if len(self.masks) == 0: + new_w, new_h = mmcv.rescale_size((self.width, self.height), scale) + rescaled_masks = np.empty((0, new_h, new_w), dtype=np.uint8) + else: + rescaled_masks = np.stack([ + mmcv.imrescale(mask, scale, interpolation=interpolation) + for mask in self.masks + ]) + height, width = rescaled_masks.shape[1:] + return BitmapMasks(rescaled_masks, height, width) + + def resize(self, out_shape, interpolation='nearest'): + """See :func:`BaseInstanceMasks.resize`.""" + if len(self.masks) == 0: + resized_masks = np.empty((0, *out_shape), dtype=np.uint8) + else: + resized_masks = np.stack([ + mmcv.imresize( + mask, out_shape[::-1], interpolation=interpolation) + for mask in self.masks + ]) + return BitmapMasks(resized_masks, *out_shape) + + def flip(self, flip_direction='horizontal'): + """See :func:`BaseInstanceMasks.flip`.""" + assert flip_direction in ('horizontal', 'vertical', 'diagonal') + + if len(self.masks) == 0: + flipped_masks = self.masks + else: + flipped_masks = np.stack([ + mmcv.imflip(mask, direction=flip_direction) + for mask in self.masks + ]) + return BitmapMasks(flipped_masks, self.height, self.width) + + def pad(self, out_shape, pad_val=0): + """See :func:`BaseInstanceMasks.pad`.""" + if len(self.masks) == 0: + padded_masks = np.empty((0, *out_shape), dtype=np.uint8) + else: + padded_masks = np.stack([ + mmcv.impad(mask, shape=out_shape, pad_val=pad_val) + for mask in self.masks + ]) + return BitmapMasks(padded_masks, *out_shape) + + def crop(self, bbox): + """See :func:`BaseInstanceMasks.crop`.""" + assert isinstance(bbox, np.ndarray) + assert bbox.ndim == 1 + + # clip the boundary + bbox = bbox.copy() + bbox[0::2] = np.clip(bbox[0::2], 0, self.width) + bbox[1::2] = np.clip(bbox[1::2], 0, self.height) + x1, y1, x2, y2 = bbox + w = np.maximum(x2 - x1, 1) + h = np.maximum(y2 - y1, 1) + + if len(self.masks) == 0: + cropped_masks = np.empty((0, h, w), dtype=np.uint8) + else: + cropped_masks = self.masks[:, y1:y1 + h, x1:x1 + w] + return BitmapMasks(cropped_masks, h, w) + + def crop_and_resize(self, + bboxes, + out_shape, + inds, + device='cpu', + interpolation='bilinear'): + """See :func:`BaseInstanceMasks.crop_and_resize`.""" + if len(self.masks) == 0: + empty_masks = np.empty((0, *out_shape), dtype=np.uint8) + return BitmapMasks(empty_masks, *out_shape) + + # convert bboxes to tensor + if isinstance(bboxes, np.ndarray): + bboxes = torch.from_numpy(bboxes).to(device=device) + if isinstance(inds, np.ndarray): + inds = torch.from_numpy(inds).to(device=device) + + num_bbox = bboxes.shape[0] + fake_inds = torch.arange( + num_bbox, device=device).to(dtype=bboxes.dtype)[:, None] + rois = torch.cat([fake_inds, bboxes], dim=1) # Nx5 + rois = rois.to(device=device) + if num_bbox > 0: + gt_masks_th = torch.from_numpy(self.masks).to(device).index_select( + 0, inds).to(dtype=rois.dtype) + targets = roi_align(gt_masks_th[:, None, :, :], rois, out_shape, + 1.0, 0, 'avg', True).squeeze(1) + resized_masks = (targets >= 0.5).cpu().numpy() + else: + resized_masks = [] + return BitmapMasks(resized_masks, *out_shape) + + def expand(self, expanded_h, expanded_w, top, left): + """See :func:`BaseInstanceMasks.expand`.""" + if len(self.masks) == 0: + expanded_mask = np.empty((0, expanded_h, expanded_w), + dtype=np.uint8) + else: + expanded_mask = np.zeros((len(self), expanded_h, expanded_w), + dtype=np.uint8) + expanded_mask[:, top:top + self.height, + left:left + self.width] = self.masks + return BitmapMasks(expanded_mask, expanded_h, expanded_w) + + def translate(self, + out_shape, + offset, + direction='horizontal', + fill_val=0, + interpolation='bilinear'): + """Translate the BitmapMasks. + + Args: + out_shape (tuple[int]): Shape for output mask, format (h, w). + offset (int | float): The offset for translate. + direction (str): The translate direction, either "horizontal" + or "vertical". + fill_val (int | float): Border value. Default 0 for masks. + interpolation (str): Same as :func:`mmcv.imtranslate`. + + Returns: + BitmapMasks: Translated BitmapMasks. + + Example: + >>> from mmdet.core.mask.structures import BitmapMasks + >>> self = BitmapMasks.random(dtype=np.uint8) + >>> out_shape = (32, 32) + >>> offset = 4 + >>> direction = 'horizontal' + >>> fill_val = 0 + >>> interpolation = 'bilinear' + >>> # Note, There seem to be issues when: + >>> # * out_shape is different than self's shape + >>> # * the mask dtype is not supported by cv2.AffineWarp + >>> new = self.translate(out_shape, offset, direction, fill_val, + >>> interpolation) + >>> assert len(new) == len(self) + >>> assert new.height, new.width == out_shape + """ + if len(self.masks) == 0: + translated_masks = np.empty((0, *out_shape), dtype=np.uint8) + else: + translated_masks = mmcv.imtranslate( + self.masks.transpose((1, 2, 0)), + offset, + direction, + border_value=fill_val, + interpolation=interpolation) + if translated_masks.ndim == 2: + translated_masks = translated_masks[:, :, None] + translated_masks = translated_masks.transpose( + (2, 0, 1)).astype(self.masks.dtype) + return BitmapMasks(translated_masks, *out_shape) + + def shear(self, + out_shape, + magnitude, + direction='horizontal', + border_value=0, + interpolation='bilinear'): + """Shear the BitmapMasks. + + Args: + out_shape (tuple[int]): Shape for output mask, format (h, w). + magnitude (int | float): The magnitude used for shear. + direction (str): The shear direction, either "horizontal" + or "vertical". + border_value (int | tuple[int]): Value used in case of a + constant border. + interpolation (str): Same as in :func:`mmcv.imshear`. + + Returns: + BitmapMasks: The sheared masks. + """ + if len(self.masks) == 0: + sheared_masks = np.empty((0, *out_shape), dtype=np.uint8) + else: + sheared_masks = mmcv.imshear( + self.masks.transpose((1, 2, 0)), + magnitude, + direction, + border_value=border_value, + interpolation=interpolation) + if sheared_masks.ndim == 2: + sheared_masks = sheared_masks[:, :, None] + sheared_masks = sheared_masks.transpose( + (2, 0, 1)).astype(self.masks.dtype) + return BitmapMasks(sheared_masks, *out_shape) + + def rotate(self, out_shape, angle, center=None, scale=1.0, fill_val=0): + """Rotate the BitmapMasks. + + Args: + out_shape (tuple[int]): Shape for output mask, format (h, w). + angle (int | float): Rotation angle in degrees. Positive values + mean counter-clockwise rotation. + center (tuple[float], optional): Center point (w, h) of the + rotation in source image. If not specified, the center of + the image will be used. + scale (int | float): Isotropic scale factor. + fill_val (int | float): Border value. Default 0 for masks. + + Returns: + BitmapMasks: Rotated BitmapMasks. + """ + if len(self.masks) == 0: + rotated_masks = np.empty((0, *out_shape), dtype=self.masks.dtype) + else: + rotated_masks = mmcv.imrotate( + self.masks.transpose((1, 2, 0)), + angle, + center=center, + scale=scale, + border_value=fill_val) + if rotated_masks.ndim == 2: + # case when only one mask, (h, w) + rotated_masks = rotated_masks[:, :, None] # (h, w, 1) + rotated_masks = rotated_masks.transpose( + (2, 0, 1)).astype(self.masks.dtype) + return BitmapMasks(rotated_masks, *out_shape) + + @property + def areas(self): + """See :py:attr:`BaseInstanceMasks.areas`.""" + return self.masks.sum((1, 2)) + + def to_ndarray(self): + """See :func:`BaseInstanceMasks.to_ndarray`.""" + return self.masks + + def to_tensor(self, dtype, device): + """See :func:`BaseInstanceMasks.to_tensor`.""" + return torch.tensor(self.masks, dtype=dtype, device=device) + + @classmethod + def random(cls, + num_masks=3, + height=32, + width=32, + dtype=np.uint8, + rng=None): + """Generate random bitmap masks for demo / testing purposes. + + Example: + >>> from mmdet.core.mask.structures import BitmapMasks + >>> self = BitmapMasks.random() + >>> print('self = {}'.format(self)) + self = BitmapMasks(num_masks=3, height=32, width=32) + """ + from mmdet.utils.util_random import ensure_rng + rng = ensure_rng(rng) + masks = (rng.rand(num_masks, height, width) > 0.1).astype(dtype) + self = cls(masks, height=height, width=width) + return self + + +class PolygonMasks(BaseInstanceMasks): + """This class represents masks in the form of polygons. + + Polygons is a list of three levels. The first level of the list + corresponds to objects, the second level to the polys that compose the + object, the third level to the poly coordinates + + Args: + masks (list[list[ndarray]]): The first level of the list + corresponds to objects, the second level to the polys that + compose the object, the third level to the poly coordinates + height (int): height of masks + width (int): width of masks + + Example: + >>> from mmdet.core.mask.structures import * # NOQA + >>> masks = [ + >>> [ np.array([0, 0, 10, 0, 10, 10., 0, 10, 0, 0]) ] + >>> ] + >>> height, width = 16, 16 + >>> self = PolygonMasks(masks, height, width) + + >>> # demo translate + >>> new = self.translate((16, 16), 4., direction='horizontal') + >>> assert np.all(new.masks[0][0][1::2] == masks[0][0][1::2]) + >>> assert np.all(new.masks[0][0][0::2] == masks[0][0][0::2] + 4) + + >>> # demo crop_and_resize + >>> num_boxes = 3 + >>> bboxes = np.array([[0, 0, 30, 10.0]] * num_boxes) + >>> out_shape = (16, 16) + >>> inds = torch.randint(0, len(self), size=(num_boxes,)) + >>> device = 'cpu' + >>> interpolation = 'bilinear' + >>> new = self.crop_and_resize( + ... bboxes, out_shape, inds, device, interpolation) + >>> assert len(new) == num_boxes + >>> assert new.height, new.width == out_shape + """ + + def __init__(self, masks, height, width): + assert isinstance(masks, list) + if len(masks) > 0: + assert isinstance(masks[0], list) + assert isinstance(masks[0][0], np.ndarray) + + self.height = height + self.width = width + self.masks = masks + + def __getitem__(self, index): + """Index the polygon masks. + + Args: + index (ndarray | List): The indices. + + Returns: + :obj:`PolygonMasks`: The indexed polygon masks. + """ + if isinstance(index, np.ndarray): + index = index.tolist() + if isinstance(index, list): + masks = [self.masks[i] for i in index] + else: + try: + masks = self.masks[index] + except Exception: + raise ValueError( + f'Unsupported input of type {type(index)} for indexing!') + if len(masks) and isinstance(masks[0], np.ndarray): + masks = [masks] # ensure a list of three levels + return PolygonMasks(masks, self.height, self.width) + + def __iter__(self): + return iter(self.masks) + + def __repr__(self): + s = self.__class__.__name__ + '(' + s += f'num_masks={len(self.masks)}, ' + s += f'height={self.height}, ' + s += f'width={self.width})' + return s + + def __len__(self): + """Number of masks.""" + return len(self.masks) + + def rescale(self, scale, interpolation=None): + """see :func:`BaseInstanceMasks.rescale`""" + new_w, new_h = mmcv.rescale_size((self.width, self.height), scale) + if len(self.masks) == 0: + rescaled_masks = PolygonMasks([], new_h, new_w) + else: + rescaled_masks = self.resize((new_h, new_w)) + return rescaled_masks + + def resize(self, out_shape, interpolation=None): + """see :func:`BaseInstanceMasks.resize`""" + if len(self.masks) == 0: + resized_masks = PolygonMasks([], *out_shape) + else: + h_scale = out_shape[0] / self.height + w_scale = out_shape[1] / self.width + resized_masks = [] + for poly_per_obj in self.masks: + resized_poly = [] + for p in poly_per_obj: + p = p.copy() + p[0::2] *= w_scale + p[1::2] *= h_scale + resized_poly.append(p) + resized_masks.append(resized_poly) + resized_masks = PolygonMasks(resized_masks, *out_shape) + return resized_masks + + def flip(self, flip_direction='horizontal'): + """see :func:`BaseInstanceMasks.flip`""" + assert flip_direction in ('horizontal', 'vertical', 'diagonal') + if len(self.masks) == 0: + flipped_masks = PolygonMasks([], self.height, self.width) + else: + flipped_masks = [] + for poly_per_obj in self.masks: + flipped_poly_per_obj = [] + for p in poly_per_obj: + p = p.copy() + if flip_direction == 'horizontal': + p[0::2] = self.width - p[0::2] + elif flip_direction == 'vertical': + p[1::2] = self.height - p[1::2] + else: + p[0::2] = self.width - p[0::2] + p[1::2] = self.height - p[1::2] + flipped_poly_per_obj.append(p) + flipped_masks.append(flipped_poly_per_obj) + flipped_masks = PolygonMasks(flipped_masks, self.height, + self.width) + return flipped_masks + + def crop(self, bbox): + """see :func:`BaseInstanceMasks.crop`""" + assert isinstance(bbox, np.ndarray) + assert bbox.ndim == 1 + + # clip the boundary + bbox = bbox.copy() + bbox[0::2] = np.clip(bbox[0::2], 0, self.width) + bbox[1::2] = np.clip(bbox[1::2], 0, self.height) + x1, y1, x2, y2 = bbox + w = np.maximum(x2 - x1, 1) + h = np.maximum(y2 - y1, 1) + + if len(self.masks) == 0: + cropped_masks = PolygonMasks([], h, w) + else: + cropped_masks = [] + for poly_per_obj in self.masks: + cropped_poly_per_obj = [] + for p in poly_per_obj: + # pycocotools will clip the boundary + p = p.copy() + p[0::2] -= bbox[0] + p[1::2] -= bbox[1] + cropped_poly_per_obj.append(p) + cropped_masks.append(cropped_poly_per_obj) + cropped_masks = PolygonMasks(cropped_masks, h, w) + return cropped_masks + + def pad(self, out_shape, pad_val=0): + """padding has no effect on polygons`""" + return PolygonMasks(self.masks, *out_shape) + + def expand(self, *args, **kwargs): + """TODO: Add expand for polygon""" + raise NotImplementedError + + def crop_and_resize(self, + bboxes, + out_shape, + inds, + device='cpu', + interpolation='bilinear'): + """see :func:`BaseInstanceMasks.crop_and_resize`""" + out_h, out_w = out_shape + if len(self.masks) == 0: + return PolygonMasks([], out_h, out_w) + + resized_masks = [] + for i in range(len(bboxes)): + mask = self.masks[inds[i]] + bbox = bboxes[i, :] + x1, y1, x2, y2 = bbox + w = np.maximum(x2 - x1, 1) + h = np.maximum(y2 - y1, 1) + h_scale = out_h / max(h, 0.1) # avoid too large scale + w_scale = out_w / max(w, 0.1) + + resized_mask = [] + for p in mask: + p = p.copy() + # crop + # pycocotools will clip the boundary + p[0::2] -= bbox[0] + p[1::2] -= bbox[1] + + # resize + p[0::2] *= w_scale + p[1::2] *= h_scale + resized_mask.append(p) + resized_masks.append(resized_mask) + return PolygonMasks(resized_masks, *out_shape) + + def translate(self, + out_shape, + offset, + direction='horizontal', + fill_val=None, + interpolation=None): + """Translate the PolygonMasks. + + Example: + >>> self = PolygonMasks.random(dtype=np.int) + >>> out_shape = (self.height, self.width) + >>> new = self.translate(out_shape, 4., direction='horizontal') + >>> assert np.all(new.masks[0][0][1::2] == self.masks[0][0][1::2]) + >>> assert np.all(new.masks[0][0][0::2] == self.masks[0][0][0::2] + 4) # noqa: E501 + """ + assert fill_val is None or fill_val == 0, 'Here fill_val is not '\ + f'used, and defaultly should be None or 0. got {fill_val}.' + if len(self.masks) == 0: + translated_masks = PolygonMasks([], *out_shape) + else: + translated_masks = [] + for poly_per_obj in self.masks: + translated_poly_per_obj = [] + for p in poly_per_obj: + p = p.copy() + if direction == 'horizontal': + p[0::2] = np.clip(p[0::2] + offset, 0, out_shape[1]) + elif direction == 'vertical': + p[1::2] = np.clip(p[1::2] + offset, 0, out_shape[0]) + translated_poly_per_obj.append(p) + translated_masks.append(translated_poly_per_obj) + translated_masks = PolygonMasks(translated_masks, *out_shape) + return translated_masks + + def shear(self, + out_shape, + magnitude, + direction='horizontal', + border_value=0, + interpolation='bilinear'): + """See :func:`BaseInstanceMasks.shear`.""" + if len(self.masks) == 0: + sheared_masks = PolygonMasks([], *out_shape) + else: + sheared_masks = [] + if direction == 'horizontal': + shear_matrix = np.stack([[1, magnitude], + [0, 1]]).astype(np.float32) + elif direction == 'vertical': + shear_matrix = np.stack([[1, 0], [magnitude, + 1]]).astype(np.float32) + for poly_per_obj in self.masks: + sheared_poly = [] + for p in poly_per_obj: + p = np.stack([p[0::2], p[1::2]], axis=0) # [2, n] + new_coords = np.matmul(shear_matrix, p) # [2, n] + new_coords[0, :] = np.clip(new_coords[0, :], 0, + out_shape[1]) + new_coords[1, :] = np.clip(new_coords[1, :], 0, + out_shape[0]) + sheared_poly.append( + new_coords.transpose((1, 0)).reshape(-1)) + sheared_masks.append(sheared_poly) + sheared_masks = PolygonMasks(sheared_masks, *out_shape) + return sheared_masks + + def rotate(self, out_shape, angle, center=None, scale=1.0, fill_val=0): + """See :func:`BaseInstanceMasks.rotate`.""" + if len(self.masks) == 0: + rotated_masks = PolygonMasks([], *out_shape) + else: + rotated_masks = [] + rotate_matrix = cv2.getRotationMatrix2D(center, -angle, scale) + for poly_per_obj in self.masks: + rotated_poly = [] + for p in poly_per_obj: + p = p.copy() + coords = np.stack([p[0::2], p[1::2]], axis=1) # [n, 2] + # pad 1 to convert from format [x, y] to homogeneous + # coordinates format [x, y, 1] + coords = np.concatenate( + (coords, np.ones((coords.shape[0], 1), coords.dtype)), + axis=1) # [n, 3] + rotated_coords = np.matmul( + rotate_matrix[None, :, :], + coords[:, :, None])[..., 0] # [n, 2, 1] -> [n, 2] + rotated_coords[:, 0] = np.clip(rotated_coords[:, 0], 0, + out_shape[1]) + rotated_coords[:, 1] = np.clip(rotated_coords[:, 1], 0, + out_shape[0]) + rotated_poly.append(rotated_coords.reshape(-1)) + rotated_masks.append(rotated_poly) + rotated_masks = PolygonMasks(rotated_masks, *out_shape) + return rotated_masks + + def to_bitmap(self): + """convert polygon masks to bitmap masks.""" + bitmap_masks = self.to_ndarray() + return BitmapMasks(bitmap_masks, self.height, self.width) + + @property + def areas(self): + """Compute areas of masks. + + This func is modified from `detectron2 + `_. + The function only works with Polygons using the shoelace formula. + + Return: + ndarray: areas of each instance + """ # noqa: W501 + area = [] + for polygons_per_obj in self.masks: + area_per_obj = 0 + for p in polygons_per_obj: + area_per_obj += self._polygon_area(p[0::2], p[1::2]) + area.append(area_per_obj) + return np.asarray(area) + + def _polygon_area(self, x, y): + """Compute the area of a component of a polygon. + + Using the shoelace formula: + https://stackoverflow.com/questions/24467972/calculate-area-of-polygon-given-x-y-coordinates + + Args: + x (ndarray): x coordinates of the component + y (ndarray): y coordinates of the component + + Return: + float: the are of the component + """ # noqa: 501 + return 0.5 * np.abs( + np.dot(x, np.roll(y, 1)) - np.dot(y, np.roll(x, 1))) + + def to_ndarray(self): + """Convert masks to the format of ndarray.""" + if len(self.masks) == 0: + return np.empty((0, self.height, self.width), dtype=np.uint8) + bitmap_masks = [] + for poly_per_obj in self.masks: + bitmap_masks.append( + polygon_to_bitmap(poly_per_obj, self.height, self.width)) + return np.stack(bitmap_masks) + + def to_tensor(self, dtype, device): + """See :func:`BaseInstanceMasks.to_tensor`.""" + if len(self.masks) == 0: + return torch.empty((0, self.height, self.width), + dtype=dtype, + device=device) + ndarray_masks = self.to_ndarray() + return torch.tensor(ndarray_masks, dtype=dtype, device=device) + + @classmethod + def random(cls, + num_masks=3, + height=32, + width=32, + n_verts=5, + dtype=np.float32, + rng=None): + """Generate random polygon masks for demo / testing purposes. + + Adapted from [1]_ + + References: + .. [1] https://gitlab.kitware.com/computer-vision/kwimage/-/blob/928cae35ca8/kwimage/structs/polygon.py#L379 # noqa: E501 + + Example: + >>> from mmdet.core.mask.structures import PolygonMasks + >>> self = PolygonMasks.random() + >>> print('self = {}'.format(self)) + """ + from mmdet.utils.util_random import ensure_rng + rng = ensure_rng(rng) + + def _gen_polygon(n, irregularity, spikeyness): + """Creates the polygon by sampling points on a circle around the + centre. Random noise is added by varying the angular spacing + between sequential points, and by varying the radial distance of + each point from the centre. + + Based on original code by Mike Ounsworth + + Args: + n (int): number of vertices + irregularity (float): [0,1] indicating how much variance there + is in the angular spacing of vertices. [0,1] will map to + [0, 2pi/numberOfVerts] + spikeyness (float): [0,1] indicating how much variance there is + in each vertex from the circle of radius aveRadius. [0,1] + will map to [0, aveRadius] + + Returns: + a list of vertices, in CCW order. + """ + from scipy.stats import truncnorm + # Generate around the unit circle + cx, cy = (0.0, 0.0) + radius = 1 + + tau = np.pi * 2 + + irregularity = np.clip(irregularity, 0, 1) * 2 * np.pi / n + spikeyness = np.clip(spikeyness, 1e-9, 1) + + # generate n angle steps + lower = (tau / n) - irregularity + upper = (tau / n) + irregularity + angle_steps = rng.uniform(lower, upper, n) + + # normalize the steps so that point 0 and point n+1 are the same + k = angle_steps.sum() / (2 * np.pi) + angles = (angle_steps / k).cumsum() + rng.uniform(0, tau) + + # Convert high and low values to be wrt the standard normal range + # https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.truncnorm.html + low = 0 + high = 2 * radius + mean = radius + std = spikeyness + a = (low - mean) / std + b = (high - mean) / std + tnorm = truncnorm(a=a, b=b, loc=mean, scale=std) + + # now generate the points + radii = tnorm.rvs(n, random_state=rng) + x_pts = cx + radii * np.cos(angles) + y_pts = cy + radii * np.sin(angles) + + points = np.hstack([x_pts[:, None], y_pts[:, None]]) + + # Scale to 0-1 space + points = points - points.min(axis=0) + points = points / points.max(axis=0) + + # Randomly place within 0-1 space + points = points * (rng.rand() * .8 + .2) + min_pt = points.min(axis=0) + max_pt = points.max(axis=0) + + high = (1 - max_pt) + low = (0 - min_pt) + offset = (rng.rand(2) * (high - low)) + low + points = points + offset + return points + + def _order_vertices(verts): + """ + References: + https://stackoverflow.com/questions/1709283/how-can-i-sort-a-coordinate-list-for-a-rectangle-counterclockwise + """ + mlat = verts.T[0].sum() / len(verts) + mlng = verts.T[1].sum() / len(verts) + + tau = np.pi * 2 + angle = (np.arctan2(mlat - verts.T[0], verts.T[1] - mlng) + + tau) % tau + sortx = angle.argsort() + verts = verts.take(sortx, axis=0) + return verts + + # Generate a random exterior for each requested mask + masks = [] + for _ in range(num_masks): + exterior = _order_vertices(_gen_polygon(n_verts, 0.9, 0.9)) + exterior = (exterior * [(width, height)]).astype(dtype) + masks.append([exterior.ravel()]) + + self = cls(masks, height, width) + return self + + +def polygon_to_bitmap(polygons, height, width): + """Convert masks from the form of polygons to bitmaps. + + Args: + polygons (list[ndarray]): masks in polygon representation + height (int): mask height + width (int): mask width + + Return: + ndarray: the converted masks in bitmap representation + """ + rles = maskUtils.frPyObjects(polygons, height, width) + rle = maskUtils.merge(rles) + bitmap_mask = maskUtils.decode(rle).astype(np.bool) + return bitmap_mask diff --git a/detection/mmdet/core/mask/utils.py b/detection/mmdet/core/mask/utils.py new file mode 100644 index 0000000..c882082 --- /dev/null +++ b/detection/mmdet/core/mask/utils.py @@ -0,0 +1,63 @@ +import mmcv +import numpy as np +import pycocotools.mask as mask_util + + +def split_combined_polys(polys, poly_lens, polys_per_mask): + """Split the combined 1-D polys into masks. + + A mask is represented as a list of polys, and a poly is represented as + a 1-D array. In dataset, all masks are concatenated into a single 1-D + tensor. Here we need to split the tensor into original representations. + + Args: + polys (list): a list (length = image num) of 1-D tensors + poly_lens (list): a list (length = image num) of poly length + polys_per_mask (list): a list (length = image num) of poly number + of each mask + + Returns: + list: a list (length = image num) of list (length = mask num) of \ + list (length = poly num) of numpy array. + """ + mask_polys_list = [] + for img_id in range(len(polys)): + polys_single = polys[img_id] + polys_lens_single = poly_lens[img_id].tolist() + polys_per_mask_single = polys_per_mask[img_id].tolist() + + split_polys = mmcv.slice_list(polys_single, polys_lens_single) + mask_polys = mmcv.slice_list(split_polys, polys_per_mask_single) + mask_polys_list.append(mask_polys) + return mask_polys_list + + +# TODO: move this function to more proper place +def encode_mask_results(mask_results): + """Encode bitmap mask to RLE code. + + Args: + mask_results (list | tuple[list]): bitmap mask results. + In mask scoring rcnn, mask_results is a tuple of (segm_results, + segm_cls_score). + + Returns: + list | tuple: RLE encoded mask. + """ + if isinstance(mask_results, tuple): # mask scoring + cls_segms, cls_mask_scores = mask_results + else: + cls_segms = mask_results + num_classes = len(cls_segms) + encoded_mask_results = [[] for _ in range(num_classes)] + for i in range(len(cls_segms)): + for cls_segm in cls_segms[i]: + encoded_mask_results[i].append( + mask_util.encode( + np.array( + cls_segm[:, :, np.newaxis], order='F', + dtype='uint8'))[0]) # encoded with RLE + if isinstance(mask_results, tuple): + return encoded_mask_results, cls_mask_scores + else: + return encoded_mask_results diff --git a/detection/mmdet/core/post_processing/__init__.py b/detection/mmdet/core/post_processing/__init__.py new file mode 100644 index 0000000..880b3f0 --- /dev/null +++ b/detection/mmdet/core/post_processing/__init__.py @@ -0,0 +1,8 @@ +from .bbox_nms import fast_nms, multiclass_nms +from .merge_augs import (merge_aug_bboxes, merge_aug_masks, + merge_aug_proposals, merge_aug_scores) + +__all__ = [ + 'multiclass_nms', 'merge_aug_proposals', 'merge_aug_bboxes', + 'merge_aug_scores', 'merge_aug_masks', 'fast_nms' +] diff --git a/detection/mmdet/core/post_processing/bbox_nms.py b/detection/mmdet/core/post_processing/bbox_nms.py new file mode 100644 index 0000000..966d3a6 --- /dev/null +++ b/detection/mmdet/core/post_processing/bbox_nms.py @@ -0,0 +1,168 @@ +import torch +from mmcv.ops.nms import batched_nms + +from mmdet.core.bbox.iou_calculators import bbox_overlaps + + +def multiclass_nms(multi_bboxes, + multi_scores, + score_thr, + nms_cfg, + max_num=-1, + score_factors=None, + return_inds=False): + """NMS for multi-class bboxes. + + Args: + multi_bboxes (Tensor): shape (n, #class*4) or (n, 4) + multi_scores (Tensor): shape (n, #class), where the last column + contains scores of the background class, but this will be ignored. + score_thr (float): bbox threshold, bboxes with scores lower than it + will not be considered. + nms_thr (float): NMS IoU threshold + max_num (int, optional): if there are more than max_num bboxes after + NMS, only top max_num will be kept. Default to -1. + score_factors (Tensor, optional): The factors multiplied to scores + before applying NMS. Default to None. + return_inds (bool, optional): Whether return the indices of kept + bboxes. Default to False. + + Returns: + tuple: (bboxes, labels, indices (optional)), tensors of shape (k, 5), + (k), and (k). Labels are 0-based. + """ + num_classes = multi_scores.size(1) - 1 + # exclude background category + if multi_bboxes.shape[1] > 4: + bboxes = multi_bboxes.view(multi_scores.size(0), -1, 4) + else: + bboxes = multi_bboxes[:, None].expand( + multi_scores.size(0), num_classes, 4) + + scores = multi_scores[:, :-1] + + labels = torch.arange(num_classes, dtype=torch.long) + labels = labels.view(1, -1).expand_as(scores) + + bboxes = bboxes.reshape(-1, 4) + scores = scores.reshape(-1) + labels = labels.reshape(-1) + + if not torch.onnx.is_in_onnx_export(): + # NonZero not supported in TensorRT + # remove low scoring boxes + valid_mask = scores > score_thr + # multiply score_factor after threshold to preserve more bboxes, improve + # mAP by 1% for YOLOv3 + if score_factors is not None: + # expand the shape to match original shape of score + score_factors = score_factors.view(-1, 1).expand( + multi_scores.size(0), num_classes) + score_factors = score_factors.reshape(-1) + scores = scores * score_factors + + if not torch.onnx.is_in_onnx_export(): + # NonZero not supported in TensorRT + inds = valid_mask.nonzero(as_tuple=False).squeeze(1) + bboxes, scores, labels = bboxes[inds], scores[inds], labels[inds] + else: + # TensorRT NMS plugin has invalid output filled with -1 + # add dummy data to make detection output correct. + bboxes = torch.cat([bboxes, bboxes.new_zeros(1, 4)], dim=0) + scores = torch.cat([scores, scores.new_zeros(1)], dim=0) + labels = torch.cat([labels, labels.new_zeros(1)], dim=0) + + if bboxes.numel() == 0: + if torch.onnx.is_in_onnx_export(): + raise RuntimeError('[ONNX Error] Can not record NMS ' + 'as it has not been executed this time') + if return_inds: + return bboxes, labels, inds + else: + return bboxes, labels + + dets, keep = batched_nms(bboxes, scores, labels, nms_cfg) + + if max_num > 0: + dets = dets[:max_num] + keep = keep[:max_num] + + if return_inds: + return dets, labels[keep], keep + else: + return dets, labels[keep] + + +def fast_nms(multi_bboxes, + multi_scores, + multi_coeffs, + score_thr, + iou_thr, + top_k, + max_num=-1): + """Fast NMS in `YOLACT `_. + + Fast NMS allows already-removed detections to suppress other detections so + that every instance can be decided to be kept or discarded in parallel, + which is not possible in traditional NMS. This relaxation allows us to + implement Fast NMS entirely in standard GPU-accelerated matrix operations. + + Args: + multi_bboxes (Tensor): shape (n, #class*4) or (n, 4) + multi_scores (Tensor): shape (n, #class+1), where the last column + contains scores of the background class, but this will be ignored. + multi_coeffs (Tensor): shape (n, #class*coeffs_dim). + score_thr (float): bbox threshold, bboxes with scores lower than it + will not be considered. + iou_thr (float): IoU threshold to be considered as conflicted. + top_k (int): if there are more than top_k bboxes before NMS, + only top top_k will be kept. + max_num (int): if there are more than max_num bboxes after NMS, + only top max_num will be kept. If -1, keep all the bboxes. + Default: -1. + + Returns: + tuple: (bboxes, labels, coefficients), tensors of shape (k, 5), (k, 1), + and (k, coeffs_dim). Labels are 0-based. + """ + + scores = multi_scores[:, :-1].t() # [#class, n] + scores, idx = scores.sort(1, descending=True) + + idx = idx[:, :top_k].contiguous() + scores = scores[:, :top_k] # [#class, topk] + num_classes, num_dets = idx.size() + boxes = multi_bboxes[idx.view(-1), :].view(num_classes, num_dets, 4) + coeffs = multi_coeffs[idx.view(-1), :].view(num_classes, num_dets, -1) + + iou = bbox_overlaps(boxes, boxes) # [#class, topk, topk] + iou.triu_(diagonal=1) + iou_max, _ = iou.max(dim=1) + + # Now just filter out the ones higher than the threshold + keep = iou_max <= iou_thr + + # Second thresholding introduces 0.2 mAP gain at negligible time cost + keep *= scores > score_thr + + # Assign each kept detection to its corresponding class + classes = torch.arange( + num_classes, device=boxes.device)[:, None].expand_as(keep) + classes = classes[keep] + + boxes = boxes[keep] + coeffs = coeffs[keep] + scores = scores[keep] + + # Only keep the top max_num highest scores across all classes + scores, idx = scores.sort(0, descending=True) + if max_num > 0: + idx = idx[:max_num] + scores = scores[:max_num] + + classes = classes[idx] + boxes = boxes[idx] + coeffs = coeffs[idx] + + cls_dets = torch.cat([boxes, scores[:, None]], dim=1) + return cls_dets, classes, coeffs diff --git a/detection/mmdet/core/post_processing/merge_augs.py b/detection/mmdet/core/post_processing/merge_augs.py new file mode 100644 index 0000000..dbcf79d --- /dev/null +++ b/detection/mmdet/core/post_processing/merge_augs.py @@ -0,0 +1,150 @@ +import copy +import warnings + +import numpy as np +import torch +from mmcv import ConfigDict +from mmcv.ops import nms + +from ..bbox import bbox_mapping_back + + +def merge_aug_proposals(aug_proposals, img_metas, cfg): + """Merge augmented proposals (multiscale, flip, etc.) + + Args: + aug_proposals (list[Tensor]): proposals from different testing + schemes, shape (n, 5). Note that they are not rescaled to the + original image size. + + img_metas (list[dict]): list of image info dict where each dict has: + 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmdet/datasets/pipelines/formatting.py:Collect`. + + cfg (dict): rpn test config. + + Returns: + Tensor: shape (n, 4), proposals corresponding to original image scale. + """ + + cfg = copy.deepcopy(cfg) + + # deprecate arguments warning + if 'nms' not in cfg or 'max_num' in cfg or 'nms_thr' in cfg: + warnings.warn( + 'In rpn_proposal or test_cfg, ' + 'nms_thr has been moved to a dict named nms as ' + 'iou_threshold, max_num has been renamed as max_per_img, ' + 'name of original arguments and the way to specify ' + 'iou_threshold of NMS will be deprecated.') + if 'nms' not in cfg: + cfg.nms = ConfigDict(dict(type='nms', iou_threshold=cfg.nms_thr)) + if 'max_num' in cfg: + if 'max_per_img' in cfg: + assert cfg.max_num == cfg.max_per_img, f'You set max_num and ' \ + f'max_per_img at the same time, but get {cfg.max_num} ' \ + f'and {cfg.max_per_img} respectively' \ + f'Please delete max_num which will be deprecated.' + else: + cfg.max_per_img = cfg.max_num + if 'nms_thr' in cfg: + assert cfg.nms.iou_threshold == cfg.nms_thr, f'You set ' \ + f'iou_threshold in nms and ' \ + f'nms_thr at the same time, but get ' \ + f'{cfg.nms.iou_threshold} and {cfg.nms_thr}' \ + f' respectively. Please delete the nms_thr ' \ + f'which will be deprecated.' + + recovered_proposals = [] + for proposals, img_info in zip(aug_proposals, img_metas): + img_shape = img_info['img_shape'] + scale_factor = img_info['scale_factor'] + flip = img_info['flip'] + flip_direction = img_info['flip_direction'] + _proposals = proposals.clone() + _proposals[:, :4] = bbox_mapping_back(_proposals[:, :4], img_shape, + scale_factor, flip, + flip_direction) + recovered_proposals.append(_proposals) + aug_proposals = torch.cat(recovered_proposals, dim=0) + merged_proposals, _ = nms(aug_proposals[:, :4].contiguous(), + aug_proposals[:, -1].contiguous(), + cfg.nms.iou_threshold) + scores = merged_proposals[:, 4] + _, order = scores.sort(0, descending=True) + num = min(cfg.max_per_img, merged_proposals.shape[0]) + order = order[:num] + merged_proposals = merged_proposals[order, :] + return merged_proposals + + +def merge_aug_bboxes(aug_bboxes, aug_scores, img_metas, rcnn_test_cfg): + """Merge augmented detection bboxes and scores. + + Args: + aug_bboxes (list[Tensor]): shape (n, 4*#class) + aug_scores (list[Tensor] or None): shape (n, #class) + img_shapes (list[Tensor]): shape (3, ). + rcnn_test_cfg (dict): rcnn test config. + + Returns: + tuple: (bboxes, scores) + """ + recovered_bboxes = [] + for bboxes, img_info in zip(aug_bboxes, img_metas): + img_shape = img_info[0]['img_shape'] + scale_factor = img_info[0]['scale_factor'] + flip = img_info[0]['flip'] + flip_direction = img_info[0]['flip_direction'] + bboxes = bbox_mapping_back(bboxes, img_shape, scale_factor, flip, + flip_direction) + recovered_bboxes.append(bboxes) + bboxes = torch.stack(recovered_bboxes).mean(dim=0) + if aug_scores is None: + return bboxes + else: + scores = torch.stack(aug_scores).mean(dim=0) + return bboxes, scores + + +def merge_aug_scores(aug_scores): + """Merge augmented bbox scores.""" + if isinstance(aug_scores[0], torch.Tensor): + return torch.mean(torch.stack(aug_scores), dim=0) + else: + return np.mean(aug_scores, axis=0) + + +def merge_aug_masks(aug_masks, img_metas, rcnn_test_cfg, weights=None): + """Merge augmented mask prediction. + + Args: + aug_masks (list[ndarray]): shape (n, #class, h, w) + img_shapes (list[ndarray]): shape (3, ). + rcnn_test_cfg (dict): rcnn test config. + + Returns: + tuple: (bboxes, scores) + """ + recovered_masks = [] + for mask, img_info in zip(aug_masks, img_metas): + flip = img_info[0]['flip'] + flip_direction = img_info[0]['flip_direction'] + if flip: + if flip_direction == 'horizontal': + mask = mask[:, :, :, ::-1] + elif flip_direction == 'vertical': + mask = mask[:, :, ::-1, :] + else: + raise ValueError( + f"Invalid flipping direction '{flip_direction}'") + recovered_masks.append(mask) + + if weights is None: + merged_masks = np.mean(recovered_masks, axis=0) + else: + merged_masks = np.average( + np.array(recovered_masks), axis=0, weights=np.array(weights)) + return merged_masks diff --git a/detection/mmdet/core/utils/__init__.py b/detection/mmdet/core/utils/__init__.py new file mode 100644 index 0000000..5c51dac --- /dev/null +++ b/detection/mmdet/core/utils/__init__.py @@ -0,0 +1,7 @@ +from .dist_utils import DistOptimizerHook, allreduce_grads, reduce_mean +from .misc import mask2ndarray, multi_apply, unmap + +__all__ = [ + 'allreduce_grads', 'DistOptimizerHook', 'reduce_mean', 'multi_apply', + 'unmap', 'mask2ndarray' +] diff --git a/detection/mmdet/core/utils/dist_utils.py b/detection/mmdet/core/utils/dist_utils.py new file mode 100644 index 0000000..5fe7775 --- /dev/null +++ b/detection/mmdet/core/utils/dist_utils.py @@ -0,0 +1,69 @@ +import warnings +from collections import OrderedDict + +import torch.distributed as dist +from mmcv.runner import OptimizerHook +from torch._utils import (_flatten_dense_tensors, _take_tensors, + _unflatten_dense_tensors) + + +def _allreduce_coalesced(tensors, world_size, bucket_size_mb=-1): + if bucket_size_mb > 0: + bucket_size_bytes = bucket_size_mb * 1024 * 1024 + buckets = _take_tensors(tensors, bucket_size_bytes) + else: + buckets = OrderedDict() + for tensor in tensors: + tp = tensor.type() + if tp not in buckets: + buckets[tp] = [] + buckets[tp].append(tensor) + buckets = buckets.values() + + for bucket in buckets: + flat_tensors = _flatten_dense_tensors(bucket) + dist.all_reduce(flat_tensors) + flat_tensors.div_(world_size) + for tensor, synced in zip( + bucket, _unflatten_dense_tensors(flat_tensors, bucket)): + tensor.copy_(synced) + + +def allreduce_grads(params, coalesce=True, bucket_size_mb=-1): + """Allreduce gradients. + + Args: + params (list[torch.Parameters]): List of parameters of a model + coalesce (bool, optional): Whether allreduce parameters as a whole. + Defaults to True. + bucket_size_mb (int, optional): Size of bucket, the unit is MB. + Defaults to -1. + """ + grads = [ + param.grad.data for param in params + if param.requires_grad and param.grad is not None + ] + world_size = dist.get_world_size() + if coalesce: + _allreduce_coalesced(grads, world_size, bucket_size_mb) + else: + for tensor in grads: + dist.all_reduce(tensor.div_(world_size)) + + +class DistOptimizerHook(OptimizerHook): + """Deprecated optimizer hook for distributed training.""" + + def __init__(self, *args, **kwargs): + warnings.warn('"DistOptimizerHook" is deprecated, please switch to' + '"mmcv.runner.OptimizerHook".') + super().__init__(*args, **kwargs) + + +def reduce_mean(tensor): + """"Obtain the mean of tensor on different GPUs.""" + if not (dist.is_available() and dist.is_initialized()): + return tensor + tensor = tensor.clone() + dist.all_reduce(tensor.div_(dist.get_world_size()), op=dist.ReduceOp.SUM) + return tensor diff --git a/detection/mmdet/core/utils/misc.py b/detection/mmdet/core/utils/misc.py new file mode 100644 index 0000000..3e22c7b --- /dev/null +++ b/detection/mmdet/core/utils/misc.py @@ -0,0 +1,61 @@ +from functools import partial + +import numpy as np +import torch +from six.moves import map, zip + +from ..mask.structures import BitmapMasks, PolygonMasks + + +def multi_apply(func, *args, **kwargs): + """Apply function to a list of arguments. + + Note: + This function applies the ``func`` to multiple inputs and + map the multiple outputs of the ``func`` into different + list. Each list contains the same type of outputs corresponding + to different inputs. + + Args: + func (Function): A function that will be applied to a list of + arguments + + Returns: + tuple(list): A tuple containing multiple list, each list contains \ + a kind of returned results by the function + """ + pfunc = partial(func, **kwargs) if kwargs else func + map_results = map(pfunc, *args) + return tuple(map(list, zip(*map_results))) + + +def unmap(data, count, inds, fill=0): + """Unmap a subset of item (data) back to the original set of items (of size + count)""" + if data.dim() == 1: + ret = data.new_full((count, ), fill) + ret[inds.type(torch.bool)] = data + else: + new_size = (count, ) + data.size()[1:] + ret = data.new_full(new_size, fill) + ret[inds.type(torch.bool), :] = data + return ret + + +def mask2ndarray(mask): + """Convert Mask to ndarray.. + + Args: + mask (:obj:`BitmapMasks` or :obj:`PolygonMasks` or + torch.Tensor or np.ndarray): The mask to be converted. + + Returns: + np.ndarray: Ndarray mask of shape (n, h, w) that has been converted + """ + if isinstance(mask, (BitmapMasks, PolygonMasks)): + mask = mask.to_ndarray() + elif isinstance(mask, torch.Tensor): + mask = mask.detach().cpu().numpy() + elif not isinstance(mask, np.ndarray): + raise TypeError(f'Unsupported {type(mask)} data type') + return mask diff --git a/detection/mmdet/core/visualization/__init__.py b/detection/mmdet/core/visualization/__init__.py new file mode 100644 index 0000000..4ff995c --- /dev/null +++ b/detection/mmdet/core/visualization/__init__.py @@ -0,0 +1,4 @@ +from .image import (color_val_matplotlib, imshow_det_bboxes, + imshow_gt_det_bboxes) + +__all__ = ['imshow_det_bboxes', 'imshow_gt_det_bboxes', 'color_val_matplotlib'] diff --git a/detection/mmdet/core/visualization/image.py b/detection/mmdet/core/visualization/image.py new file mode 100644 index 0000000..5a14838 --- /dev/null +++ b/detection/mmdet/core/visualization/image.py @@ -0,0 +1,303 @@ +import matplotlib.pyplot as plt +import mmcv +import numpy as np +import pycocotools.mask as mask_util +from matplotlib.collections import PatchCollection +from matplotlib.patches import Polygon + +from ..utils import mask2ndarray + +EPS = 1e-2 + + +def color_val_matplotlib(color): + """Convert various input in BGR order to normalized RGB matplotlib color + tuples, + + Args: + color (:obj:`Color`/str/tuple/int/ndarray): Color inputs + + Returns: + tuple[float]: A tuple of 3 normalized floats indicating RGB channels. + """ + color = mmcv.color_val(color) + color = [color / 255 for color in color[::-1]] + return tuple(color) + + +def imshow_det_bboxes(img, + bboxes, + labels, + segms=None, + class_names=None, + score_thr=0, + bbox_color='green', + text_color='green', + mask_color=None, + thickness=2, + font_size=13, + win_name='', + show=True, + wait_time=0, + out_file=None): + """Draw bboxes and class labels (with scores) on an image. + + Args: + img (str or ndarray): The image to be displayed. + bboxes (ndarray): Bounding boxes (with scores), shaped (n, 4) or + (n, 5). + labels (ndarray): Labels of bboxes. + segms (ndarray or None): Masks, shaped (n,h,w) or None + class_names (list[str]): Names of each classes. + score_thr (float): Minimum score of bboxes to be shown. Default: 0 + bbox_color (str or tuple(int) or :obj:`Color`):Color of bbox lines. + The tuple of color should be in BGR order. Default: 'green' + text_color (str or tuple(int) or :obj:`Color`):Color of texts. + The tuple of color should be in BGR order. Default: 'green' + mask_color (str or tuple(int) or :obj:`Color`, optional): + Color of masks. The tuple of color should be in BGR order. + Default: None + thickness (int): Thickness of lines. Default: 2 + font_size (int): Font size of texts. Default: 13 + show (bool): Whether to show the image. Default: True + win_name (str): The window name. Default: '' + wait_time (float): Value of waitKey param. Default: 0. + out_file (str, optional): The filename to write the image. + Default: None + + Returns: + ndarray: The image with bboxes drawn on it. + """ + assert bboxes.ndim == 2, \ + f' bboxes ndim should be 2, but its ndim is {bboxes.ndim}.' + assert labels.ndim == 1, \ + f' labels ndim should be 1, but its ndim is {labels.ndim}.' + assert bboxes.shape[0] == labels.shape[0], \ + 'bboxes.shape[0] and labels.shape[0] should have the same length.' + assert bboxes.shape[1] == 4 or bboxes.shape[1] == 5, \ + f' bboxes.shape[1] should be 4 or 5, but its {bboxes.shape[1]}.' + img = mmcv.imread(img).astype(np.uint8) + + if score_thr > 0: + assert bboxes.shape[1] == 5 + scores = bboxes[:, -1] + inds = scores > score_thr + bboxes = bboxes[inds, :] + labels = labels[inds] + if segms is not None: + segms = segms[inds, ...] + + mask_colors = [] + if labels.shape[0] > 0: + if mask_color is None: + # random color + np.random.seed(42) + mask_colors = [ + np.random.randint(0, 256, (1, 3), dtype=np.uint8) + for _ in range(max(labels) + 1) + ] + else: + # specify color + mask_colors = [ + np.array(mmcv.color_val(mask_color)[::-1], dtype=np.uint8) + ] * ( + max(labels) + 1) + + bbox_color = color_val_matplotlib(bbox_color) + text_color = color_val_matplotlib(text_color) + + img = mmcv.bgr2rgb(img) + width, height = img.shape[1], img.shape[0] + img = np.ascontiguousarray(img) + + fig = plt.figure(win_name, frameon=False) + plt.title(win_name) + canvas = fig.canvas + dpi = fig.get_dpi() + # add a small EPS to avoid precision lost due to matplotlib's truncation + # (https://github.com/matplotlib/matplotlib/issues/15363) + fig.set_size_inches((width + EPS) / dpi, (height + EPS) / dpi) + + # remove white edges by set subplot margin + plt.subplots_adjust(left=0, right=1, bottom=0, top=1) + ax = plt.gca() + ax.axis('off') + + polygons = [] + color = [] + for i, (bbox, label) in enumerate(zip(bboxes, labels)): + bbox_int = bbox.astype(np.int32) + poly = [[bbox_int[0], bbox_int[1]], [bbox_int[0], bbox_int[3]], + [bbox_int[2], bbox_int[3]], [bbox_int[2], bbox_int[1]]] + np_poly = np.array(poly).reshape((4, 2)) + polygons.append(Polygon(np_poly)) + color.append(bbox_color) + label_text = class_names[ + label] if class_names is not None else f'class {label}' + if len(bbox) > 4: + label_text += f'|{bbox[-1]:.02f}' + ax.text( + bbox_int[0], + bbox_int[1], + f'{label_text}', + bbox={ + 'facecolor': 'black', + 'alpha': 0.8, + 'pad': 0.7, + 'edgecolor': 'none' + }, + color=text_color, + fontsize=font_size, + verticalalignment='top', + horizontalalignment='left') + if segms is not None: + color_mask = mask_colors[labels[i]] + mask = segms[i].astype(bool) + img[mask] = img[mask] * 0.5 + color_mask * 0.5 + + plt.imshow(img) + + p = PatchCollection( + polygons, facecolor='none', edgecolors=color, linewidths=thickness) + ax.add_collection(p) + + stream, _ = canvas.print_to_buffer() + buffer = np.frombuffer(stream, dtype='uint8') + img_rgba = buffer.reshape(height, width, 4) + rgb, alpha = np.split(img_rgba, [3], axis=2) + img = rgb.astype('uint8') + img = mmcv.rgb2bgr(img) + + if show: + # We do not use cv2 for display because in some cases, opencv will + # conflict with Qt, it will output a warning: Current thread + # is not the object's thread. You can refer to + # https://github.com/opencv/opencv-python/issues/46 for details + if wait_time == 0: + plt.show() + else: + plt.show(block=False) + plt.pause(wait_time) + if out_file is not None: + mmcv.imwrite(img, out_file) + + plt.close() + + return img + + +def imshow_gt_det_bboxes(img, + annotation, + result, + class_names=None, + score_thr=0, + gt_bbox_color=(255, 102, 61), + gt_text_color=(255, 102, 61), + gt_mask_color=(255, 102, 61), + det_bbox_color=(72, 101, 241), + det_text_color=(72, 101, 241), + det_mask_color=(72, 101, 241), + thickness=2, + font_size=13, + win_name='', + show=True, + wait_time=0, + out_file=None): + """General visualization GT and result function. + + Args: + img (str or ndarray): The image to be displayed.) + annotation (dict): Ground truth annotations where contain keys of + 'gt_bboxes' and 'gt_labels' or 'gt_masks' + result (tuple[list] or list): The detection result, can be either + (bbox, segm) or just bbox. + class_names (list[str]): Names of each classes. + score_thr (float): Minimum score of bboxes to be shown. Default: 0 + gt_bbox_color (str or tuple(int) or :obj:`Color`):Color of bbox lines. + The tuple of color should be in BGR order. Default: (255, 102, 61) + gt_text_color (str or tuple(int) or :obj:`Color`):Color of texts. + The tuple of color should be in BGR order. Default: (255, 102, 61) + gt_mask_color (str or tuple(int) or :obj:`Color`, optional): + Color of masks. The tuple of color should be in BGR order. + Default: (255, 102, 61) + det_bbox_color (str or tuple(int) or :obj:`Color`):Color of bbox lines. + The tuple of color should be in BGR order. Default: (72, 101, 241) + det_text_color (str or tuple(int) or :obj:`Color`):Color of texts. + The tuple of color should be in BGR order. Default: (72, 101, 241) + det_mask_color (str or tuple(int) or :obj:`Color`, optional): + Color of masks. The tuple of color should be in BGR order. + Default: (72, 101, 241) + thickness (int): Thickness of lines. Default: 2 + font_size (int): Font size of texts. Default: 13 + win_name (str): The window name. Default: '' + show (bool): Whether to show the image. Default: True + wait_time (float): Value of waitKey param. Default: 0. + out_file (str, optional): The filename to write the image. + Default: None + + Returns: + ndarray: The image with bboxes or masks drawn on it. + """ + assert 'gt_bboxes' in annotation + assert 'gt_labels' in annotation + assert isinstance( + result, + (tuple, list)), f'Expected tuple or list, but get {type(result)}' + + gt_masks = annotation.get('gt_masks', None) + if gt_masks is not None: + gt_masks = mask2ndarray(gt_masks) + + img = mmcv.imread(img) + + img = imshow_det_bboxes( + img, + annotation['gt_bboxes'], + annotation['gt_labels'], + gt_masks, + class_names=class_names, + bbox_color=gt_bbox_color, + text_color=gt_text_color, + mask_color=gt_mask_color, + thickness=thickness, + font_size=font_size, + win_name=win_name, + show=False) + + if isinstance(result, tuple): + bbox_result, segm_result = result + if isinstance(segm_result, tuple): + segm_result = segm_result[0] # ms rcnn + else: + bbox_result, segm_result = result, None + + bboxes = np.vstack(bbox_result) + labels = [ + np.full(bbox.shape[0], i, dtype=np.int32) + for i, bbox in enumerate(bbox_result) + ] + labels = np.concatenate(labels) + + segms = None + if segm_result is not None and len(labels) > 0: # non empty + segms = mmcv.concat_list(segm_result) + segms = mask_util.decode(segms) + segms = segms.transpose(2, 0, 1) + + img = imshow_det_bboxes( + img, + bboxes, + labels, + segms=segms, + class_names=class_names, + score_thr=score_thr, + bbox_color=det_bbox_color, + text_color=det_text_color, + mask_color=det_mask_color, + thickness=thickness, + font_size=font_size, + win_name=win_name, + show=show, + wait_time=wait_time, + out_file=out_file) + return img diff --git a/detection/mmdet/datasets/__init__.py b/detection/mmdet/datasets/__init__.py new file mode 100644 index 0000000..9b18b30 --- /dev/null +++ b/detection/mmdet/datasets/__init__.py @@ -0,0 +1,24 @@ +from .builder import DATASETS, PIPELINES, build_dataloader, build_dataset +from .cityscapes import CityscapesDataset +from .coco import CocoDataset +from .custom import CustomDataset +from .dataset_wrappers import (ClassBalancedDataset, ConcatDataset, + RepeatDataset) +from .deepfashion import DeepFashionDataset +from .lvis import LVISDataset, LVISV1Dataset, LVISV05Dataset +from .samplers import DistributedGroupSampler, DistributedSampler, GroupSampler +from .utils import (NumClassCheckHook, get_loading_pipeline, + replace_ImageToTensor) +from .voc import VOCDataset +from .wider_face import WIDERFaceDataset +from .xml_style import XMLDataset + +__all__ = [ + 'CustomDataset', 'XMLDataset', 'CocoDataset', 'DeepFashionDataset', + 'VOCDataset', 'CityscapesDataset', 'LVISDataset', 'LVISV05Dataset', + 'LVISV1Dataset', 'GroupSampler', 'DistributedGroupSampler', + 'DistributedSampler', 'build_dataloader', 'ConcatDataset', 'RepeatDataset', + 'ClassBalancedDataset', 'WIDERFaceDataset', 'DATASETS', 'PIPELINES', + 'build_dataset', 'replace_ImageToTensor', 'get_loading_pipeline', + 'NumClassCheckHook' +] diff --git a/detection/mmdet/datasets/builder.py b/detection/mmdet/datasets/builder.py new file mode 100644 index 0000000..c9466a5 --- /dev/null +++ b/detection/mmdet/datasets/builder.py @@ -0,0 +1,143 @@ +import copy +import platform +import random +from functools import partial + +import numpy as np +from mmcv.parallel import collate +from mmcv.runner import get_dist_info +from mmcv.utils import Registry, build_from_cfg +from torch.utils.data import DataLoader + +from .samplers import DistributedGroupSampler, DistributedSampler, GroupSampler + +if platform.system() != 'Windows': + # https://github.com/pytorch/pytorch/issues/973 + import resource + rlimit = resource.getrlimit(resource.RLIMIT_NOFILE) + hard_limit = rlimit[1] + soft_limit = min(4096, hard_limit) + resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, hard_limit)) + +DATASETS = Registry('dataset') +PIPELINES = Registry('pipeline') + + +def _concat_dataset(cfg, default_args=None): + from .dataset_wrappers import ConcatDataset + ann_files = cfg['ann_file'] + img_prefixes = cfg.get('img_prefix', None) + seg_prefixes = cfg.get('seg_prefix', None) + proposal_files = cfg.get('proposal_file', None) + separate_eval = cfg.get('separate_eval', True) + + datasets = [] + num_dset = len(ann_files) + for i in range(num_dset): + data_cfg = copy.deepcopy(cfg) + # pop 'separate_eval' since it is not a valid key for common datasets. + if 'separate_eval' in data_cfg: + data_cfg.pop('separate_eval') + data_cfg['ann_file'] = ann_files[i] + if isinstance(img_prefixes, (list, tuple)): + data_cfg['img_prefix'] = img_prefixes[i] + if isinstance(seg_prefixes, (list, tuple)): + data_cfg['seg_prefix'] = seg_prefixes[i] + if isinstance(proposal_files, (list, tuple)): + data_cfg['proposal_file'] = proposal_files[i] + datasets.append(build_dataset(data_cfg, default_args)) + + return ConcatDataset(datasets, separate_eval) + + +def build_dataset(cfg, default_args=None): + from .dataset_wrappers import (ConcatDataset, RepeatDataset, + ClassBalancedDataset) + if isinstance(cfg, (list, tuple)): + dataset = ConcatDataset([build_dataset(c, default_args) for c in cfg]) + elif cfg['type'] == 'ConcatDataset': + dataset = ConcatDataset( + [build_dataset(c, default_args) for c in cfg['datasets']], + cfg.get('separate_eval', True)) + elif cfg['type'] == 'RepeatDataset': + dataset = RepeatDataset( + build_dataset(cfg['dataset'], default_args), cfg['times']) + elif cfg['type'] == 'ClassBalancedDataset': + dataset = ClassBalancedDataset( + build_dataset(cfg['dataset'], default_args), cfg['oversample_thr']) + elif isinstance(cfg.get('ann_file'), (list, tuple)): + dataset = _concat_dataset(cfg, default_args) + else: + dataset = build_from_cfg(cfg, DATASETS, default_args) + + return dataset + + +def build_dataloader(dataset, + samples_per_gpu, + workers_per_gpu, + num_gpus=1, + dist=True, + shuffle=True, + seed=None, + **kwargs): + """Build PyTorch DataLoader. + + In distributed training, each GPU/process has a dataloader. + In non-distributed training, there is only one dataloader for all GPUs. + + Args: + dataset (Dataset): A PyTorch dataset. + samples_per_gpu (int): Number of training samples on each GPU, i.e., + batch size of each GPU. + workers_per_gpu (int): How many subprocesses to use for data loading + for each GPU. + num_gpus (int): Number of GPUs. Only used in non-distributed training. + dist (bool): Distributed training/test or not. Default: True. + shuffle (bool): Whether to shuffle the data at every epoch. + Default: True. + kwargs: any keyword argument to be used to initialize DataLoader + + Returns: + DataLoader: A PyTorch dataloader. + """ + rank, world_size = get_dist_info() + if dist: + # DistributedGroupSampler will definitely shuffle the data to satisfy + # that images on each GPU are in the same group + if shuffle: + sampler = DistributedGroupSampler( + dataset, samples_per_gpu, world_size, rank, seed=seed) + else: + sampler = DistributedSampler( + dataset, world_size, rank, shuffle=False, seed=seed) + batch_size = samples_per_gpu + num_workers = workers_per_gpu + else: + sampler = GroupSampler(dataset, samples_per_gpu) if shuffle else None + batch_size = num_gpus * samples_per_gpu + num_workers = num_gpus * workers_per_gpu + + init_fn = partial( + worker_init_fn, num_workers=num_workers, rank=rank, + seed=seed) if seed is not None else None + + data_loader = DataLoader( + dataset, + batch_size=batch_size, + sampler=sampler, + num_workers=num_workers, + collate_fn=partial(collate, samples_per_gpu=samples_per_gpu), + pin_memory=False, + worker_init_fn=init_fn, + **kwargs) + + return data_loader + + +def worker_init_fn(worker_id, num_workers, rank, seed): + # The seed of each worker equals to + # num_worker * rank + worker_id + user_seed + worker_seed = num_workers * rank + worker_id + seed + np.random.seed(worker_seed) + random.seed(worker_seed) diff --git a/detection/mmdet/datasets/cityscapes.py b/detection/mmdet/datasets/cityscapes.py new file mode 100644 index 0000000..71eead8 --- /dev/null +++ b/detection/mmdet/datasets/cityscapes.py @@ -0,0 +1,334 @@ +# Modified from https://github.com/facebookresearch/detectron2/blob/master/detectron2/data/datasets/cityscapes.py # noqa +# and https://github.com/mcordts/cityscapesScripts/blob/master/cityscapesscripts/evaluation/evalInstanceLevelSemanticLabeling.py # noqa + +import glob +import os +import os.path as osp +import tempfile +from collections import OrderedDict + +import mmcv +import numpy as np +import pycocotools.mask as maskUtils +from mmcv.utils import print_log + +from .builder import DATASETS +from .coco import CocoDataset + + +@DATASETS.register_module() +class CityscapesDataset(CocoDataset): + + CLASSES = ('person', 'rider', 'car', 'truck', 'bus', 'train', 'motorcycle', + 'bicycle') + + def _filter_imgs(self, min_size=32): + """Filter images too small or without ground truths.""" + valid_inds = [] + # obtain images that contain annotation + ids_with_ann = set(_['image_id'] for _ in self.coco.anns.values()) + # obtain images that contain annotations of the required categories + ids_in_cat = set() + for i, class_id in enumerate(self.cat_ids): + ids_in_cat |= set(self.coco.cat_img_map[class_id]) + # merge the image id sets of the two conditions and use the merged set + # to filter out images if self.filter_empty_gt=True + ids_in_cat &= ids_with_ann + + valid_img_ids = [] + for i, img_info in enumerate(self.data_infos): + img_id = img_info['id'] + ann_ids = self.coco.getAnnIds(imgIds=[img_id]) + ann_info = self.coco.loadAnns(ann_ids) + all_iscrowd = all([_['iscrowd'] for _ in ann_info]) + if self.filter_empty_gt and (self.img_ids[i] not in ids_in_cat + or all_iscrowd): + continue + if min(img_info['width'], img_info['height']) >= min_size: + valid_inds.append(i) + valid_img_ids.append(img_id) + self.img_ids = valid_img_ids + return valid_inds + + def _parse_ann_info(self, img_info, ann_info): + """Parse bbox and mask annotation. + + Args: + img_info (dict): Image info of an image. + ann_info (list[dict]): Annotation info of an image. + + Returns: + dict: A dict containing the following keys: bboxes, \ + bboxes_ignore, labels, masks, seg_map. \ + "masks" are already decoded into binary masks. + """ + gt_bboxes = [] + gt_labels = [] + gt_bboxes_ignore = [] + gt_masks_ann = [] + + for i, ann in enumerate(ann_info): + if ann.get('ignore', False): + continue + x1, y1, w, h = ann['bbox'] + if ann['area'] <= 0 or w < 1 or h < 1: + continue + if ann['category_id'] not in self.cat_ids: + continue + bbox = [x1, y1, x1 + w, y1 + h] + if ann.get('iscrowd', False): + gt_bboxes_ignore.append(bbox) + else: + gt_bboxes.append(bbox) + gt_labels.append(self.cat2label[ann['category_id']]) + gt_masks_ann.append(ann['segmentation']) + + if gt_bboxes: + gt_bboxes = np.array(gt_bboxes, dtype=np.float32) + gt_labels = np.array(gt_labels, dtype=np.int64) + else: + gt_bboxes = np.zeros((0, 4), dtype=np.float32) + gt_labels = np.array([], dtype=np.int64) + + if gt_bboxes_ignore: + gt_bboxes_ignore = np.array(gt_bboxes_ignore, dtype=np.float32) + else: + gt_bboxes_ignore = np.zeros((0, 4), dtype=np.float32) + + ann = dict( + bboxes=gt_bboxes, + labels=gt_labels, + bboxes_ignore=gt_bboxes_ignore, + masks=gt_masks_ann, + seg_map=img_info['segm_file']) + + return ann + + def results2txt(self, results, outfile_prefix): + """Dump the detection results to a txt file. + + Args: + results (list[list | tuple]): Testing results of the + dataset. + outfile_prefix (str): The filename prefix of the json files. + If the prefix is "somepath/xxx", + the txt files will be named "somepath/xxx.txt". + + Returns: + list[str]: Result txt files which contains corresponding \ + instance segmentation images. + """ + try: + import cityscapesscripts.helpers.labels as CSLabels + except ImportError: + raise ImportError('Please run "pip install citscapesscripts" to ' + 'install cityscapesscripts first.') + result_files = [] + os.makedirs(outfile_prefix, exist_ok=True) + prog_bar = mmcv.ProgressBar(len(self)) + for idx in range(len(self)): + result = results[idx] + filename = self.data_infos[idx]['filename'] + basename = osp.splitext(osp.basename(filename))[0] + pred_txt = osp.join(outfile_prefix, basename + '_pred.txt') + + bbox_result, segm_result = result + bboxes = np.vstack(bbox_result) + # segm results + if isinstance(segm_result, tuple): + # Some detectors use different scores for bbox and mask, + # like Mask Scoring R-CNN. Score of segm will be used instead + # of bbox score. + segms = mmcv.concat_list(segm_result[0]) + mask_score = segm_result[1] + else: + # use bbox score for mask score + segms = mmcv.concat_list(segm_result) + mask_score = [bbox[-1] for bbox in bboxes] + labels = [ + np.full(bbox.shape[0], i, dtype=np.int32) + for i, bbox in enumerate(bbox_result) + ] + labels = np.concatenate(labels) + + assert len(bboxes) == len(segms) == len(labels) + num_instances = len(bboxes) + prog_bar.update() + with open(pred_txt, 'w') as fout: + for i in range(num_instances): + pred_class = labels[i] + classes = self.CLASSES[pred_class] + class_id = CSLabels.name2label[classes].id + score = mask_score[i] + mask = maskUtils.decode(segms[i]).astype(np.uint8) + png_filename = osp.join(outfile_prefix, + basename + f'_{i}_{classes}.png') + mmcv.imwrite(mask, png_filename) + fout.write(f'{osp.basename(png_filename)} {class_id} ' + f'{score}\n') + result_files.append(pred_txt) + + return result_files + + def format_results(self, results, txtfile_prefix=None): + """Format the results to txt (standard format for Cityscapes + evaluation). + + Args: + results (list): Testing results of the dataset. + txtfile_prefix (str | None): The prefix of txt files. It includes + the file path and the prefix of filename, e.g., "a/b/prefix". + If not specified, a temp file will be created. Default: None. + + Returns: + tuple: (result_files, tmp_dir), result_files is a dict containing \ + the json filepaths, tmp_dir is the temporal directory created \ + for saving txt/png files when txtfile_prefix is not specified. + """ + assert isinstance(results, list), 'results must be a list' + assert len(results) == len(self), ( + 'The length of results is not equal to the dataset len: {} != {}'. + format(len(results), len(self))) + + assert isinstance(results, list), 'results must be a list' + assert len(results) == len(self), ( + 'The length of results is not equal to the dataset len: {} != {}'. + format(len(results), len(self))) + + if txtfile_prefix is None: + tmp_dir = tempfile.TemporaryDirectory() + txtfile_prefix = osp.join(tmp_dir.name, 'results') + else: + tmp_dir = None + result_files = self.results2txt(results, txtfile_prefix) + + return result_files, tmp_dir + + def evaluate(self, + results, + metric='bbox', + logger=None, + outfile_prefix=None, + classwise=False, + proposal_nums=(100, 300, 1000), + iou_thrs=np.arange(0.5, 0.96, 0.05)): + """Evaluation in Cityscapes/COCO protocol. + + Args: + results (list[list | tuple]): Testing results of the dataset. + metric (str | list[str]): Metrics to be evaluated. Options are + 'bbox', 'segm', 'proposal', 'proposal_fast'. + logger (logging.Logger | str | None): Logger used for printing + related information during evaluation. Default: None. + outfile_prefix (str | None): The prefix of output file. It includes + the file path and the prefix of filename, e.g., "a/b/prefix". + If results are evaluated with COCO protocol, it would be the + prefix of output json file. For example, the metric is 'bbox' + and 'segm', then json files would be "a/b/prefix.bbox.json" and + "a/b/prefix.segm.json". + If results are evaluated with cityscapes protocol, it would be + the prefix of output txt/png files. The output files would be + png images under folder "a/b/prefix/xxx/" and the file name of + images would be written into a txt file + "a/b/prefix/xxx_pred.txt", where "xxx" is the video name of + cityscapes. If not specified, a temp file will be created. + Default: None. + classwise (bool): Whether to evaluating the AP for each class. + proposal_nums (Sequence[int]): Proposal number used for evaluating + recalls, such as recall@100, recall@1000. + Default: (100, 300, 1000). + iou_thrs (Sequence[float]): IoU threshold used for evaluating + recalls. If set to a list, the average recall of all IoUs will + also be computed. Default: 0.5. + + Returns: + dict[str, float]: COCO style evaluation metric or cityscapes mAP \ + and AP@50. + """ + eval_results = dict() + + metrics = metric.copy() if isinstance(metric, list) else [metric] + + if 'cityscapes' in metrics: + eval_results.update( + self._evaluate_cityscapes(results, outfile_prefix, logger)) + metrics.remove('cityscapes') + + # left metrics are all coco metric + if len(metrics) > 0: + # create CocoDataset with CityscapesDataset annotation + self_coco = CocoDataset(self.ann_file, self.pipeline.transforms, + None, self.data_root, self.img_prefix, + self.seg_prefix, self.proposal_file, + self.test_mode, self.filter_empty_gt) + # TODO: remove this in the future + # reload annotations of correct class + self_coco.CLASSES = self.CLASSES + self_coco.data_infos = self_coco.load_annotations(self.ann_file) + eval_results.update( + self_coco.evaluate(results, metrics, logger, outfile_prefix, + classwise, proposal_nums, iou_thrs)) + + return eval_results + + def _evaluate_cityscapes(self, results, txtfile_prefix, logger): + """Evaluation in Cityscapes protocol. + + Args: + results (list): Testing results of the dataset. + txtfile_prefix (str | None): The prefix of output txt file + logger (logging.Logger | str | None): Logger used for printing + related information during evaluation. Default: None. + + Returns: + dict[str: float]: Cityscapes evaluation results, contains 'mAP' \ + and 'AP@50'. + """ + + try: + import cityscapesscripts.evaluation.evalInstanceLevelSemanticLabeling as CSEval # noqa + except ImportError: + raise ImportError('Please run "pip install citscapesscripts" to ' + 'install cityscapesscripts first.') + msg = 'Evaluating in Cityscapes style' + if logger is None: + msg = '\n' + msg + print_log(msg, logger=logger) + + result_files, tmp_dir = self.format_results(results, txtfile_prefix) + + if tmp_dir is None: + result_dir = osp.join(txtfile_prefix, 'results') + else: + result_dir = osp.join(tmp_dir.name, 'results') + + eval_results = OrderedDict() + print_log(f'Evaluating results under {result_dir} ...', logger=logger) + + # set global states in cityscapes evaluation API + CSEval.args.cityscapesPath = os.path.join(self.img_prefix, '../..') + CSEval.args.predictionPath = os.path.abspath(result_dir) + CSEval.args.predictionWalk = None + CSEval.args.JSONOutput = False + CSEval.args.colorized = False + CSEval.args.gtInstancesFile = os.path.join(result_dir, + 'gtInstances.json') + CSEval.args.groundTruthSearch = os.path.join( + self.img_prefix.replace('leftImg8bit', 'gtFine'), + '*/*_gtFine_instanceIds.png') + + groundTruthImgList = glob.glob(CSEval.args.groundTruthSearch) + assert len(groundTruthImgList), 'Cannot find ground truth images' \ + f' in {CSEval.args.groundTruthSearch}.' + predictionImgList = [] + for gt in groundTruthImgList: + predictionImgList.append(CSEval.getPrediction(gt, CSEval.args)) + CSEval_results = CSEval.evaluateImgLists(predictionImgList, + groundTruthImgList, + CSEval.args)['averages'] + + eval_results['mAP'] = CSEval_results['allAp'] + eval_results['AP@50'] = CSEval_results['allAp50%'] + if tmp_dir is not None: + tmp_dir.cleanup() + return eval_results diff --git a/detection/mmdet/datasets/coco.py b/detection/mmdet/datasets/coco.py new file mode 100644 index 0000000..3a8e1bc --- /dev/null +++ b/detection/mmdet/datasets/coco.py @@ -0,0 +1,546 @@ +import itertools +import logging +import os.path as osp +import tempfile +from collections import OrderedDict + +import mmcv +import numpy as np +import pycocotools +from mmcv.utils import print_log +from pycocotools.coco import COCO +from pycocotools.cocoeval import COCOeval +from terminaltables import AsciiTable + +from mmdet.core import eval_recalls +from .builder import DATASETS +from .custom import CustomDataset + + +@DATASETS.register_module() +class CocoDataset(CustomDataset): + + CLASSES = ('person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', + 'train', 'truck', 'boat', 'traffic light', 'fire hydrant', + 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', + 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', + 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee', + 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', + 'baseball glove', 'skateboard', 'surfboard', 'tennis racket', + 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', + 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', + 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', + 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', + 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave', + 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', + 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush') + + def load_annotations(self, ann_file): + """Load annotation from COCO style annotation file. + + Args: + ann_file (str): Path of annotation file. + + Returns: + list[dict]: Annotation info from COCO api. + """ + if not getattr(pycocotools, '__version__', '0') >= '12.0.2': + raise AssertionError( + 'Incompatible version of pycocotools is installed. ' + 'Run pip uninstall pycocotools first. Then run pip ' + 'install mmpycocotools to install open-mmlab forked ' + 'pycocotools.') + + self.coco = COCO(ann_file) + self.cat_ids = self.coco.get_cat_ids(cat_names=self.CLASSES) + self.cat2label = {cat_id: i for i, cat_id in enumerate(self.cat_ids)} + self.img_ids = self.coco.get_img_ids() + data_infos = [] + total_ann_ids = [] + for i in self.img_ids: + info = self.coco.load_imgs([i])[0] + info['filename'] = info['file_name'] + data_infos.append(info) + ann_ids = self.coco.get_ann_ids(img_ids=[i]) + total_ann_ids.extend(ann_ids) + assert len(set(total_ann_ids)) == len( + total_ann_ids), f"Annotation ids in '{ann_file}' are not unique!" + return data_infos + + def get_ann_info(self, idx): + """Get COCO annotation by index. + + Args: + idx (int): Index of data. + + Returns: + dict: Annotation info of specified index. + """ + + img_id = self.data_infos[idx]['id'] + ann_ids = self.coco.get_ann_ids(img_ids=[img_id]) + ann_info = self.coco.load_anns(ann_ids) + return self._parse_ann_info(self.data_infos[idx], ann_info) + + def get_cat_ids(self, idx): + """Get COCO category ids by index. + + Args: + idx (int): Index of data. + + Returns: + list[int]: All categories in the image of specified index. + """ + + img_id = self.data_infos[idx]['id'] + ann_ids = self.coco.get_ann_ids(img_ids=[img_id]) + ann_info = self.coco.load_anns(ann_ids) + return [ann['category_id'] for ann in ann_info] + + def _filter_imgs(self, min_size=32): + """Filter images too small or without ground truths.""" + valid_inds = [] + # obtain images that contain annotation + ids_with_ann = set(_['image_id'] for _ in self.coco.anns.values()) + # obtain images that contain annotations of the required categories + ids_in_cat = set() + for i, class_id in enumerate(self.cat_ids): + ids_in_cat |= set(self.coco.cat_img_map[class_id]) + # merge the image id sets of the two conditions and use the merged set + # to filter out images if self.filter_empty_gt=True + ids_in_cat &= ids_with_ann + + valid_img_ids = [] + for i, img_info in enumerate(self.data_infos): + img_id = self.img_ids[i] + if self.filter_empty_gt and img_id not in ids_in_cat: + continue + if min(img_info['width'], img_info['height']) >= min_size: + valid_inds.append(i) + valid_img_ids.append(img_id) + self.img_ids = valid_img_ids + return valid_inds + + def _parse_ann_info(self, img_info, ann_info): + """Parse bbox and mask annotation. + + Args: + ann_info (list[dict]): Annotation info of an image. + with_mask (bool): Whether to parse mask annotations. + + Returns: + dict: A dict containing the following keys: bboxes, bboxes_ignore,\ + labels, masks, seg_map. "masks" are raw annotations and not \ + decoded into binary masks. + """ + gt_bboxes = [] + gt_labels = [] + gt_bboxes_ignore = [] + gt_masks_ann = [] + for i, ann in enumerate(ann_info): + if ann.get('ignore', False): + continue + x1, y1, w, h = ann['bbox'] + inter_w = max(0, min(x1 + w, img_info['width']) - max(x1, 0)) + inter_h = max(0, min(y1 + h, img_info['height']) - max(y1, 0)) + if inter_w * inter_h == 0: + continue + if ann['area'] <= 0 or w < 1 or h < 1: + continue + if ann['category_id'] not in self.cat_ids: + continue + bbox = [x1, y1, x1 + w, y1 + h] + if ann.get('iscrowd', False): + gt_bboxes_ignore.append(bbox) + else: + gt_bboxes.append(bbox) + gt_labels.append(self.cat2label[ann['category_id']]) + gt_masks_ann.append(ann.get('segmentation', None)) + + if gt_bboxes: + gt_bboxes = np.array(gt_bboxes, dtype=np.float32) + gt_labels = np.array(gt_labels, dtype=np.int64) + else: + gt_bboxes = np.zeros((0, 4), dtype=np.float32) + gt_labels = np.array([], dtype=np.int64) + + if gt_bboxes_ignore: + gt_bboxes_ignore = np.array(gt_bboxes_ignore, dtype=np.float32) + else: + gt_bboxes_ignore = np.zeros((0, 4), dtype=np.float32) + + seg_map = img_info['filename'].replace('jpg', 'png') + + ann = dict( + bboxes=gt_bboxes, + labels=gt_labels, + bboxes_ignore=gt_bboxes_ignore, + masks=gt_masks_ann, + seg_map=seg_map) + + return ann + + def xyxy2xywh(self, bbox): + """Convert ``xyxy`` style bounding boxes to ``xywh`` style for COCO + evaluation. + + Args: + bbox (numpy.ndarray): The bounding boxes, shape (4, ), in + ``xyxy`` order. + + Returns: + list[float]: The converted bounding boxes, in ``xywh`` order. + """ + + _bbox = bbox.tolist() + return [ + _bbox[0], + _bbox[1], + _bbox[2] - _bbox[0], + _bbox[3] - _bbox[1], + ] + + def _proposal2json(self, results): + """Convert proposal results to COCO json style.""" + json_results = [] + for idx in range(len(self)): + img_id = self.img_ids[idx] + bboxes = results[idx] + for i in range(bboxes.shape[0]): + data = dict() + data['image_id'] = img_id + data['bbox'] = self.xyxy2xywh(bboxes[i]) + data['score'] = float(bboxes[i][4]) + data['category_id'] = 1 + json_results.append(data) + return json_results + + def _det2json(self, results): + """Convert detection results to COCO json style.""" + json_results = [] + for idx in range(len(self)): + img_id = self.img_ids[idx] + result = results[idx] + for label in range(len(result)): + bboxes = result[label] + for i in range(bboxes.shape[0]): + data = dict() + data['image_id'] = img_id + data['bbox'] = self.xyxy2xywh(bboxes[i]) + data['score'] = float(bboxes[i][4]) + data['category_id'] = self.cat_ids[label] + json_results.append(data) + return json_results + + def _segm2json(self, results): + """Convert instance segmentation results to COCO json style.""" + bbox_json_results = [] + segm_json_results = [] + for idx in range(len(self)): + img_id = self.img_ids[idx] + det, seg = results[idx] + for label in range(len(det)): + # bbox results + bboxes = det[label] + for i in range(bboxes.shape[0]): + data = dict() + data['image_id'] = img_id + data['bbox'] = self.xyxy2xywh(bboxes[i]) + data['score'] = float(bboxes[i][4]) + data['category_id'] = self.cat_ids[label] + bbox_json_results.append(data) + + # segm results + # some detectors use different scores for bbox and mask + if isinstance(seg, tuple): + segms = seg[0][label] + mask_score = seg[1][label] + else: + segms = seg[label] + mask_score = [bbox[4] for bbox in bboxes] + for i in range(bboxes.shape[0]): + data = dict() + data['image_id'] = img_id + data['bbox'] = self.xyxy2xywh(bboxes[i]) + data['score'] = float(mask_score[i]) + data['category_id'] = self.cat_ids[label] + if isinstance(segms[i]['counts'], bytes): + segms[i]['counts'] = segms[i]['counts'].decode() + data['segmentation'] = segms[i] + segm_json_results.append(data) + return bbox_json_results, segm_json_results + + def results2json(self, results, outfile_prefix): + """Dump the detection results to a COCO style json file. + + There are 3 types of results: proposals, bbox predictions, mask + predictions, and they have different data types. This method will + automatically recognize the type, and dump them to json files. + + Args: + results (list[list | tuple | ndarray]): Testing results of the + dataset. + outfile_prefix (str): The filename prefix of the json files. If the + prefix is "somepath/xxx", the json files will be named + "somepath/xxx.bbox.json", "somepath/xxx.segm.json", + "somepath/xxx.proposal.json". + + Returns: + dict[str: str]: Possible keys are "bbox", "segm", "proposal", and \ + values are corresponding filenames. + """ + result_files = dict() + if isinstance(results[0], list): + json_results = self._det2json(results) + result_files['bbox'] = f'{outfile_prefix}.bbox.json' + result_files['proposal'] = f'{outfile_prefix}.bbox.json' + mmcv.dump(json_results, result_files['bbox']) + elif isinstance(results[0], tuple): + json_results = self._segm2json(results) + result_files['bbox'] = f'{outfile_prefix}.bbox.json' + result_files['proposal'] = f'{outfile_prefix}.bbox.json' + result_files['segm'] = f'{outfile_prefix}.segm.json' + mmcv.dump(json_results[0], result_files['bbox']) + mmcv.dump(json_results[1], result_files['segm']) + elif isinstance(results[0], np.ndarray): + json_results = self._proposal2json(results) + result_files['proposal'] = f'{outfile_prefix}.proposal.json' + mmcv.dump(json_results, result_files['proposal']) + else: + raise TypeError('invalid type of results') + return result_files + + def fast_eval_recall(self, results, proposal_nums, iou_thrs, logger=None): + gt_bboxes = [] + for i in range(len(self.img_ids)): + ann_ids = self.coco.get_ann_ids(img_ids=self.img_ids[i]) + ann_info = self.coco.load_anns(ann_ids) + if len(ann_info) == 0: + gt_bboxes.append(np.zeros((0, 4))) + continue + bboxes = [] + for ann in ann_info: + if ann.get('ignore', False) or ann['iscrowd']: + continue + x1, y1, w, h = ann['bbox'] + bboxes.append([x1, y1, x1 + w, y1 + h]) + bboxes = np.array(bboxes, dtype=np.float32) + if bboxes.shape[0] == 0: + bboxes = np.zeros((0, 4)) + gt_bboxes.append(bboxes) + + recalls = eval_recalls( + gt_bboxes, results, proposal_nums, iou_thrs, logger=logger) + ar = recalls.mean(axis=1) + return ar + + def format_results(self, results, jsonfile_prefix=None, **kwargs): + """Format the results to json (standard format for COCO evaluation). + + Args: + results (list[tuple | numpy.ndarray]): Testing results of the + dataset. + jsonfile_prefix (str | None): The prefix of json files. It includes + the file path and the prefix of filename, e.g., "a/b/prefix". + If not specified, a temp file will be created. Default: None. + + Returns: + tuple: (result_files, tmp_dir), result_files is a dict containing \ + the json filepaths, tmp_dir is the temporal directory created \ + for saving json files when jsonfile_prefix is not specified. + """ + assert isinstance(results, list), 'results must be a list' + assert len(results) == len(self), ( + 'The length of results is not equal to the dataset len: {} != {}'. + format(len(results), len(self))) + + if jsonfile_prefix is None: + tmp_dir = tempfile.TemporaryDirectory() + jsonfile_prefix = osp.join(tmp_dir.name, 'results') + else: + tmp_dir = None + result_files = self.results2json(results, jsonfile_prefix) + return result_files, tmp_dir + + def evaluate(self, + results, + metric='bbox', + logger=None, + jsonfile_prefix=None, + classwise=False, + proposal_nums=(100, 300, 1000), + iou_thrs=None, + metric_items=None): + """Evaluation in COCO protocol. + + Args: + results (list[list | tuple]): Testing results of the dataset. + metric (str | list[str]): Metrics to be evaluated. Options are + 'bbox', 'segm', 'proposal', 'proposal_fast'. + logger (logging.Logger | str | None): Logger used for printing + related information during evaluation. Default: None. + jsonfile_prefix (str | None): The prefix of json files. It includes + the file path and the prefix of filename, e.g., "a/b/prefix". + If not specified, a temp file will be created. Default: None. + classwise (bool): Whether to evaluating the AP for each class. + proposal_nums (Sequence[int]): Proposal number used for evaluating + recalls, such as recall@100, recall@1000. + Default: (100, 300, 1000). + iou_thrs (Sequence[float], optional): IoU threshold used for + evaluating recalls/mAPs. If set to a list, the average of all + IoUs will also be computed. If not specified, [0.50, 0.55, + 0.60, 0.65, 0.70, 0.75, 0.80, 0.85, 0.90, 0.95] will be used. + Default: None. + metric_items (list[str] | str, optional): Metric items that will + be returned. If not specified, ``['AR@100', 'AR@300', + 'AR@1000', 'AR_s@1000', 'AR_m@1000', 'AR_l@1000' ]`` will be + used when ``metric=='proposal'``, ``['mAP', 'mAP_50', 'mAP_75', + 'mAP_s', 'mAP_m', 'mAP_l']`` will be used when + ``metric=='bbox' or metric=='segm'``. + + Returns: + dict[str, float]: COCO style evaluation metric. + """ + + metrics = metric if isinstance(metric, list) else [metric] + allowed_metrics = ['bbox', 'segm', 'proposal', 'proposal_fast'] + for metric in metrics: + if metric not in allowed_metrics: + raise KeyError(f'metric {metric} is not supported') + if iou_thrs is None: + iou_thrs = np.linspace( + .5, 0.95, int(np.round((0.95 - .5) / .05)) + 1, endpoint=True) + if metric_items is not None: + if not isinstance(metric_items, list): + metric_items = [metric_items] + + result_files, tmp_dir = self.format_results(results, jsonfile_prefix) + + eval_results = OrderedDict() + cocoGt = self.coco + for metric in metrics: + msg = f'Evaluating {metric}...' + if logger is None: + msg = '\n' + msg + print_log(msg, logger=logger) + + if metric == 'proposal_fast': + ar = self.fast_eval_recall( + results, proposal_nums, iou_thrs, logger='silent') + log_msg = [] + for i, num in enumerate(proposal_nums): + eval_results[f'AR@{num}'] = ar[i] + log_msg.append(f'\nAR@{num}\t{ar[i]:.4f}') + log_msg = ''.join(log_msg) + print_log(log_msg, logger=logger) + continue + + if metric not in result_files: + raise KeyError(f'{metric} is not in results') + try: + cocoDt = cocoGt.loadRes(result_files[metric]) + except IndexError: + print_log( + 'The testing results of the whole dataset is empty.', + logger=logger, + level=logging.ERROR) + break + + iou_type = 'bbox' if metric == 'proposal' else metric + cocoEval = COCOeval(cocoGt, cocoDt, iou_type) + cocoEval.params.catIds = self.cat_ids + cocoEval.params.imgIds = self.img_ids + cocoEval.params.maxDets = list(proposal_nums) + cocoEval.params.iouThrs = iou_thrs + # mapping of cocoEval.stats + coco_metric_names = { + 'mAP': 0, + 'mAP_50': 1, + 'mAP_75': 2, + 'mAP_s': 3, + 'mAP_m': 4, + 'mAP_l': 5, + 'AR@100': 6, + 'AR@300': 7, + 'AR@1000': 8, + 'AR_s@1000': 9, + 'AR_m@1000': 10, + 'AR_l@1000': 11 + } + if metric_items is not None: + for metric_item in metric_items: + if metric_item not in coco_metric_names: + raise KeyError( + f'metric item {metric_item} is not supported') + + if metric == 'proposal': + cocoEval.params.useCats = 0 + cocoEval.evaluate() + cocoEval.accumulate() + cocoEval.summarize() + if metric_items is None: + metric_items = [ + 'AR@100', 'AR@300', 'AR@1000', 'AR_s@1000', + 'AR_m@1000', 'AR_l@1000' + ] + + for item in metric_items: + val = float( + f'{cocoEval.stats[coco_metric_names[item]]:.3f}') + eval_results[item] = val + else: + cocoEval.evaluate() + cocoEval.accumulate() + cocoEval.summarize() + if classwise: # Compute per-category AP + # Compute per-category AP + # from https://github.com/facebookresearch/detectron2/ + precisions = cocoEval.eval['precision'] + # precision: (iou, recall, cls, area range, max dets) + assert len(self.cat_ids) == precisions.shape[2] + + results_per_category = [] + for idx, catId in enumerate(self.cat_ids): + # area range index 0: all area ranges + # max dets index -1: typically 100 per image + nm = self.coco.loadCats(catId)[0] + precision = precisions[:, :, idx, 0, -1] + precision = precision[precision > -1] + if precision.size: + ap = np.mean(precision) + else: + ap = float('nan') + results_per_category.append( + (f'{nm["name"]}', f'{float(ap):0.3f}')) + + num_columns = min(6, len(results_per_category) * 2) + results_flatten = list( + itertools.chain(*results_per_category)) + headers = ['category', 'AP'] * (num_columns // 2) + results_2d = itertools.zip_longest(*[ + results_flatten[i::num_columns] + for i in range(num_columns) + ]) + table_data = [headers] + table_data += [result for result in results_2d] + table = AsciiTable(table_data) + print_log('\n' + table.table, logger=logger) + + if metric_items is None: + metric_items = [ + 'mAP', 'mAP_50', 'mAP_75', 'mAP_s', 'mAP_m', 'mAP_l' + ] + + for metric_item in metric_items: + key = f'{metric}_{metric_item}' + val = float( + f'{cocoEval.stats[coco_metric_names[metric_item]]:.3f}' + ) + eval_results[key] = val + ap = cocoEval.stats[:6] + eval_results[f'{metric}_mAP_copypaste'] = ( + f'{ap[0]:.3f} {ap[1]:.3f} {ap[2]:.3f} {ap[3]:.3f} ' + f'{ap[4]:.3f} {ap[5]:.3f}') + if tmp_dir is not None: + tmp_dir.cleanup() + return eval_results diff --git a/detection/mmdet/datasets/custom.py b/detection/mmdet/datasets/custom.py new file mode 100644 index 0000000..1a2351c --- /dev/null +++ b/detection/mmdet/datasets/custom.py @@ -0,0 +1,323 @@ +import os.path as osp +import warnings +from collections import OrderedDict + +import mmcv +import numpy as np +from mmcv.utils import print_log +from torch.utils.data import Dataset + +from mmdet.core import eval_map, eval_recalls +from .builder import DATASETS +from .pipelines import Compose + + +@DATASETS.register_module() +class CustomDataset(Dataset): + """Custom dataset for detection. + + The annotation format is shown as follows. The `ann` field is optional for + testing. + + .. code-block:: none + + [ + { + 'filename': 'a.jpg', + 'width': 1280, + 'height': 720, + 'ann': { + 'bboxes': (n, 4) in (x1, y1, x2, y2) order. + 'labels': (n, ), + 'bboxes_ignore': (k, 4), (optional field) + 'labels_ignore': (k, 4) (optional field) + } + }, + ... + ] + + Args: + ann_file (str): Annotation file path. + pipeline (list[dict]): Processing pipeline. + classes (str | Sequence[str], optional): Specify classes to load. + If is None, ``cls.CLASSES`` will be used. Default: None. + data_root (str, optional): Data root for ``ann_file``, + ``img_prefix``, ``seg_prefix``, ``proposal_file`` if specified. + test_mode (bool, optional): If set True, annotation will not be loaded. + filter_empty_gt (bool, optional): If set true, images without bounding + boxes of the dataset's classes will be filtered out. This option + only works when `test_mode=False`, i.e., we never filter images + during tests. + """ + + CLASSES = None + + def __init__(self, + ann_file, + pipeline, + classes=None, + data_root=None, + img_prefix='', + seg_prefix=None, + proposal_file=None, + test_mode=False, + filter_empty_gt=True): + self.ann_file = ann_file + self.data_root = data_root + self.img_prefix = img_prefix + self.seg_prefix = seg_prefix + self.proposal_file = proposal_file + self.test_mode = test_mode + self.filter_empty_gt = filter_empty_gt + self.CLASSES = self.get_classes(classes) + + # join paths if data_root is specified + if self.data_root is not None: + if not osp.isabs(self.ann_file): + self.ann_file = osp.join(self.data_root, self.ann_file) + if not (self.img_prefix is None or osp.isabs(self.img_prefix)): + self.img_prefix = osp.join(self.data_root, self.img_prefix) + if not (self.seg_prefix is None or osp.isabs(self.seg_prefix)): + self.seg_prefix = osp.join(self.data_root, self.seg_prefix) + if not (self.proposal_file is None + or osp.isabs(self.proposal_file)): + self.proposal_file = osp.join(self.data_root, + self.proposal_file) + # load annotations (and proposals) + self.data_infos = self.load_annotations(self.ann_file) + + if self.proposal_file is not None: + self.proposals = self.load_proposals(self.proposal_file) + else: + self.proposals = None + + # filter images too small and containing no annotations + if not test_mode: + valid_inds = self._filter_imgs() + self.data_infos = [self.data_infos[i] for i in valid_inds] + if self.proposals is not None: + self.proposals = [self.proposals[i] for i in valid_inds] + # set group flag for the sampler + self._set_group_flag() + + # processing pipeline + self.pipeline = Compose(pipeline) + + def __len__(self): + """Total number of samples of data.""" + return len(self.data_infos) + + def load_annotations(self, ann_file): + """Load annotation from annotation file.""" + return mmcv.load(ann_file) + + def load_proposals(self, proposal_file): + """Load proposal from proposal file.""" + return mmcv.load(proposal_file) + + def get_ann_info(self, idx): + """Get annotation by index. + + Args: + idx (int): Index of data. + + Returns: + dict: Annotation info of specified index. + """ + + return self.data_infos[idx]['ann'] + + def get_cat_ids(self, idx): + """Get category ids by index. + + Args: + idx (int): Index of data. + + Returns: + list[int]: All categories in the image of specified index. + """ + + return self.data_infos[idx]['ann']['labels'].astype(np.int).tolist() + + def pre_pipeline(self, results): + """Prepare results dict for pipeline.""" + results['img_prefix'] = self.img_prefix + results['seg_prefix'] = self.seg_prefix + results['proposal_file'] = self.proposal_file + results['bbox_fields'] = [] + results['mask_fields'] = [] + results['seg_fields'] = [] + + def _filter_imgs(self, min_size=32): + """Filter images too small.""" + if self.filter_empty_gt: + warnings.warn( + 'CustomDataset does not support filtering empty gt images.') + valid_inds = [] + for i, img_info in enumerate(self.data_infos): + if min(img_info['width'], img_info['height']) >= min_size: + valid_inds.append(i) + return valid_inds + + def _set_group_flag(self): + """Set flag according to image aspect ratio. + + Images with aspect ratio greater than 1 will be set as group 1, + otherwise group 0. + """ + self.flag = np.zeros(len(self), dtype=np.uint8) + for i in range(len(self)): + img_info = self.data_infos[i] + if img_info['width'] / img_info['height'] > 1: + self.flag[i] = 1 + + def _rand_another(self, idx): + """Get another random index from the same group as the given index.""" + pool = np.where(self.flag == self.flag[idx])[0] + return np.random.choice(pool) + + def __getitem__(self, idx): + """Get training/test data after pipeline. + + Args: + idx (int): Index of data. + + Returns: + dict: Training/test data (with annotation if `test_mode` is set \ + True). + """ + + if self.test_mode: + return self.prepare_test_img(idx) + while True: + data = self.prepare_train_img(idx) + if data is None: + idx = self._rand_another(idx) + continue + return data + + def prepare_train_img(self, idx): + """Get training data and annotations after pipeline. + + Args: + idx (int): Index of data. + + Returns: + dict: Training data and annotation after pipeline with new keys \ + introduced by pipeline. + """ + + img_info = self.data_infos[idx] + ann_info = self.get_ann_info(idx) + results = dict(img_info=img_info, ann_info=ann_info) + if self.proposals is not None: + results['proposals'] = self.proposals[idx] + self.pre_pipeline(results) + return self.pipeline(results) + + def prepare_test_img(self, idx): + """Get testing data after pipeline. + + Args: + idx (int): Index of data. + + Returns: + dict: Testing data after pipeline with new keys introduced by \ + pipeline. + """ + + img_info = self.data_infos[idx] + results = dict(img_info=img_info) + if self.proposals is not None: + results['proposals'] = self.proposals[idx] + self.pre_pipeline(results) + return self.pipeline(results) + + @classmethod + def get_classes(cls, classes=None): + """Get class names of current dataset. + + Args: + classes (Sequence[str] | str | None): If classes is None, use + default CLASSES defined by builtin dataset. If classes is a + string, take it as a file name. The file contains the name of + classes where each line contains one class name. If classes is + a tuple or list, override the CLASSES defined by the dataset. + + Returns: + tuple[str] or list[str]: Names of categories of the dataset. + """ + if classes is None: + return cls.CLASSES + + if isinstance(classes, str): + # take it as a file path + class_names = mmcv.list_from_file(classes) + elif isinstance(classes, (tuple, list)): + class_names = classes + else: + raise ValueError(f'Unsupported type {type(classes)} of classes.') + + return class_names + + def format_results(self, results, **kwargs): + """Place holder to format result to dataset specific output.""" + + def evaluate(self, + results, + metric='mAP', + logger=None, + proposal_nums=(100, 300, 1000), + iou_thr=0.5, + scale_ranges=None): + """Evaluate the dataset. + + Args: + results (list): Testing results of the dataset. + metric (str | list[str]): Metrics to be evaluated. + logger (logging.Logger | None | str): Logger used for printing + related information during evaluation. Default: None. + proposal_nums (Sequence[int]): Proposal number used for evaluating + recalls, such as recall@100, recall@1000. + Default: (100, 300, 1000). + iou_thr (float | list[float]): IoU threshold. Default: 0.5. + scale_ranges (list[tuple] | None): Scale ranges for evaluating mAP. + Default: None. + """ + + if not isinstance(metric, str): + assert len(metric) == 1 + metric = metric[0] + allowed_metrics = ['mAP', 'recall'] + if metric not in allowed_metrics: + raise KeyError(f'metric {metric} is not supported') + annotations = [self.get_ann_info(i) for i in range(len(self))] + eval_results = OrderedDict() + iou_thrs = [iou_thr] if isinstance(iou_thr, float) else iou_thr + if metric == 'mAP': + assert isinstance(iou_thrs, list) + mean_aps = [] + for iou_thr in iou_thrs: + print_log(f'\n{"-" * 15}iou_thr: {iou_thr}{"-" * 15}') + mean_ap, _ = eval_map( + results, + annotations, + scale_ranges=scale_ranges, + iou_thr=iou_thr, + dataset=self.CLASSES, + logger=logger) + mean_aps.append(mean_ap) + eval_results[f'AP{int(iou_thr * 100):02d}'] = round(mean_ap, 3) + eval_results['mAP'] = sum(mean_aps) / len(mean_aps) + elif metric == 'recall': + gt_bboxes = [ann['bboxes'] for ann in annotations] + recalls = eval_recalls( + gt_bboxes, results, proposal_nums, iou_thr, logger=logger) + for i, num in enumerate(proposal_nums): + for j, iou in enumerate(iou_thrs): + eval_results[f'recall@{num}@{iou}'] = recalls[i, j] + if recalls.shape[1] > 1: + ar = recalls.mean(axis=1) + for i, num in enumerate(proposal_nums): + eval_results[f'AR@{num}'] = ar[i] + return eval_results diff --git a/detection/mmdet/datasets/dataset_wrappers.py b/detection/mmdet/datasets/dataset_wrappers.py new file mode 100644 index 0000000..55ad5cb --- /dev/null +++ b/detection/mmdet/datasets/dataset_wrappers.py @@ -0,0 +1,282 @@ +import bisect +import math +from collections import defaultdict + +import numpy as np +from mmcv.utils import print_log +from torch.utils.data.dataset import ConcatDataset as _ConcatDataset + +from .builder import DATASETS +from .coco import CocoDataset + + +@DATASETS.register_module() +class ConcatDataset(_ConcatDataset): + """A wrapper of concatenated dataset. + + Same as :obj:`torch.utils.data.dataset.ConcatDataset`, but + concat the group flag for image aspect ratio. + + Args: + datasets (list[:obj:`Dataset`]): A list of datasets. + separate_eval (bool): Whether to evaluate the results + separately if it is used as validation dataset. + Defaults to True. + """ + + def __init__(self, datasets, separate_eval=True): + super(ConcatDataset, self).__init__(datasets) + self.CLASSES = datasets[0].CLASSES + self.separate_eval = separate_eval + if not separate_eval: + if any([isinstance(ds, CocoDataset) for ds in datasets]): + raise NotImplementedError( + 'Evaluating concatenated CocoDataset as a whole is not' + ' supported! Please set "separate_eval=True"') + elif len(set([type(ds) for ds in datasets])) != 1: + raise NotImplementedError( + 'All the datasets should have same types') + + if hasattr(datasets[0], 'flag'): + flags = [] + for i in range(0, len(datasets)): + flags.append(datasets[i].flag) + self.flag = np.concatenate(flags) + + def get_cat_ids(self, idx): + """Get category ids of concatenated dataset by index. + + Args: + idx (int): Index of data. + + Returns: + list[int]: All categories in the image of specified index. + """ + + if idx < 0: + if -idx > len(self): + raise ValueError( + 'absolute value of index should not exceed dataset length') + idx = len(self) + idx + dataset_idx = bisect.bisect_right(self.cumulative_sizes, idx) + if dataset_idx == 0: + sample_idx = idx + else: + sample_idx = idx - self.cumulative_sizes[dataset_idx - 1] + return self.datasets[dataset_idx].get_cat_ids(sample_idx) + + def evaluate(self, results, logger=None, **kwargs): + """Evaluate the results. + + Args: + results (list[list | tuple]): Testing results of the dataset. + logger (logging.Logger | str | None): Logger used for printing + related information during evaluation. Default: None. + + Returns: + dict[str: float]: AP results of the total dataset or each separate + dataset if `self.separate_eval=True`. + """ + assert len(results) == self.cumulative_sizes[-1], \ + ('Dataset and results have different sizes: ' + f'{self.cumulative_sizes[-1]} v.s. {len(results)}') + + # Check whether all the datasets support evaluation + for dataset in self.datasets: + assert hasattr(dataset, 'evaluate'), \ + f'{type(dataset)} does not implement evaluate function' + + if self.separate_eval: + dataset_idx = -1 + total_eval_results = dict() + for size, dataset in zip(self.cumulative_sizes, self.datasets): + start_idx = 0 if dataset_idx == -1 else \ + self.cumulative_sizes[dataset_idx] + end_idx = self.cumulative_sizes[dataset_idx + 1] + + results_per_dataset = results[start_idx:end_idx] + print_log( + f'\nEvaluateing {dataset.ann_file} with ' + f'{len(results_per_dataset)} images now', + logger=logger) + + eval_results_per_dataset = dataset.evaluate( + results_per_dataset, logger=logger, **kwargs) + dataset_idx += 1 + for k, v in eval_results_per_dataset.items(): + total_eval_results.update({f'{dataset_idx}_{k}': v}) + + return total_eval_results + elif any([isinstance(ds, CocoDataset) for ds in self.datasets]): + raise NotImplementedError( + 'Evaluating concatenated CocoDataset as a whole is not' + ' supported! Please set "separate_eval=True"') + elif len(set([type(ds) for ds in self.datasets])) != 1: + raise NotImplementedError( + 'All the datasets should have same types') + else: + original_data_infos = self.datasets[0].data_infos + self.datasets[0].data_infos = sum( + [dataset.data_infos for dataset in self.datasets], []) + eval_results = self.datasets[0].evaluate( + results, logger=logger, **kwargs) + self.datasets[0].data_infos = original_data_infos + return eval_results + + +@DATASETS.register_module() +class RepeatDataset(object): + """A wrapper of repeated dataset. + + The length of repeated dataset will be `times` larger than the original + dataset. This is useful when the data loading time is long but the dataset + is small. Using RepeatDataset can reduce the data loading time between + epochs. + + Args: + dataset (:obj:`Dataset`): The dataset to be repeated. + times (int): Repeat times. + """ + + def __init__(self, dataset, times): + self.dataset = dataset + self.times = times + self.CLASSES = dataset.CLASSES + if hasattr(self.dataset, 'flag'): + self.flag = np.tile(self.dataset.flag, times) + + self._ori_len = len(self.dataset) + + def __getitem__(self, idx): + return self.dataset[idx % self._ori_len] + + def get_cat_ids(self, idx): + """Get category ids of repeat dataset by index. + + Args: + idx (int): Index of data. + + Returns: + list[int]: All categories in the image of specified index. + """ + + return self.dataset.get_cat_ids(idx % self._ori_len) + + def __len__(self): + """Length after repetition.""" + return self.times * self._ori_len + + +# Modified from https://github.com/facebookresearch/detectron2/blob/41d475b75a230221e21d9cac5d69655e3415e3a4/detectron2/data/samplers/distributed_sampler.py#L57 # noqa +@DATASETS.register_module() +class ClassBalancedDataset(object): + """A wrapper of repeated dataset with repeat factor. + + Suitable for training on class imbalanced datasets like LVIS. Following + the sampling strategy in the `paper `_, + in each epoch, an image may appear multiple times based on its + "repeat factor". + The repeat factor for an image is a function of the frequency the rarest + category labeled in that image. The "frequency of category c" in [0, 1] + is defined by the fraction of images in the training set (without repeats) + in which category c appears. + The dataset needs to instantiate :func:`self.get_cat_ids` to support + ClassBalancedDataset. + + The repeat factor is computed as followed. + + 1. For each category c, compute the fraction # of images + that contain it: :math:`f(c)` + 2. For each category c, compute the category-level repeat factor: + :math:`r(c) = max(1, sqrt(t/f(c)))` + 3. For each image I, compute the image-level repeat factor: + :math:`r(I) = max_{c in I} r(c)` + + Args: + dataset (:obj:`CustomDataset`): The dataset to be repeated. + oversample_thr (float): frequency threshold below which data is + repeated. For categories with ``f_c >= oversample_thr``, there is + no oversampling. For categories with ``f_c < oversample_thr``, the + degree of oversampling following the square-root inverse frequency + heuristic above. + filter_empty_gt (bool, optional): If set true, images without bounding + boxes will not be oversampled. Otherwise, they will be categorized + as the pure background class and involved into the oversampling. + Default: True. + """ + + def __init__(self, dataset, oversample_thr, filter_empty_gt=True): + self.dataset = dataset + self.oversample_thr = oversample_thr + self.filter_empty_gt = filter_empty_gt + self.CLASSES = dataset.CLASSES + + repeat_factors = self._get_repeat_factors(dataset, oversample_thr) + repeat_indices = [] + for dataset_idx, repeat_factor in enumerate(repeat_factors): + repeat_indices.extend([dataset_idx] * math.ceil(repeat_factor)) + self.repeat_indices = repeat_indices + + flags = [] + if hasattr(self.dataset, 'flag'): + for flag, repeat_factor in zip(self.dataset.flag, repeat_factors): + flags.extend([flag] * int(math.ceil(repeat_factor))) + assert len(flags) == len(repeat_indices) + self.flag = np.asarray(flags, dtype=np.uint8) + + def _get_repeat_factors(self, dataset, repeat_thr): + """Get repeat factor for each images in the dataset. + + Args: + dataset (:obj:`CustomDataset`): The dataset + repeat_thr (float): The threshold of frequency. If an image + contains the categories whose frequency below the threshold, + it would be repeated. + + Returns: + list[float]: The repeat factors for each images in the dataset. + """ + + # 1. For each category c, compute the fraction # of images + # that contain it: f(c) + category_freq = defaultdict(int) + num_images = len(dataset) + for idx in range(num_images): + cat_ids = set(self.dataset.get_cat_ids(idx)) + if len(cat_ids) == 0 and not self.filter_empty_gt: + cat_ids = set([len(self.CLASSES)]) + for cat_id in cat_ids: + category_freq[cat_id] += 1 + for k, v in category_freq.items(): + category_freq[k] = v / num_images + + # 2. For each category c, compute the category-level repeat factor: + # r(c) = max(1, sqrt(t/f(c))) + category_repeat = { + cat_id: max(1.0, math.sqrt(repeat_thr / cat_freq)) + for cat_id, cat_freq in category_freq.items() + } + + # 3. For each image I, compute the image-level repeat factor: + # r(I) = max_{c in I} r(c) + repeat_factors = [] + for idx in range(num_images): + cat_ids = set(self.dataset.get_cat_ids(idx)) + if len(cat_ids) == 0 and not self.filter_empty_gt: + cat_ids = set([len(self.CLASSES)]) + repeat_factor = 1 + if len(cat_ids) > 0: + repeat_factor = max( + {category_repeat[cat_id] + for cat_id in cat_ids}) + repeat_factors.append(repeat_factor) + + return repeat_factors + + def __getitem__(self, idx): + ori_index = self.repeat_indices[idx] + return self.dataset[ori_index] + + def __len__(self): + """Length after repetition.""" + return len(self.repeat_indices) diff --git a/detection/mmdet/datasets/deepfashion.py b/detection/mmdet/datasets/deepfashion.py new file mode 100644 index 0000000..1125376 --- /dev/null +++ b/detection/mmdet/datasets/deepfashion.py @@ -0,0 +1,10 @@ +from .builder import DATASETS +from .coco import CocoDataset + + +@DATASETS.register_module() +class DeepFashionDataset(CocoDataset): + + CLASSES = ('top', 'skirt', 'leggings', 'dress', 'outer', 'pants', 'bag', + 'neckwear', 'headwear', 'eyeglass', 'belt', 'footwear', 'hair', + 'skin', 'face') diff --git a/detection/mmdet/datasets/lvis.py b/detection/mmdet/datasets/lvis.py new file mode 100644 index 0000000..122c64e --- /dev/null +++ b/detection/mmdet/datasets/lvis.py @@ -0,0 +1,742 @@ +import itertools +import logging +import os.path as osp +import tempfile +from collections import OrderedDict + +import numpy as np +from mmcv.utils import print_log +from terminaltables import AsciiTable + +from .builder import DATASETS +from .coco import CocoDataset + + +@DATASETS.register_module() +class LVISV05Dataset(CocoDataset): + + CLASSES = ( + 'acorn', 'aerosol_can', 'air_conditioner', 'airplane', 'alarm_clock', + 'alcohol', 'alligator', 'almond', 'ambulance', 'amplifier', 'anklet', + 'antenna', 'apple', 'apple_juice', 'applesauce', 'apricot', 'apron', + 'aquarium', 'armband', 'armchair', 'armoire', 'armor', 'artichoke', + 'trash_can', 'ashtray', 'asparagus', 'atomizer', 'avocado', 'award', + 'awning', 'ax', 'baby_buggy', 'basketball_backboard', 'backpack', + 'handbag', 'suitcase', 'bagel', 'bagpipe', 'baguet', 'bait', 'ball', + 'ballet_skirt', 'balloon', 'bamboo', 'banana', 'Band_Aid', 'bandage', + 'bandanna', 'banjo', 'banner', 'barbell', 'barge', 'barrel', + 'barrette', 'barrow', 'baseball_base', 'baseball', 'baseball_bat', + 'baseball_cap', 'baseball_glove', 'basket', 'basketball_hoop', + 'basketball', 'bass_horn', 'bat_(animal)', 'bath_mat', 'bath_towel', + 'bathrobe', 'bathtub', 'batter_(food)', 'battery', 'beachball', 'bead', + 'beaker', 'bean_curd', 'beanbag', 'beanie', 'bear', 'bed', + 'bedspread', 'cow', 'beef_(food)', 'beeper', 'beer_bottle', 'beer_can', + 'beetle', 'bell', 'bell_pepper', 'belt', 'belt_buckle', 'bench', + 'beret', 'bib', 'Bible', 'bicycle', 'visor', 'binder', 'binoculars', + 'bird', 'birdfeeder', 'birdbath', 'birdcage', 'birdhouse', + 'birthday_cake', 'birthday_card', 'biscuit_(bread)', 'pirate_flag', + 'black_sheep', 'blackboard', 'blanket', 'blazer', 'blender', 'blimp', + 'blinker', 'blueberry', 'boar', 'gameboard', 'boat', 'bobbin', + 'bobby_pin', 'boiled_egg', 'bolo_tie', 'deadbolt', 'bolt', 'bonnet', + 'book', 'book_bag', 'bookcase', 'booklet', 'bookmark', + 'boom_microphone', 'boot', 'bottle', 'bottle_opener', 'bouquet', + 'bow_(weapon)', 'bow_(decorative_ribbons)', 'bow-tie', 'bowl', + 'pipe_bowl', 'bowler_hat', 'bowling_ball', 'bowling_pin', + 'boxing_glove', 'suspenders', 'bracelet', 'brass_plaque', 'brassiere', + 'bread-bin', 'breechcloth', 'bridal_gown', 'briefcase', + 'bristle_brush', 'broccoli', 'broach', 'broom', 'brownie', + 'brussels_sprouts', 'bubble_gum', 'bucket', 'horse_buggy', 'bull', + 'bulldog', 'bulldozer', 'bullet_train', 'bulletin_board', + 'bulletproof_vest', 'bullhorn', 'corned_beef', 'bun', 'bunk_bed', + 'buoy', 'burrito', 'bus_(vehicle)', 'business_card', 'butcher_knife', + 'butter', 'butterfly', 'button', 'cab_(taxi)', 'cabana', 'cabin_car', + 'cabinet', 'locker', 'cake', 'calculator', 'calendar', 'calf', + 'camcorder', 'camel', 'camera', 'camera_lens', 'camper_(vehicle)', + 'can', 'can_opener', 'candelabrum', 'candle', 'candle_holder', + 'candy_bar', 'candy_cane', 'walking_cane', 'canister', 'cannon', + 'canoe', 'cantaloup', 'canteen', 'cap_(headwear)', 'bottle_cap', + 'cape', 'cappuccino', 'car_(automobile)', 'railcar_(part_of_a_train)', + 'elevator_car', 'car_battery', 'identity_card', 'card', 'cardigan', + 'cargo_ship', 'carnation', 'horse_carriage', 'carrot', 'tote_bag', + 'cart', 'carton', 'cash_register', 'casserole', 'cassette', 'cast', + 'cat', 'cauliflower', 'caviar', 'cayenne_(spice)', 'CD_player', + 'celery', 'cellular_telephone', 'chain_mail', 'chair', 'chaise_longue', + 'champagne', 'chandelier', 'chap', 'checkbook', 'checkerboard', + 'cherry', 'chessboard', 'chest_of_drawers_(furniture)', + 'chicken_(animal)', 'chicken_wire', 'chickpea', 'Chihuahua', + 'chili_(vegetable)', 'chime', 'chinaware', 'crisp_(potato_chip)', + 'poker_chip', 'chocolate_bar', 'chocolate_cake', 'chocolate_milk', + 'chocolate_mousse', 'choker', 'chopping_board', 'chopstick', + 'Christmas_tree', 'slide', 'cider', 'cigar_box', 'cigarette', + 'cigarette_case', 'cistern', 'clarinet', 'clasp', 'cleansing_agent', + 'clementine', 'clip', 'clipboard', 'clock', 'clock_tower', + 'clothes_hamper', 'clothespin', 'clutch_bag', 'coaster', 'coat', + 'coat_hanger', 'coatrack', 'cock', 'coconut', 'coffee_filter', + 'coffee_maker', 'coffee_table', 'coffeepot', 'coil', 'coin', + 'colander', 'coleslaw', 'coloring_material', 'combination_lock', + 'pacifier', 'comic_book', 'computer_keyboard', 'concrete_mixer', + 'cone', 'control', 'convertible_(automobile)', 'sofa_bed', 'cookie', + 'cookie_jar', 'cooking_utensil', 'cooler_(for_food)', + 'cork_(bottle_plug)', 'corkboard', 'corkscrew', 'edible_corn', + 'cornbread', 'cornet', 'cornice', 'cornmeal', 'corset', + 'romaine_lettuce', 'costume', 'cougar', 'coverall', 'cowbell', + 'cowboy_hat', 'crab_(animal)', 'cracker', 'crape', 'crate', 'crayon', + 'cream_pitcher', 'credit_card', 'crescent_roll', 'crib', 'crock_pot', + 'crossbar', 'crouton', 'crow', 'crown', 'crucifix', 'cruise_ship', + 'police_cruiser', 'crumb', 'crutch', 'cub_(animal)', 'cube', + 'cucumber', 'cufflink', 'cup', 'trophy_cup', 'cupcake', 'hair_curler', + 'curling_iron', 'curtain', 'cushion', 'custard', 'cutting_tool', + 'cylinder', 'cymbal', 'dachshund', 'dagger', 'dartboard', + 'date_(fruit)', 'deck_chair', 'deer', 'dental_floss', 'desk', + 'detergent', 'diaper', 'diary', 'die', 'dinghy', 'dining_table', 'tux', + 'dish', 'dish_antenna', 'dishrag', 'dishtowel', 'dishwasher', + 'dishwasher_detergent', 'diskette', 'dispenser', 'Dixie_cup', 'dog', + 'dog_collar', 'doll', 'dollar', 'dolphin', 'domestic_ass', 'eye_mask', + 'doorbell', 'doorknob', 'doormat', 'doughnut', 'dove', 'dragonfly', + 'drawer', 'underdrawers', 'dress', 'dress_hat', 'dress_suit', + 'dresser', 'drill', 'drinking_fountain', 'drone', 'dropper', + 'drum_(musical_instrument)', 'drumstick', 'duck', 'duckling', + 'duct_tape', 'duffel_bag', 'dumbbell', 'dumpster', 'dustpan', + 'Dutch_oven', 'eagle', 'earphone', 'earplug', 'earring', 'easel', + 'eclair', 'eel', 'egg', 'egg_roll', 'egg_yolk', 'eggbeater', + 'eggplant', 'electric_chair', 'refrigerator', 'elephant', 'elk', + 'envelope', 'eraser', 'escargot', 'eyepatch', 'falcon', 'fan', + 'faucet', 'fedora', 'ferret', 'Ferris_wheel', 'ferry', 'fig_(fruit)', + 'fighter_jet', 'figurine', 'file_cabinet', 'file_(tool)', 'fire_alarm', + 'fire_engine', 'fire_extinguisher', 'fire_hose', 'fireplace', + 'fireplug', 'fish', 'fish_(food)', 'fishbowl', 'fishing_boat', + 'fishing_rod', 'flag', 'flagpole', 'flamingo', 'flannel', 'flash', + 'flashlight', 'fleece', 'flip-flop_(sandal)', 'flipper_(footwear)', + 'flower_arrangement', 'flute_glass', 'foal', 'folding_chair', + 'food_processor', 'football_(American)', 'football_helmet', + 'footstool', 'fork', 'forklift', 'freight_car', 'French_toast', + 'freshener', 'frisbee', 'frog', 'fruit_juice', 'fruit_salad', + 'frying_pan', 'fudge', 'funnel', 'futon', 'gag', 'garbage', + 'garbage_truck', 'garden_hose', 'gargle', 'gargoyle', 'garlic', + 'gasmask', 'gazelle', 'gelatin', 'gemstone', 'giant_panda', + 'gift_wrap', 'ginger', 'giraffe', 'cincture', + 'glass_(drink_container)', 'globe', 'glove', 'goat', 'goggles', + 'goldfish', 'golf_club', 'golfcart', 'gondola_(boat)', 'goose', + 'gorilla', 'gourd', 'surgical_gown', 'grape', 'grasshopper', 'grater', + 'gravestone', 'gravy_boat', 'green_bean', 'green_onion', 'griddle', + 'grillroom', 'grinder_(tool)', 'grits', 'grizzly', 'grocery_bag', + 'guacamole', 'guitar', 'gull', 'gun', 'hair_spray', 'hairbrush', + 'hairnet', 'hairpin', 'ham', 'hamburger', 'hammer', 'hammock', + 'hamper', 'hamster', 'hair_dryer', 'hand_glass', 'hand_towel', + 'handcart', 'handcuff', 'handkerchief', 'handle', 'handsaw', + 'hardback_book', 'harmonium', 'hat', 'hatbox', 'hatch', 'veil', + 'headband', 'headboard', 'headlight', 'headscarf', 'headset', + 'headstall_(for_horses)', 'hearing_aid', 'heart', 'heater', + 'helicopter', 'helmet', 'heron', 'highchair', 'hinge', 'hippopotamus', + 'hockey_stick', 'hog', 'home_plate_(baseball)', 'honey', 'fume_hood', + 'hook', 'horse', 'hose', 'hot-air_balloon', 'hotplate', 'hot_sauce', + 'hourglass', 'houseboat', 'hummingbird', 'hummus', 'polar_bear', + 'icecream', 'popsicle', 'ice_maker', 'ice_pack', 'ice_skate', + 'ice_tea', 'igniter', 'incense', 'inhaler', 'iPod', + 'iron_(for_clothing)', 'ironing_board', 'jacket', 'jam', 'jean', + 'jeep', 'jelly_bean', 'jersey', 'jet_plane', 'jewelry', 'joystick', + 'jumpsuit', 'kayak', 'keg', 'kennel', 'kettle', 'key', 'keycard', + 'kilt', 'kimono', 'kitchen_sink', 'kitchen_table', 'kite', 'kitten', + 'kiwi_fruit', 'knee_pad', 'knife', 'knight_(chess_piece)', + 'knitting_needle', 'knob', 'knocker_(on_a_door)', 'koala', 'lab_coat', + 'ladder', 'ladle', 'ladybug', 'lamb_(animal)', 'lamb-chop', 'lamp', + 'lamppost', 'lampshade', 'lantern', 'lanyard', 'laptop_computer', + 'lasagna', 'latch', 'lawn_mower', 'leather', 'legging_(clothing)', + 'Lego', 'lemon', 'lemonade', 'lettuce', 'license_plate', 'life_buoy', + 'life_jacket', 'lightbulb', 'lightning_rod', 'lime', 'limousine', + 'linen_paper', 'lion', 'lip_balm', 'lipstick', 'liquor', 'lizard', + 'Loafer_(type_of_shoe)', 'log', 'lollipop', 'lotion', + 'speaker_(stero_equipment)', 'loveseat', 'machine_gun', 'magazine', + 'magnet', 'mail_slot', 'mailbox_(at_home)', 'mallet', 'mammoth', + 'mandarin_orange', 'manger', 'manhole', 'map', 'marker', 'martini', + 'mascot', 'mashed_potato', 'masher', 'mask', 'mast', + 'mat_(gym_equipment)', 'matchbox', 'mattress', 'measuring_cup', + 'measuring_stick', 'meatball', 'medicine', 'melon', 'microphone', + 'microscope', 'microwave_oven', 'milestone', 'milk', 'minivan', + 'mint_candy', 'mirror', 'mitten', 'mixer_(kitchen_tool)', 'money', + 'monitor_(computer_equipment) computer_monitor', 'monkey', 'motor', + 'motor_scooter', 'motor_vehicle', 'motorboat', 'motorcycle', + 'mound_(baseball)', 'mouse_(animal_rodent)', + 'mouse_(computer_equipment)', 'mousepad', 'muffin', 'mug', 'mushroom', + 'music_stool', 'musical_instrument', 'nailfile', 'nameplate', 'napkin', + 'neckerchief', 'necklace', 'necktie', 'needle', 'nest', 'newsstand', + 'nightshirt', 'nosebag_(for_animals)', 'noseband_(for_animals)', + 'notebook', 'notepad', 'nut', 'nutcracker', 'oar', 'octopus_(food)', + 'octopus_(animal)', 'oil_lamp', 'olive_oil', 'omelet', 'onion', + 'orange_(fruit)', 'orange_juice', 'oregano', 'ostrich', 'ottoman', + 'overalls_(clothing)', 'owl', 'packet', 'inkpad', 'pad', 'paddle', + 'padlock', 'paintbox', 'paintbrush', 'painting', 'pajamas', 'palette', + 'pan_(for_cooking)', 'pan_(metal_container)', 'pancake', 'pantyhose', + 'papaya', 'paperclip', 'paper_plate', 'paper_towel', 'paperback_book', + 'paperweight', 'parachute', 'parakeet', 'parasail_(sports)', + 'parchment', 'parka', 'parking_meter', 'parrot', + 'passenger_car_(part_of_a_train)', 'passenger_ship', 'passport', + 'pastry', 'patty_(food)', 'pea_(food)', 'peach', 'peanut_butter', + 'pear', 'peeler_(tool_for_fruit_and_vegetables)', 'pegboard', + 'pelican', 'pen', 'pencil', 'pencil_box', 'pencil_sharpener', + 'pendulum', 'penguin', 'pennant', 'penny_(coin)', 'pepper', + 'pepper_mill', 'perfume', 'persimmon', 'baby', 'pet', 'petfood', + 'pew_(church_bench)', 'phonebook', 'phonograph_record', 'piano', + 'pickle', 'pickup_truck', 'pie', 'pigeon', 'piggy_bank', 'pillow', + 'pin_(non_jewelry)', 'pineapple', 'pinecone', 'ping-pong_ball', + 'pinwheel', 'tobacco_pipe', 'pipe', 'pistol', 'pita_(bread)', + 'pitcher_(vessel_for_liquid)', 'pitchfork', 'pizza', 'place_mat', + 'plate', 'platter', 'playing_card', 'playpen', 'pliers', + 'plow_(farm_equipment)', 'pocket_watch', 'pocketknife', + 'poker_(fire_stirring_tool)', 'pole', 'police_van', 'polo_shirt', + 'poncho', 'pony', 'pool_table', 'pop_(soda)', 'portrait', + 'postbox_(public)', 'postcard', 'poster', 'pot', 'flowerpot', 'potato', + 'potholder', 'pottery', 'pouch', 'power_shovel', 'prawn', 'printer', + 'projectile_(weapon)', 'projector', 'propeller', 'prune', 'pudding', + 'puffer_(fish)', 'puffin', 'pug-dog', 'pumpkin', 'puncher', 'puppet', + 'puppy', 'quesadilla', 'quiche', 'quilt', 'rabbit', 'race_car', + 'racket', 'radar', 'radiator', 'radio_receiver', 'radish', 'raft', + 'rag_doll', 'raincoat', 'ram_(animal)', 'raspberry', 'rat', + 'razorblade', 'reamer_(juicer)', 'rearview_mirror', 'receipt', + 'recliner', 'record_player', 'red_cabbage', 'reflector', + 'remote_control', 'rhinoceros', 'rib_(food)', 'rifle', 'ring', + 'river_boat', 'road_map', 'robe', 'rocking_chair', 'roller_skate', + 'Rollerblade', 'rolling_pin', 'root_beer', + 'router_(computer_equipment)', 'rubber_band', 'runner_(carpet)', + 'plastic_bag', 'saddle_(on_an_animal)', 'saddle_blanket', 'saddlebag', + 'safety_pin', 'sail', 'salad', 'salad_plate', 'salami', + 'salmon_(fish)', 'salmon_(food)', 'salsa', 'saltshaker', + 'sandal_(type_of_shoe)', 'sandwich', 'satchel', 'saucepan', 'saucer', + 'sausage', 'sawhorse', 'saxophone', 'scale_(measuring_instrument)', + 'scarecrow', 'scarf', 'school_bus', 'scissors', 'scoreboard', + 'scrambled_eggs', 'scraper', 'scratcher', 'screwdriver', + 'scrubbing_brush', 'sculpture', 'seabird', 'seahorse', 'seaplane', + 'seashell', 'seedling', 'serving_dish', 'sewing_machine', 'shaker', + 'shampoo', 'shark', 'sharpener', 'Sharpie', 'shaver_(electric)', + 'shaving_cream', 'shawl', 'shears', 'sheep', 'shepherd_dog', + 'sherbert', 'shield', 'shirt', 'shoe', 'shopping_bag', 'shopping_cart', + 'short_pants', 'shot_glass', 'shoulder_bag', 'shovel', 'shower_head', + 'shower_curtain', 'shredder_(for_paper)', 'sieve', 'signboard', 'silo', + 'sink', 'skateboard', 'skewer', 'ski', 'ski_boot', 'ski_parka', + 'ski_pole', 'skirt', 'sled', 'sleeping_bag', 'sling_(bandage)', + 'slipper_(footwear)', 'smoothie', 'snake', 'snowboard', 'snowman', + 'snowmobile', 'soap', 'soccer_ball', 'sock', 'soda_fountain', + 'carbonated_water', 'sofa', 'softball', 'solar_array', 'sombrero', + 'soup', 'soup_bowl', 'soupspoon', 'sour_cream', 'soya_milk', + 'space_shuttle', 'sparkler_(fireworks)', 'spatula', 'spear', + 'spectacles', 'spice_rack', 'spider', 'sponge', 'spoon', 'sportswear', + 'spotlight', 'squirrel', 'stapler_(stapling_machine)', 'starfish', + 'statue_(sculpture)', 'steak_(food)', 'steak_knife', + 'steamer_(kitchen_appliance)', 'steering_wheel', 'stencil', + 'stepladder', 'step_stool', 'stereo_(sound_system)', 'stew', 'stirrer', + 'stirrup', 'stockings_(leg_wear)', 'stool', 'stop_sign', 'brake_light', + 'stove', 'strainer', 'strap', 'straw_(for_drinking)', 'strawberry', + 'street_sign', 'streetlight', 'string_cheese', 'stylus', 'subwoofer', + 'sugar_bowl', 'sugarcane_(plant)', 'suit_(clothing)', 'sunflower', + 'sunglasses', 'sunhat', 'sunscreen', 'surfboard', 'sushi', 'mop', + 'sweat_pants', 'sweatband', 'sweater', 'sweatshirt', 'sweet_potato', + 'swimsuit', 'sword', 'syringe', 'Tabasco_sauce', 'table-tennis_table', + 'table', 'table_lamp', 'tablecloth', 'tachometer', 'taco', 'tag', + 'taillight', 'tambourine', 'army_tank', 'tank_(storage_vessel)', + 'tank_top_(clothing)', 'tape_(sticky_cloth_or_paper)', 'tape_measure', + 'tapestry', 'tarp', 'tartan', 'tassel', 'tea_bag', 'teacup', + 'teakettle', 'teapot', 'teddy_bear', 'telephone', 'telephone_booth', + 'telephone_pole', 'telephoto_lens', 'television_camera', + 'television_set', 'tennis_ball', 'tennis_racket', 'tequila', + 'thermometer', 'thermos_bottle', 'thermostat', 'thimble', 'thread', + 'thumbtack', 'tiara', 'tiger', 'tights_(clothing)', 'timer', 'tinfoil', + 'tinsel', 'tissue_paper', 'toast_(food)', 'toaster', 'toaster_oven', + 'toilet', 'toilet_tissue', 'tomato', 'tongs', 'toolbox', 'toothbrush', + 'toothpaste', 'toothpick', 'cover', 'tortilla', 'tow_truck', 'towel', + 'towel_rack', 'toy', 'tractor_(farm_equipment)', 'traffic_light', + 'dirt_bike', 'trailer_truck', 'train_(railroad_vehicle)', 'trampoline', + 'tray', 'tree_house', 'trench_coat', 'triangle_(musical_instrument)', + 'tricycle', 'tripod', 'trousers', 'truck', 'truffle_(chocolate)', + 'trunk', 'vat', 'turban', 'turkey_(bird)', 'turkey_(food)', 'turnip', + 'turtle', 'turtleneck_(clothing)', 'typewriter', 'umbrella', + 'underwear', 'unicycle', 'urinal', 'urn', 'vacuum_cleaner', 'valve', + 'vase', 'vending_machine', 'vent', 'videotape', 'vinegar', 'violin', + 'vodka', 'volleyball', 'vulture', 'waffle', 'waffle_iron', 'wagon', + 'wagon_wheel', 'walking_stick', 'wall_clock', 'wall_socket', 'wallet', + 'walrus', 'wardrobe', 'wasabi', 'automatic_washer', 'watch', + 'water_bottle', 'water_cooler', 'water_faucet', 'water_filter', + 'water_heater', 'water_jug', 'water_gun', 'water_scooter', 'water_ski', + 'water_tower', 'watering_can', 'watermelon', 'weathervane', 'webcam', + 'wedding_cake', 'wedding_ring', 'wet_suit', 'wheel', 'wheelchair', + 'whipped_cream', 'whiskey', 'whistle', 'wick', 'wig', 'wind_chime', + 'windmill', 'window_box_(for_plants)', 'windshield_wiper', 'windsock', + 'wine_bottle', 'wine_bucket', 'wineglass', 'wing_chair', + 'blinder_(for_horses)', 'wok', 'wolf', 'wooden_spoon', 'wreath', + 'wrench', 'wristband', 'wristlet', 'yacht', 'yak', 'yogurt', + 'yoke_(animal_equipment)', 'zebra', 'zucchini') + + def load_annotations(self, ann_file): + """Load annotation from lvis style annotation file. + + Args: + ann_file (str): Path of annotation file. + + Returns: + list[dict]: Annotation info from LVIS api. + """ + + try: + import lvis + assert lvis.__version__ >= '10.5.3' + from lvis import LVIS + except AssertionError: + raise AssertionError('Incompatible version of lvis is installed. ' + 'Run pip uninstall lvis first. Then run pip ' + 'install mmlvis to install open-mmlab forked ' + 'lvis. ') + except ImportError: + raise ImportError('Package lvis is not installed. Please run pip ' + 'install mmlvis to install open-mmlab forked ' + 'lvis.') + self.coco = LVIS(ann_file) + self.cat_ids = self.coco.get_cat_ids() + self.cat2label = {cat_id: i for i, cat_id in enumerate(self.cat_ids)} + self.img_ids = self.coco.get_img_ids() + data_infos = [] + for i in self.img_ids: + info = self.coco.load_imgs([i])[0] + if info['file_name'].startswith('COCO'): + # Convert form the COCO 2014 file naming convention of + # COCO_[train/val/test]2014_000000000000.jpg to the 2017 + # naming convention of 000000000000.jpg + # (LVIS v1 will fix this naming issue) + info['filename'] = info['file_name'][-16:] + else: + info['filename'] = info['file_name'] + data_infos.append(info) + return data_infos + + def evaluate(self, + results, + metric='bbox', + logger=None, + jsonfile_prefix=None, + classwise=False, + proposal_nums=(100, 300, 1000), + iou_thrs=np.arange(0.5, 0.96, 0.05)): + """Evaluation in LVIS protocol. + + Args: + results (list[list | tuple]): Testing results of the dataset. + metric (str | list[str]): Metrics to be evaluated. Options are + 'bbox', 'segm', 'proposal', 'proposal_fast'. + logger (logging.Logger | str | None): Logger used for printing + related information during evaluation. Default: None. + jsonfile_prefix (str | None): + classwise (bool): Whether to evaluating the AP for each class. + proposal_nums (Sequence[int]): Proposal number used for evaluating + recalls, such as recall@100, recall@1000. + Default: (100, 300, 1000). + iou_thrs (Sequence[float]): IoU threshold used for evaluating + recalls. If set to a list, the average recall of all IoUs will + also be computed. Default: 0.5. + + Returns: + dict[str, float]: LVIS style metrics. + """ + + try: + import lvis + assert lvis.__version__ >= '10.5.3' + from lvis import LVISResults, LVISEval + except AssertionError: + raise AssertionError('Incompatible version of lvis is installed. ' + 'Run pip uninstall lvis first. Then run pip ' + 'install mmlvis to install open-mmlab forked ' + 'lvis. ') + except ImportError: + raise ImportError('Package lvis is not installed. Please run pip ' + 'install mmlvis to install open-mmlab forked ' + 'lvis.') + assert isinstance(results, list), 'results must be a list' + assert len(results) == len(self), ( + 'The length of results is not equal to the dataset len: {} != {}'. + format(len(results), len(self))) + + metrics = metric if isinstance(metric, list) else [metric] + allowed_metrics = ['bbox', 'segm', 'proposal', 'proposal_fast'] + for metric in metrics: + if metric not in allowed_metrics: + raise KeyError('metric {} is not supported'.format(metric)) + + if jsonfile_prefix is None: + tmp_dir = tempfile.TemporaryDirectory() + jsonfile_prefix = osp.join(tmp_dir.name, 'results') + else: + tmp_dir = None + result_files = self.results2json(results, jsonfile_prefix) + + eval_results = OrderedDict() + # get original api + lvis_gt = self.coco + for metric in metrics: + msg = 'Evaluating {}...'.format(metric) + if logger is None: + msg = '\n' + msg + print_log(msg, logger=logger) + + if metric == 'proposal_fast': + ar = self.fast_eval_recall( + results, proposal_nums, iou_thrs, logger='silent') + log_msg = [] + for i, num in enumerate(proposal_nums): + eval_results['AR@{}'.format(num)] = ar[i] + log_msg.append('\nAR@{}\t{:.4f}'.format(num, ar[i])) + log_msg = ''.join(log_msg) + print_log(log_msg, logger=logger) + continue + + if metric not in result_files: + raise KeyError('{} is not in results'.format(metric)) + try: + lvis_dt = LVISResults(lvis_gt, result_files[metric]) + except IndexError: + print_log( + 'The testing results of the whole dataset is empty.', + logger=logger, + level=logging.ERROR) + break + + iou_type = 'bbox' if metric == 'proposal' else metric + lvis_eval = LVISEval(lvis_gt, lvis_dt, iou_type) + lvis_eval.params.imgIds = self.img_ids + if metric == 'proposal': + lvis_eval.params.useCats = 0 + lvis_eval.params.maxDets = list(proposal_nums) + lvis_eval.evaluate() + lvis_eval.accumulate() + lvis_eval.summarize() + for k, v in lvis_eval.get_results().items(): + if k.startswith('AR'): + val = float('{:.3f}'.format(float(v))) + eval_results[k] = val + else: + lvis_eval.evaluate() + lvis_eval.accumulate() + lvis_eval.summarize() + lvis_results = lvis_eval.get_results() + if classwise: # Compute per-category AP + # Compute per-category AP + # from https://github.com/facebookresearch/detectron2/ + precisions = lvis_eval.eval['precision'] + # precision: (iou, recall, cls, area range, max dets) + assert len(self.cat_ids) == precisions.shape[2] + + results_per_category = [] + for idx, catId in enumerate(self.cat_ids): + # area range index 0: all area ranges + # max dets index -1: typically 100 per image + nm = self.coco.load_cats(catId)[0] + precision = precisions[:, :, idx, 0, -1] + precision = precision[precision > -1] + if precision.size: + ap = np.mean(precision) + else: + ap = float('nan') + results_per_category.append( + (f'{nm["name"]}', f'{float(ap):0.3f}')) + + num_columns = min(6, len(results_per_category) * 2) + results_flatten = list( + itertools.chain(*results_per_category)) + headers = ['category', 'AP'] * (num_columns // 2) + results_2d = itertools.zip_longest(*[ + results_flatten[i::num_columns] + for i in range(num_columns) + ]) + table_data = [headers] + table_data += [result for result in results_2d] + table = AsciiTable(table_data) + print_log('\n' + table.table, logger=logger) + + for k, v in lvis_results.items(): + if k.startswith('AP'): + key = '{}_{}'.format(metric, k) + val = float('{:.3f}'.format(float(v))) + eval_results[key] = val + ap_summary = ' '.join([ + '{}:{:.3f}'.format(k, float(v)) + for k, v in lvis_results.items() if k.startswith('AP') + ]) + eval_results['{}_mAP_copypaste'.format(metric)] = ap_summary + lvis_eval.print_results() + if tmp_dir is not None: + tmp_dir.cleanup() + return eval_results + + +LVISDataset = LVISV05Dataset +DATASETS.register_module(name='LVISDataset', module=LVISDataset) + + +@DATASETS.register_module() +class LVISV1Dataset(LVISDataset): + + CLASSES = ( + 'aerosol_can', 'air_conditioner', 'airplane', 'alarm_clock', 'alcohol', + 'alligator', 'almond', 'ambulance', 'amplifier', 'anklet', 'antenna', + 'apple', 'applesauce', 'apricot', 'apron', 'aquarium', + 'arctic_(type_of_shoe)', 'armband', 'armchair', 'armoire', 'armor', + 'artichoke', 'trash_can', 'ashtray', 'asparagus', 'atomizer', + 'avocado', 'award', 'awning', 'ax', 'baboon', 'baby_buggy', + 'basketball_backboard', 'backpack', 'handbag', 'suitcase', 'bagel', + 'bagpipe', 'baguet', 'bait', 'ball', 'ballet_skirt', 'balloon', + 'bamboo', 'banana', 'Band_Aid', 'bandage', 'bandanna', 'banjo', + 'banner', 'barbell', 'barge', 'barrel', 'barrette', 'barrow', + 'baseball_base', 'baseball', 'baseball_bat', 'baseball_cap', + 'baseball_glove', 'basket', 'basketball', 'bass_horn', 'bat_(animal)', + 'bath_mat', 'bath_towel', 'bathrobe', 'bathtub', 'batter_(food)', + 'battery', 'beachball', 'bead', 'bean_curd', 'beanbag', 'beanie', + 'bear', 'bed', 'bedpan', 'bedspread', 'cow', 'beef_(food)', 'beeper', + 'beer_bottle', 'beer_can', 'beetle', 'bell', 'bell_pepper', 'belt', + 'belt_buckle', 'bench', 'beret', 'bib', 'Bible', 'bicycle', 'visor', + 'billboard', 'binder', 'binoculars', 'bird', 'birdfeeder', 'birdbath', + 'birdcage', 'birdhouse', 'birthday_cake', 'birthday_card', + 'pirate_flag', 'black_sheep', 'blackberry', 'blackboard', 'blanket', + 'blazer', 'blender', 'blimp', 'blinker', 'blouse', 'blueberry', + 'gameboard', 'boat', 'bob', 'bobbin', 'bobby_pin', 'boiled_egg', + 'bolo_tie', 'deadbolt', 'bolt', 'bonnet', 'book', 'bookcase', + 'booklet', 'bookmark', 'boom_microphone', 'boot', 'bottle', + 'bottle_opener', 'bouquet', 'bow_(weapon)', 'bow_(decorative_ribbons)', + 'bow-tie', 'bowl', 'pipe_bowl', 'bowler_hat', 'bowling_ball', 'box', + 'boxing_glove', 'suspenders', 'bracelet', 'brass_plaque', 'brassiere', + 'bread-bin', 'bread', 'breechcloth', 'bridal_gown', 'briefcase', + 'broccoli', 'broach', 'broom', 'brownie', 'brussels_sprouts', + 'bubble_gum', 'bucket', 'horse_buggy', 'bull', 'bulldog', 'bulldozer', + 'bullet_train', 'bulletin_board', 'bulletproof_vest', 'bullhorn', + 'bun', 'bunk_bed', 'buoy', 'burrito', 'bus_(vehicle)', 'business_card', + 'butter', 'butterfly', 'button', 'cab_(taxi)', 'cabana', 'cabin_car', + 'cabinet', 'locker', 'cake', 'calculator', 'calendar', 'calf', + 'camcorder', 'camel', 'camera', 'camera_lens', 'camper_(vehicle)', + 'can', 'can_opener', 'candle', 'candle_holder', 'candy_bar', + 'candy_cane', 'walking_cane', 'canister', 'canoe', 'cantaloup', + 'canteen', 'cap_(headwear)', 'bottle_cap', 'cape', 'cappuccino', + 'car_(automobile)', 'railcar_(part_of_a_train)', 'elevator_car', + 'car_battery', 'identity_card', 'card', 'cardigan', 'cargo_ship', + 'carnation', 'horse_carriage', 'carrot', 'tote_bag', 'cart', 'carton', + 'cash_register', 'casserole', 'cassette', 'cast', 'cat', 'cauliflower', + 'cayenne_(spice)', 'CD_player', 'celery', 'cellular_telephone', + 'chain_mail', 'chair', 'chaise_longue', 'chalice', 'chandelier', + 'chap', 'checkbook', 'checkerboard', 'cherry', 'chessboard', + 'chicken_(animal)', 'chickpea', 'chili_(vegetable)', 'chime', + 'chinaware', 'crisp_(potato_chip)', 'poker_chip', 'chocolate_bar', + 'chocolate_cake', 'chocolate_milk', 'chocolate_mousse', 'choker', + 'chopping_board', 'chopstick', 'Christmas_tree', 'slide', 'cider', + 'cigar_box', 'cigarette', 'cigarette_case', 'cistern', 'clarinet', + 'clasp', 'cleansing_agent', 'cleat_(for_securing_rope)', 'clementine', + 'clip', 'clipboard', 'clippers_(for_plants)', 'cloak', 'clock', + 'clock_tower', 'clothes_hamper', 'clothespin', 'clutch_bag', 'coaster', + 'coat', 'coat_hanger', 'coatrack', 'cock', 'cockroach', + 'cocoa_(beverage)', 'coconut', 'coffee_maker', 'coffee_table', + 'coffeepot', 'coil', 'coin', 'colander', 'coleslaw', + 'coloring_material', 'combination_lock', 'pacifier', 'comic_book', + 'compass', 'computer_keyboard', 'condiment', 'cone', 'control', + 'convertible_(automobile)', 'sofa_bed', 'cooker', 'cookie', + 'cooking_utensil', 'cooler_(for_food)', 'cork_(bottle_plug)', + 'corkboard', 'corkscrew', 'edible_corn', 'cornbread', 'cornet', + 'cornice', 'cornmeal', 'corset', 'costume', 'cougar', 'coverall', + 'cowbell', 'cowboy_hat', 'crab_(animal)', 'crabmeat', 'cracker', + 'crape', 'crate', 'crayon', 'cream_pitcher', 'crescent_roll', 'crib', + 'crock_pot', 'crossbar', 'crouton', 'crow', 'crowbar', 'crown', + 'crucifix', 'cruise_ship', 'police_cruiser', 'crumb', 'crutch', + 'cub_(animal)', 'cube', 'cucumber', 'cufflink', 'cup', 'trophy_cup', + 'cupboard', 'cupcake', 'hair_curler', 'curling_iron', 'curtain', + 'cushion', 'cylinder', 'cymbal', 'dagger', 'dalmatian', 'dartboard', + 'date_(fruit)', 'deck_chair', 'deer', 'dental_floss', 'desk', + 'detergent', 'diaper', 'diary', 'die', 'dinghy', 'dining_table', 'tux', + 'dish', 'dish_antenna', 'dishrag', 'dishtowel', 'dishwasher', + 'dishwasher_detergent', 'dispenser', 'diving_board', 'Dixie_cup', + 'dog', 'dog_collar', 'doll', 'dollar', 'dollhouse', 'dolphin', + 'domestic_ass', 'doorknob', 'doormat', 'doughnut', 'dove', 'dragonfly', + 'drawer', 'underdrawers', 'dress', 'dress_hat', 'dress_suit', + 'dresser', 'drill', 'drone', 'dropper', 'drum_(musical_instrument)', + 'drumstick', 'duck', 'duckling', 'duct_tape', 'duffel_bag', 'dumbbell', + 'dumpster', 'dustpan', 'eagle', 'earphone', 'earplug', 'earring', + 'easel', 'eclair', 'eel', 'egg', 'egg_roll', 'egg_yolk', 'eggbeater', + 'eggplant', 'electric_chair', 'refrigerator', 'elephant', 'elk', + 'envelope', 'eraser', 'escargot', 'eyepatch', 'falcon', 'fan', + 'faucet', 'fedora', 'ferret', 'Ferris_wheel', 'ferry', 'fig_(fruit)', + 'fighter_jet', 'figurine', 'file_cabinet', 'file_(tool)', 'fire_alarm', + 'fire_engine', 'fire_extinguisher', 'fire_hose', 'fireplace', + 'fireplug', 'first-aid_kit', 'fish', 'fish_(food)', 'fishbowl', + 'fishing_rod', 'flag', 'flagpole', 'flamingo', 'flannel', 'flap', + 'flash', 'flashlight', 'fleece', 'flip-flop_(sandal)', + 'flipper_(footwear)', 'flower_arrangement', 'flute_glass', 'foal', + 'folding_chair', 'food_processor', 'football_(American)', + 'football_helmet', 'footstool', 'fork', 'forklift', 'freight_car', + 'French_toast', 'freshener', 'frisbee', 'frog', 'fruit_juice', + 'frying_pan', 'fudge', 'funnel', 'futon', 'gag', 'garbage', + 'garbage_truck', 'garden_hose', 'gargle', 'gargoyle', 'garlic', + 'gasmask', 'gazelle', 'gelatin', 'gemstone', 'generator', + 'giant_panda', 'gift_wrap', 'ginger', 'giraffe', 'cincture', + 'glass_(drink_container)', 'globe', 'glove', 'goat', 'goggles', + 'goldfish', 'golf_club', 'golfcart', 'gondola_(boat)', 'goose', + 'gorilla', 'gourd', 'grape', 'grater', 'gravestone', 'gravy_boat', + 'green_bean', 'green_onion', 'griddle', 'grill', 'grits', 'grizzly', + 'grocery_bag', 'guitar', 'gull', 'gun', 'hairbrush', 'hairnet', + 'hairpin', 'halter_top', 'ham', 'hamburger', 'hammer', 'hammock', + 'hamper', 'hamster', 'hair_dryer', 'hand_glass', 'hand_towel', + 'handcart', 'handcuff', 'handkerchief', 'handle', 'handsaw', + 'hardback_book', 'harmonium', 'hat', 'hatbox', 'veil', 'headband', + 'headboard', 'headlight', 'headscarf', 'headset', + 'headstall_(for_horses)', 'heart', 'heater', 'helicopter', 'helmet', + 'heron', 'highchair', 'hinge', 'hippopotamus', 'hockey_stick', 'hog', + 'home_plate_(baseball)', 'honey', 'fume_hood', 'hook', 'hookah', + 'hornet', 'horse', 'hose', 'hot-air_balloon', 'hotplate', 'hot_sauce', + 'hourglass', 'houseboat', 'hummingbird', 'hummus', 'polar_bear', + 'icecream', 'popsicle', 'ice_maker', 'ice_pack', 'ice_skate', + 'igniter', 'inhaler', 'iPod', 'iron_(for_clothing)', 'ironing_board', + 'jacket', 'jam', 'jar', 'jean', 'jeep', 'jelly_bean', 'jersey', + 'jet_plane', 'jewel', 'jewelry', 'joystick', 'jumpsuit', 'kayak', + 'keg', 'kennel', 'kettle', 'key', 'keycard', 'kilt', 'kimono', + 'kitchen_sink', 'kitchen_table', 'kite', 'kitten', 'kiwi_fruit', + 'knee_pad', 'knife', 'knitting_needle', 'knob', 'knocker_(on_a_door)', + 'koala', 'lab_coat', 'ladder', 'ladle', 'ladybug', 'lamb_(animal)', + 'lamb-chop', 'lamp', 'lamppost', 'lampshade', 'lantern', 'lanyard', + 'laptop_computer', 'lasagna', 'latch', 'lawn_mower', 'leather', + 'legging_(clothing)', 'Lego', 'legume', 'lemon', 'lemonade', 'lettuce', + 'license_plate', 'life_buoy', 'life_jacket', 'lightbulb', + 'lightning_rod', 'lime', 'limousine', 'lion', 'lip_balm', 'liquor', + 'lizard', 'log', 'lollipop', 'speaker_(stero_equipment)', 'loveseat', + 'machine_gun', 'magazine', 'magnet', 'mail_slot', 'mailbox_(at_home)', + 'mallard', 'mallet', 'mammoth', 'manatee', 'mandarin_orange', 'manger', + 'manhole', 'map', 'marker', 'martini', 'mascot', 'mashed_potato', + 'masher', 'mask', 'mast', 'mat_(gym_equipment)', 'matchbox', + 'mattress', 'measuring_cup', 'measuring_stick', 'meatball', 'medicine', + 'melon', 'microphone', 'microscope', 'microwave_oven', 'milestone', + 'milk', 'milk_can', 'milkshake', 'minivan', 'mint_candy', 'mirror', + 'mitten', 'mixer_(kitchen_tool)', 'money', + 'monitor_(computer_equipment) computer_monitor', 'monkey', 'motor', + 'motor_scooter', 'motor_vehicle', 'motorcycle', 'mound_(baseball)', + 'mouse_(computer_equipment)', 'mousepad', 'muffin', 'mug', 'mushroom', + 'music_stool', 'musical_instrument', 'nailfile', 'napkin', + 'neckerchief', 'necklace', 'necktie', 'needle', 'nest', 'newspaper', + 'newsstand', 'nightshirt', 'nosebag_(for_animals)', + 'noseband_(for_animals)', 'notebook', 'notepad', 'nut', 'nutcracker', + 'oar', 'octopus_(food)', 'octopus_(animal)', 'oil_lamp', 'olive_oil', + 'omelet', 'onion', 'orange_(fruit)', 'orange_juice', 'ostrich', + 'ottoman', 'oven', 'overalls_(clothing)', 'owl', 'packet', 'inkpad', + 'pad', 'paddle', 'padlock', 'paintbrush', 'painting', 'pajamas', + 'palette', 'pan_(for_cooking)', 'pan_(metal_container)', 'pancake', + 'pantyhose', 'papaya', 'paper_plate', 'paper_towel', 'paperback_book', + 'paperweight', 'parachute', 'parakeet', 'parasail_(sports)', 'parasol', + 'parchment', 'parka', 'parking_meter', 'parrot', + 'passenger_car_(part_of_a_train)', 'passenger_ship', 'passport', + 'pastry', 'patty_(food)', 'pea_(food)', 'peach', 'peanut_butter', + 'pear', 'peeler_(tool_for_fruit_and_vegetables)', 'wooden_leg', + 'pegboard', 'pelican', 'pen', 'pencil', 'pencil_box', + 'pencil_sharpener', 'pendulum', 'penguin', 'pennant', 'penny_(coin)', + 'pepper', 'pepper_mill', 'perfume', 'persimmon', 'person', 'pet', + 'pew_(church_bench)', 'phonebook', 'phonograph_record', 'piano', + 'pickle', 'pickup_truck', 'pie', 'pigeon', 'piggy_bank', 'pillow', + 'pin_(non_jewelry)', 'pineapple', 'pinecone', 'ping-pong_ball', + 'pinwheel', 'tobacco_pipe', 'pipe', 'pistol', 'pita_(bread)', + 'pitcher_(vessel_for_liquid)', 'pitchfork', 'pizza', 'place_mat', + 'plate', 'platter', 'playpen', 'pliers', 'plow_(farm_equipment)', + 'plume', 'pocket_watch', 'pocketknife', 'poker_(fire_stirring_tool)', + 'pole', 'polo_shirt', 'poncho', 'pony', 'pool_table', 'pop_(soda)', + 'postbox_(public)', 'postcard', 'poster', 'pot', 'flowerpot', 'potato', + 'potholder', 'pottery', 'pouch', 'power_shovel', 'prawn', 'pretzel', + 'printer', 'projectile_(weapon)', 'projector', 'propeller', 'prune', + 'pudding', 'puffer_(fish)', 'puffin', 'pug-dog', 'pumpkin', 'puncher', + 'puppet', 'puppy', 'quesadilla', 'quiche', 'quilt', 'rabbit', + 'race_car', 'racket', 'radar', 'radiator', 'radio_receiver', 'radish', + 'raft', 'rag_doll', 'raincoat', 'ram_(animal)', 'raspberry', 'rat', + 'razorblade', 'reamer_(juicer)', 'rearview_mirror', 'receipt', + 'recliner', 'record_player', 'reflector', 'remote_control', + 'rhinoceros', 'rib_(food)', 'rifle', 'ring', 'river_boat', 'road_map', + 'robe', 'rocking_chair', 'rodent', 'roller_skate', 'Rollerblade', + 'rolling_pin', 'root_beer', 'router_(computer_equipment)', + 'rubber_band', 'runner_(carpet)', 'plastic_bag', + 'saddle_(on_an_animal)', 'saddle_blanket', 'saddlebag', 'safety_pin', + 'sail', 'salad', 'salad_plate', 'salami', 'salmon_(fish)', + 'salmon_(food)', 'salsa', 'saltshaker', 'sandal_(type_of_shoe)', + 'sandwich', 'satchel', 'saucepan', 'saucer', 'sausage', 'sawhorse', + 'saxophone', 'scale_(measuring_instrument)', 'scarecrow', 'scarf', + 'school_bus', 'scissors', 'scoreboard', 'scraper', 'screwdriver', + 'scrubbing_brush', 'sculpture', 'seabird', 'seahorse', 'seaplane', + 'seashell', 'sewing_machine', 'shaker', 'shampoo', 'shark', + 'sharpener', 'Sharpie', 'shaver_(electric)', 'shaving_cream', 'shawl', + 'shears', 'sheep', 'shepherd_dog', 'sherbert', 'shield', 'shirt', + 'shoe', 'shopping_bag', 'shopping_cart', 'short_pants', 'shot_glass', + 'shoulder_bag', 'shovel', 'shower_head', 'shower_cap', + 'shower_curtain', 'shredder_(for_paper)', 'signboard', 'silo', 'sink', + 'skateboard', 'skewer', 'ski', 'ski_boot', 'ski_parka', 'ski_pole', + 'skirt', 'skullcap', 'sled', 'sleeping_bag', 'sling_(bandage)', + 'slipper_(footwear)', 'smoothie', 'snake', 'snowboard', 'snowman', + 'snowmobile', 'soap', 'soccer_ball', 'sock', 'sofa', 'softball', + 'solar_array', 'sombrero', 'soup', 'soup_bowl', 'soupspoon', + 'sour_cream', 'soya_milk', 'space_shuttle', 'sparkler_(fireworks)', + 'spatula', 'spear', 'spectacles', 'spice_rack', 'spider', 'crawfish', + 'sponge', 'spoon', 'sportswear', 'spotlight', 'squid_(food)', + 'squirrel', 'stagecoach', 'stapler_(stapling_machine)', 'starfish', + 'statue_(sculpture)', 'steak_(food)', 'steak_knife', 'steering_wheel', + 'stepladder', 'step_stool', 'stereo_(sound_system)', 'stew', 'stirrer', + 'stirrup', 'stool', 'stop_sign', 'brake_light', 'stove', 'strainer', + 'strap', 'straw_(for_drinking)', 'strawberry', 'street_sign', + 'streetlight', 'string_cheese', 'stylus', 'subwoofer', 'sugar_bowl', + 'sugarcane_(plant)', 'suit_(clothing)', 'sunflower', 'sunglasses', + 'sunhat', 'surfboard', 'sushi', 'mop', 'sweat_pants', 'sweatband', + 'sweater', 'sweatshirt', 'sweet_potato', 'swimsuit', 'sword', + 'syringe', 'Tabasco_sauce', 'table-tennis_table', 'table', + 'table_lamp', 'tablecloth', 'tachometer', 'taco', 'tag', 'taillight', + 'tambourine', 'army_tank', 'tank_(storage_vessel)', + 'tank_top_(clothing)', 'tape_(sticky_cloth_or_paper)', 'tape_measure', + 'tapestry', 'tarp', 'tartan', 'tassel', 'tea_bag', 'teacup', + 'teakettle', 'teapot', 'teddy_bear', 'telephone', 'telephone_booth', + 'telephone_pole', 'telephoto_lens', 'television_camera', + 'television_set', 'tennis_ball', 'tennis_racket', 'tequila', + 'thermometer', 'thermos_bottle', 'thermostat', 'thimble', 'thread', + 'thumbtack', 'tiara', 'tiger', 'tights_(clothing)', 'timer', 'tinfoil', + 'tinsel', 'tissue_paper', 'toast_(food)', 'toaster', 'toaster_oven', + 'toilet', 'toilet_tissue', 'tomato', 'tongs', 'toolbox', 'toothbrush', + 'toothpaste', 'toothpick', 'cover', 'tortilla', 'tow_truck', 'towel', + 'towel_rack', 'toy', 'tractor_(farm_equipment)', 'traffic_light', + 'dirt_bike', 'trailer_truck', 'train_(railroad_vehicle)', 'trampoline', + 'tray', 'trench_coat', 'triangle_(musical_instrument)', 'tricycle', + 'tripod', 'trousers', 'truck', 'truffle_(chocolate)', 'trunk', 'vat', + 'turban', 'turkey_(food)', 'turnip', 'turtle', 'turtleneck_(clothing)', + 'typewriter', 'umbrella', 'underwear', 'unicycle', 'urinal', 'urn', + 'vacuum_cleaner', 'vase', 'vending_machine', 'vent', 'vest', + 'videotape', 'vinegar', 'violin', 'vodka', 'volleyball', 'vulture', + 'waffle', 'waffle_iron', 'wagon', 'wagon_wheel', 'walking_stick', + 'wall_clock', 'wall_socket', 'wallet', 'walrus', 'wardrobe', + 'washbasin', 'automatic_washer', 'watch', 'water_bottle', + 'water_cooler', 'water_faucet', 'water_heater', 'water_jug', + 'water_gun', 'water_scooter', 'water_ski', 'water_tower', + 'watering_can', 'watermelon', 'weathervane', 'webcam', 'wedding_cake', + 'wedding_ring', 'wet_suit', 'wheel', 'wheelchair', 'whipped_cream', + 'whistle', 'wig', 'wind_chime', 'windmill', 'window_box_(for_plants)', + 'windshield_wiper', 'windsock', 'wine_bottle', 'wine_bucket', + 'wineglass', 'blinder_(for_horses)', 'wok', 'wolf', 'wooden_spoon', + 'wreath', 'wrench', 'wristband', 'wristlet', 'yacht', 'yogurt', + 'yoke_(animal_equipment)', 'zebra', 'zucchini') + + def load_annotations(self, ann_file): + try: + import lvis + assert lvis.__version__ >= '10.5.3' + from lvis import LVIS + except AssertionError: + raise AssertionError('Incompatible version of lvis is installed. ' + 'Run pip uninstall lvis first. Then run pip ' + 'install mmlvis to install open-mmlab forked ' + 'lvis. ') + except ImportError: + raise ImportError('Package lvis is not installed. Please run pip ' + 'install mmlvis to install open-mmlab forked ' + 'lvis.') + self.coco = LVIS(ann_file) + self.cat_ids = self.coco.get_cat_ids() + self.cat2label = {cat_id: i for i, cat_id in enumerate(self.cat_ids)} + self.img_ids = self.coco.get_img_ids() + data_infos = [] + for i in self.img_ids: + info = self.coco.load_imgs([i])[0] + # coco_url is used in LVISv1 instead of file_name + # e.g. http://images.cocodataset.org/train2017/000000391895.jpg + # train/val split in specified in url + info['filename'] = info['coco_url'].replace( + 'http://images.cocodataset.org/', '') + data_infos.append(info) + return data_infos diff --git a/detection/mmdet/datasets/pipelines/__init__.py b/detection/mmdet/datasets/pipelines/__init__.py new file mode 100644 index 0000000..c6f424d --- /dev/null +++ b/detection/mmdet/datasets/pipelines/__init__.py @@ -0,0 +1,25 @@ +from .auto_augment import (AutoAugment, BrightnessTransform, ColorTransform, + ContrastTransform, EqualizeTransform, Rotate, Shear, + Translate) +from .compose import Compose +from .formating import (Collect, DefaultFormatBundle, ImageToTensor, + ToDataContainer, ToTensor, Transpose, to_tensor) +from .instaboost import InstaBoost +from .loading import (LoadAnnotations, LoadImageFromFile, LoadImageFromWebcam, + LoadMultiChannelImageFromFiles, LoadProposals) +from .test_time_aug import MultiScaleFlipAug +from .transforms import (Albu, CutOut, Expand, MinIoURandomCrop, Normalize, + Pad, PhotoMetricDistortion, RandomCenterCropPad, + RandomCrop, RandomFlip, Resize, SegRescale) + +__all__ = [ + 'Compose', 'to_tensor', 'ToTensor', 'ImageToTensor', 'ToDataContainer', + 'Transpose', 'Collect', 'DefaultFormatBundle', 'LoadAnnotations', + 'LoadImageFromFile', 'LoadImageFromWebcam', + 'LoadMultiChannelImageFromFiles', 'LoadProposals', 'MultiScaleFlipAug', + 'Resize', 'RandomFlip', 'Pad', 'RandomCrop', 'Normalize', 'SegRescale', + 'MinIoURandomCrop', 'Expand', 'PhotoMetricDistortion', 'Albu', + 'InstaBoost', 'RandomCenterCropPad', 'AutoAugment', 'CutOut', 'Shear', + 'Rotate', 'ColorTransform', 'EqualizeTransform', 'BrightnessTransform', + 'ContrastTransform', 'Translate' +] diff --git a/detection/mmdet/datasets/pipelines/auto_augment.py b/detection/mmdet/datasets/pipelines/auto_augment.py new file mode 100644 index 0000000..e19adae --- /dev/null +++ b/detection/mmdet/datasets/pipelines/auto_augment.py @@ -0,0 +1,890 @@ +import copy + +import cv2 +import mmcv +import numpy as np + +from ..builder import PIPELINES +from .compose import Compose + +_MAX_LEVEL = 10 + + +def level_to_value(level, max_value): + """Map from level to values based on max_value.""" + return (level / _MAX_LEVEL) * max_value + + +def enhance_level_to_value(level, a=1.8, b=0.1): + """Map from level to values.""" + return (level / _MAX_LEVEL) * a + b + + +def random_negative(value, random_negative_prob): + """Randomly negate value based on random_negative_prob.""" + return -value if np.random.rand() < random_negative_prob else value + + +def bbox2fields(): + """The key correspondence from bboxes to labels, masks and + segmentations.""" + bbox2label = { + 'gt_bboxes': 'gt_labels', + 'gt_bboxes_ignore': 'gt_labels_ignore' + } + bbox2mask = { + 'gt_bboxes': 'gt_masks', + 'gt_bboxes_ignore': 'gt_masks_ignore' + } + bbox2seg = { + 'gt_bboxes': 'gt_semantic_seg', + } + return bbox2label, bbox2mask, bbox2seg + + +@PIPELINES.register_module() +class AutoAugment(object): + """Auto augmentation. + + This data augmentation is proposed in `Learning Data Augmentation + Strategies for Object Detection `_. + + TODO: Implement 'Shear', 'Sharpness' and 'Rotate' transforms + + Args: + policies (list[list[dict]]): The policies of auto augmentation. Each + policy in ``policies`` is a specific augmentation policy, and is + composed by several augmentations (dict). When AutoAugment is + called, a random policy in ``policies`` will be selected to + augment images. + + Examples: + >>> replace = (104, 116, 124) + >>> policies = [ + >>> [ + >>> dict(type='Sharpness', prob=0.0, level=8), + >>> dict( + >>> type='Shear', + >>> prob=0.4, + >>> level=0, + >>> replace=replace, + >>> axis='x') + >>> ], + >>> [ + >>> dict( + >>> type='Rotate', + >>> prob=0.6, + >>> level=10, + >>> replace=replace), + >>> dict(type='Color', prob=1.0, level=6) + >>> ] + >>> ] + >>> augmentation = AutoAugment(policies) + >>> img = np.ones(100, 100, 3) + >>> gt_bboxes = np.ones(10, 4) + >>> results = dict(img=img, gt_bboxes=gt_bboxes) + >>> results = augmentation(results) + """ + + def __init__(self, policies): + assert isinstance(policies, list) and len(policies) > 0, \ + 'Policies must be a non-empty list.' + for policy in policies: + assert isinstance(policy, list) and len(policy) > 0, \ + 'Each policy in policies must be a non-empty list.' + for augment in policy: + assert isinstance(augment, dict) and 'type' in augment, \ + 'Each specific augmentation must be a dict with key' \ + ' "type".' + + self.policies = copy.deepcopy(policies) + self.transforms = [Compose(policy) for policy in self.policies] + + def __call__(self, results): + transform = np.random.choice(self.transforms) + return transform(results) + + def __repr__(self): + return f'{self.__class__.__name__}(policies={self.policies})' + + +@PIPELINES.register_module() +class Shear(object): + """Apply Shear Transformation to image (and its corresponding bbox, mask, + segmentation). + + Args: + level (int | float): The level should be in range [0,_MAX_LEVEL]. + img_fill_val (int | float | tuple): The filled values for image border. + If float, the same fill value will be used for all the three + channels of image. If tuple, the should be 3 elements. + seg_ignore_label (int): The fill value used for segmentation map. + Note this value must equals ``ignore_label`` in ``semantic_head`` + of the corresponding config. Default 255. + prob (float): The probability for performing Shear and should be in + range [0, 1]. + direction (str): The direction for shear, either "horizontal" + or "vertical". + max_shear_magnitude (float): The maximum magnitude for Shear + transformation. + random_negative_prob (float): The probability that turns the + offset negative. Should be in range [0,1] + interpolation (str): Same as in :func:`mmcv.imshear`. + """ + + def __init__(self, + level, + img_fill_val=128, + seg_ignore_label=255, + prob=0.5, + direction='horizontal', + max_shear_magnitude=0.3, + random_negative_prob=0.5, + interpolation='bilinear'): + assert isinstance(level, (int, float)), 'The level must be type ' \ + f'int or float, got {type(level)}.' + assert 0 <= level <= _MAX_LEVEL, 'The level should be in range ' \ + f'[0,{_MAX_LEVEL}], got {level}.' + if isinstance(img_fill_val, (float, int)): + img_fill_val = tuple([float(img_fill_val)] * 3) + elif isinstance(img_fill_val, tuple): + assert len(img_fill_val) == 3, 'img_fill_val as tuple must ' \ + f'have 3 elements. got {len(img_fill_val)}.' + img_fill_val = tuple([float(val) for val in img_fill_val]) + else: + raise ValueError( + 'img_fill_val must be float or tuple with 3 elements.') + assert np.all([0 <= val <= 255 for val in img_fill_val]), 'all ' \ + 'elements of img_fill_val should between range [0,255].' \ + f'got {img_fill_val}.' + assert 0 <= prob <= 1.0, 'The probability of shear should be in ' \ + f'range [0,1]. got {prob}.' + assert direction in ('horizontal', 'vertical'), 'direction must ' \ + f'in be either "horizontal" or "vertical". got {direction}.' + assert isinstance(max_shear_magnitude, float), 'max_shear_magnitude ' \ + f'should be type float. got {type(max_shear_magnitude)}.' + assert 0. <= max_shear_magnitude <= 1., 'Defaultly ' \ + 'max_shear_magnitude should be in range [0,1]. ' \ + f'got {max_shear_magnitude}.' + self.level = level + self.magnitude = level_to_value(level, max_shear_magnitude) + self.img_fill_val = img_fill_val + self.seg_ignore_label = seg_ignore_label + self.prob = prob + self.direction = direction + self.max_shear_magnitude = max_shear_magnitude + self.random_negative_prob = random_negative_prob + self.interpolation = interpolation + + def _shear_img(self, + results, + magnitude, + direction='horizontal', + interpolation='bilinear'): + """Shear the image. + + Args: + results (dict): Result dict from loading pipeline. + magnitude (int | float): The magnitude used for shear. + direction (str): The direction for shear, either "horizontal" + or "vertical". + interpolation (str): Same as in :func:`mmcv.imshear`. + """ + for key in results.get('img_fields', ['img']): + img = results[key] + img_sheared = mmcv.imshear( + img, + magnitude, + direction, + border_value=self.img_fill_val, + interpolation=interpolation) + results[key] = img_sheared.astype(img.dtype) + + def _shear_bboxes(self, results, magnitude): + """Shear the bboxes.""" + h, w, c = results['img_shape'] + if self.direction == 'horizontal': + shear_matrix = np.stack([[1, magnitude], + [0, 1]]).astype(np.float32) # [2, 2] + else: + shear_matrix = np.stack([[1, 0], [magnitude, + 1]]).astype(np.float32) + for key in results.get('bbox_fields', []): + min_x, min_y, max_x, max_y = np.split( + results[key], results[key].shape[-1], axis=-1) + coordinates = np.stack([[min_x, min_y], [max_x, min_y], + [min_x, max_y], + [max_x, max_y]]) # [4, 2, nb_box, 1] + coordinates = coordinates[..., 0].transpose( + (2, 1, 0)).astype(np.float32) # [nb_box, 2, 4] + new_coords = np.matmul(shear_matrix[None, :, :], + coordinates) # [nb_box, 2, 4] + min_x = np.min(new_coords[:, 0, :], axis=-1) + min_y = np.min(new_coords[:, 1, :], axis=-1) + max_x = np.max(new_coords[:, 0, :], axis=-1) + max_y = np.max(new_coords[:, 1, :], axis=-1) + min_x = np.clip(min_x, a_min=0, a_max=w) + min_y = np.clip(min_y, a_min=0, a_max=h) + max_x = np.clip(max_x, a_min=min_x, a_max=w) + max_y = np.clip(max_y, a_min=min_y, a_max=h) + results[key] = np.stack([min_x, min_y, max_x, max_y], + axis=-1).astype(results[key].dtype) + + def _shear_masks(self, + results, + magnitude, + direction='horizontal', + fill_val=0, + interpolation='bilinear'): + """Shear the masks.""" + h, w, c = results['img_shape'] + for key in results.get('mask_fields', []): + masks = results[key] + results[key] = masks.shear((h, w), + magnitude, + direction, + border_value=fill_val, + interpolation=interpolation) + + def _shear_seg(self, + results, + magnitude, + direction='horizontal', + fill_val=255, + interpolation='bilinear'): + """Shear the segmentation maps.""" + for key in results.get('seg_fields', []): + seg = results[key] + results[key] = mmcv.imshear( + seg, + magnitude, + direction, + border_value=fill_val, + interpolation=interpolation).astype(seg.dtype) + + def _filter_invalid(self, results, min_bbox_size=0): + """Filter bboxes and corresponding masks too small after shear + augmentation.""" + bbox2label, bbox2mask, _ = bbox2fields() + for key in results.get('bbox_fields', []): + bbox_w = results[key][:, 2] - results[key][:, 0] + bbox_h = results[key][:, 3] - results[key][:, 1] + valid_inds = (bbox_w > min_bbox_size) & (bbox_h > min_bbox_size) + valid_inds = np.nonzero(valid_inds)[0] + results[key] = results[key][valid_inds] + # label fields. e.g. gt_labels and gt_labels_ignore + label_key = bbox2label.get(key) + if label_key in results: + results[label_key] = results[label_key][valid_inds] + # mask fields, e.g. gt_masks and gt_masks_ignore + mask_key = bbox2mask.get(key) + if mask_key in results: + results[mask_key] = results[mask_key][valid_inds] + + def __call__(self, results): + """Call function to shear images, bounding boxes, masks and semantic + segmentation maps. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Sheared results. + """ + if np.random.rand() > self.prob: + return results + magnitude = random_negative(self.magnitude, self.random_negative_prob) + self._shear_img(results, magnitude, self.direction, self.interpolation) + self._shear_bboxes(results, magnitude) + # fill_val set to 0 for background of mask. + self._shear_masks( + results, + magnitude, + self.direction, + fill_val=0, + interpolation=self.interpolation) + self._shear_seg( + results, + magnitude, + self.direction, + fill_val=self.seg_ignore_label, + interpolation=self.interpolation) + self._filter_invalid(results) + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(level={self.level}, ' + repr_str += f'img_fill_val={self.img_fill_val}, ' + repr_str += f'seg_ignore_label={self.seg_ignore_label}, ' + repr_str += f'prob={self.prob}, ' + repr_str += f'direction={self.direction}, ' + repr_str += f'max_shear_magnitude={self.max_shear_magnitude}, ' + repr_str += f'random_negative_prob={self.random_negative_prob}, ' + repr_str += f'interpolation={self.interpolation})' + return repr_str + + +@PIPELINES.register_module() +class Rotate(object): + """Apply Rotate Transformation to image (and its corresponding bbox, mask, + segmentation). + + Args: + level (int | float): The level should be in range (0,_MAX_LEVEL]. + scale (int | float): Isotropic scale factor. Same in + ``mmcv.imrotate``. + center (int | float | tuple[float]): Center point (w, h) of the + rotation in the source image. If None, the center of the + image will be used. Same in ``mmcv.imrotate``. + img_fill_val (int | float | tuple): The fill value for image border. + If float, the same value will be used for all the three + channels of image. If tuple, the should be 3 elements (e.g. + equals the number of channels for image). + seg_ignore_label (int): The fill value used for segmentation map. + Note this value must equals ``ignore_label`` in ``semantic_head`` + of the corresponding config. Default 255. + prob (float): The probability for perform transformation and + should be in range 0 to 1. + max_rotate_angle (int | float): The maximum angles for rotate + transformation. + random_negative_prob (float): The probability that turns the + offset negative. + """ + + def __init__(self, + level, + scale=1, + center=None, + img_fill_val=128, + seg_ignore_label=255, + prob=0.5, + max_rotate_angle=30, + random_negative_prob=0.5): + assert isinstance(level, (int, float)), \ + f'The level must be type int or float. got {type(level)}.' + assert 0 <= level <= _MAX_LEVEL, \ + f'The level should be in range (0,{_MAX_LEVEL}]. got {level}.' + assert isinstance(scale, (int, float)), \ + f'The scale must be type int or float. got type {type(scale)}.' + if isinstance(center, (int, float)): + center = (center, center) + elif isinstance(center, tuple): + assert len(center) == 2, 'center with type tuple must have '\ + f'2 elements. got {len(center)} elements.' + else: + assert center is None, 'center must be None or type int, '\ + f'float or tuple, got type {type(center)}.' + if isinstance(img_fill_val, (float, int)): + img_fill_val = tuple([float(img_fill_val)] * 3) + elif isinstance(img_fill_val, tuple): + assert len(img_fill_val) == 3, 'img_fill_val as tuple must '\ + f'have 3 elements. got {len(img_fill_val)}.' + img_fill_val = tuple([float(val) for val in img_fill_val]) + else: + raise ValueError( + 'img_fill_val must be float or tuple with 3 elements.') + assert np.all([0 <= val <= 255 for val in img_fill_val]), \ + 'all elements of img_fill_val should between range [0,255]. '\ + f'got {img_fill_val}.' + assert 0 <= prob <= 1.0, 'The probability should be in range [0,1]. '\ + 'got {prob}.' + assert isinstance(max_rotate_angle, (int, float)), 'max_rotate_angle '\ + f'should be type int or float. got type {type(max_rotate_angle)}.' + self.level = level + self.scale = scale + # Rotation angle in degrees. Positive values mean + # clockwise rotation. + self.angle = level_to_value(level, max_rotate_angle) + self.center = center + self.img_fill_val = img_fill_val + self.seg_ignore_label = seg_ignore_label + self.prob = prob + self.max_rotate_angle = max_rotate_angle + self.random_negative_prob = random_negative_prob + + def _rotate_img(self, results, angle, center=None, scale=1.0): + """Rotate the image. + + Args: + results (dict): Result dict from loading pipeline. + angle (float): Rotation angle in degrees, positive values + mean clockwise rotation. Same in ``mmcv.imrotate``. + center (tuple[float], optional): Center point (w, h) of the + rotation. Same in ``mmcv.imrotate``. + scale (int | float): Isotropic scale factor. Same in + ``mmcv.imrotate``. + """ + for key in results.get('img_fields', ['img']): + img = results[key].copy() + img_rotated = mmcv.imrotate( + img, angle, center, scale, border_value=self.img_fill_val) + results[key] = img_rotated.astype(img.dtype) + + def _rotate_bboxes(self, results, rotate_matrix): + """Rotate the bboxes.""" + h, w, c = results['img_shape'] + for key in results.get('bbox_fields', []): + min_x, min_y, max_x, max_y = np.split( + results[key], results[key].shape[-1], axis=-1) + coordinates = np.stack([[min_x, min_y], [max_x, min_y], + [min_x, max_y], + [max_x, max_y]]) # [4, 2, nb_bbox, 1] + # pad 1 to convert from format [x, y] to homogeneous + # coordinates format [x, y, 1] + coordinates = np.concatenate( + (coordinates, + np.ones((4, 1, coordinates.shape[2], 1), coordinates.dtype)), + axis=1) # [4, 3, nb_bbox, 1] + coordinates = coordinates.transpose( + (2, 0, 1, 3)) # [nb_bbox, 4, 3, 1] + rotated_coords = np.matmul(rotate_matrix, + coordinates) # [nb_bbox, 4, 2, 1] + rotated_coords = rotated_coords[..., 0] # [nb_bbox, 4, 2] + min_x, min_y = np.min( + rotated_coords[:, :, 0], axis=1), np.min( + rotated_coords[:, :, 1], axis=1) + max_x, max_y = np.max( + rotated_coords[:, :, 0], axis=1), np.max( + rotated_coords[:, :, 1], axis=1) + min_x, min_y = np.clip( + min_x, a_min=0, a_max=w), np.clip( + min_y, a_min=0, a_max=h) + max_x, max_y = np.clip( + max_x, a_min=min_x, a_max=w), np.clip( + max_y, a_min=min_y, a_max=h) + results[key] = np.stack([min_x, min_y, max_x, max_y], + axis=-1).astype(results[key].dtype) + + def _rotate_masks(self, + results, + angle, + center=None, + scale=1.0, + fill_val=0): + """Rotate the masks.""" + h, w, c = results['img_shape'] + for key in results.get('mask_fields', []): + masks = results[key] + results[key] = masks.rotate((h, w), angle, center, scale, fill_val) + + def _rotate_seg(self, + results, + angle, + center=None, + scale=1.0, + fill_val=255): + """Rotate the segmentation map.""" + for key in results.get('seg_fields', []): + seg = results[key].copy() + results[key] = mmcv.imrotate( + seg, angle, center, scale, + border_value=fill_val).astype(seg.dtype) + + def _filter_invalid(self, results, min_bbox_size=0): + """Filter bboxes and corresponding masks too small after rotate + augmentation.""" + bbox2label, bbox2mask, _ = bbox2fields() + for key in results.get('bbox_fields', []): + bbox_w = results[key][:, 2] - results[key][:, 0] + bbox_h = results[key][:, 3] - results[key][:, 1] + valid_inds = (bbox_w > min_bbox_size) & (bbox_h > min_bbox_size) + valid_inds = np.nonzero(valid_inds)[0] + results[key] = results[key][valid_inds] + # label fields. e.g. gt_labels and gt_labels_ignore + label_key = bbox2label.get(key) + if label_key in results: + results[label_key] = results[label_key][valid_inds] + # mask fields, e.g. gt_masks and gt_masks_ignore + mask_key = bbox2mask.get(key) + if mask_key in results: + results[mask_key] = results[mask_key][valid_inds] + + def __call__(self, results): + """Call function to rotate images, bounding boxes, masks and semantic + segmentation maps. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Rotated results. + """ + if np.random.rand() > self.prob: + return results + h, w = results['img'].shape[:2] + center = self.center + if center is None: + center = ((w - 1) * 0.5, (h - 1) * 0.5) + angle = random_negative(self.angle, self.random_negative_prob) + self._rotate_img(results, angle, center, self.scale) + rotate_matrix = cv2.getRotationMatrix2D(center, -angle, self.scale) + self._rotate_bboxes(results, rotate_matrix) + self._rotate_masks(results, angle, center, self.scale, fill_val=0) + self._rotate_seg( + results, angle, center, self.scale, fill_val=self.seg_ignore_label) + self._filter_invalid(results) + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(level={self.level}, ' + repr_str += f'scale={self.scale}, ' + repr_str += f'center={self.center}, ' + repr_str += f'img_fill_val={self.img_fill_val}, ' + repr_str += f'seg_ignore_label={self.seg_ignore_label}, ' + repr_str += f'prob={self.prob}, ' + repr_str += f'max_rotate_angle={self.max_rotate_angle}, ' + repr_str += f'random_negative_prob={self.random_negative_prob})' + return repr_str + + +@PIPELINES.register_module() +class Translate(object): + """Translate the images, bboxes, masks and segmentation maps horizontally + or vertically. + + Args: + level (int | float): The level for Translate and should be in + range [0,_MAX_LEVEL]. + prob (float): The probability for performing translation and + should be in range [0, 1]. + img_fill_val (int | float | tuple): The filled value for image + border. If float, the same fill value will be used for all + the three channels of image. If tuple, the should be 3 + elements (e.g. equals the number of channels for image). + seg_ignore_label (int): The fill value used for segmentation map. + Note this value must equals ``ignore_label`` in ``semantic_head`` + of the corresponding config. Default 255. + direction (str): The translate direction, either "horizontal" + or "vertical". + max_translate_offset (int | float): The maximum pixel's offset for + Translate. + random_negative_prob (float): The probability that turns the + offset negative. + min_size (int | float): The minimum pixel for filtering + invalid bboxes after the translation. + """ + + def __init__(self, + level, + prob=0.5, + img_fill_val=128, + seg_ignore_label=255, + direction='horizontal', + max_translate_offset=250., + random_negative_prob=0.5, + min_size=0): + assert isinstance(level, (int, float)), \ + 'The level must be type int or float.' + assert 0 <= level <= _MAX_LEVEL, \ + 'The level used for calculating Translate\'s offset should be ' \ + 'in range [0,_MAX_LEVEL]' + assert 0 <= prob <= 1.0, \ + 'The probability of translation should be in range [0, 1].' + if isinstance(img_fill_val, (float, int)): + img_fill_val = tuple([float(img_fill_val)] * 3) + elif isinstance(img_fill_val, tuple): + assert len(img_fill_val) == 3, \ + 'img_fill_val as tuple must have 3 elements.' + img_fill_val = tuple([float(val) for val in img_fill_val]) + else: + raise ValueError('img_fill_val must be type float or tuple.') + assert np.all([0 <= val <= 255 for val in img_fill_val]), \ + 'all elements of img_fill_val should between range [0,255].' + assert direction in ('horizontal', 'vertical'), \ + 'direction should be "horizontal" or "vertical".' + assert isinstance(max_translate_offset, (int, float)), \ + 'The max_translate_offset must be type int or float.' + # the offset used for translation + self.offset = int(level_to_value(level, max_translate_offset)) + self.level = level + self.prob = prob + self.img_fill_val = img_fill_val + self.seg_ignore_label = seg_ignore_label + self.direction = direction + self.max_translate_offset = max_translate_offset + self.random_negative_prob = random_negative_prob + self.min_size = min_size + + def _translate_img(self, results, offset, direction='horizontal'): + """Translate the image. + + Args: + results (dict): Result dict from loading pipeline. + offset (int | float): The offset for translate. + direction (str): The translate direction, either "horizontal" + or "vertical". + """ + for key in results.get('img_fields', ['img']): + img = results[key].copy() + results[key] = mmcv.imtranslate( + img, offset, direction, self.img_fill_val).astype(img.dtype) + + def _translate_bboxes(self, results, offset): + """Shift bboxes horizontally or vertically, according to offset.""" + h, w, c = results['img_shape'] + for key in results.get('bbox_fields', []): + min_x, min_y, max_x, max_y = np.split( + results[key], results[key].shape[-1], axis=-1) + if self.direction == 'horizontal': + min_x = np.maximum(0, min_x + offset) + max_x = np.minimum(w, max_x + offset) + elif self.direction == 'vertical': + min_y = np.maximum(0, min_y + offset) + max_y = np.minimum(h, max_y + offset) + + # the boxes translated outside of image will be filtered along with + # the corresponding masks, by invoking ``_filter_invalid``. + results[key] = np.concatenate([min_x, min_y, max_x, max_y], + axis=-1) + + def _translate_masks(self, + results, + offset, + direction='horizontal', + fill_val=0): + """Translate masks horizontally or vertically.""" + h, w, c = results['img_shape'] + for key in results.get('mask_fields', []): + masks = results[key] + results[key] = masks.translate((h, w), offset, direction, fill_val) + + def _translate_seg(self, + results, + offset, + direction='horizontal', + fill_val=255): + """Translate segmentation maps horizontally or vertically.""" + for key in results.get('seg_fields', []): + seg = results[key].copy() + results[key] = mmcv.imtranslate(seg, offset, direction, + fill_val).astype(seg.dtype) + + def _filter_invalid(self, results, min_size=0): + """Filter bboxes and masks too small or translated out of image.""" + bbox2label, bbox2mask, _ = bbox2fields() + for key in results.get('bbox_fields', []): + bbox_w = results[key][:, 2] - results[key][:, 0] + bbox_h = results[key][:, 3] - results[key][:, 1] + valid_inds = (bbox_w > min_size) & (bbox_h > min_size) + valid_inds = np.nonzero(valid_inds)[0] + results[key] = results[key][valid_inds] + # label fields. e.g. gt_labels and gt_labels_ignore + label_key = bbox2label.get(key) + if label_key in results: + results[label_key] = results[label_key][valid_inds] + # mask fields, e.g. gt_masks and gt_masks_ignore + mask_key = bbox2mask.get(key) + if mask_key in results: + results[mask_key] = results[mask_key][valid_inds] + return results + + def __call__(self, results): + """Call function to translate images, bounding boxes, masks and + semantic segmentation maps. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Translated results. + """ + if np.random.rand() > self.prob: + return results + offset = random_negative(self.offset, self.random_negative_prob) + self._translate_img(results, offset, self.direction) + self._translate_bboxes(results, offset) + # fill_val defaultly 0 for BitmapMasks and None for PolygonMasks. + self._translate_masks(results, offset, self.direction) + # fill_val set to ``seg_ignore_label`` for the ignored value + # of segmentation map. + self._translate_seg( + results, offset, self.direction, fill_val=self.seg_ignore_label) + self._filter_invalid(results, min_size=self.min_size) + return results + + +@PIPELINES.register_module() +class ColorTransform(object): + """Apply Color transformation to image. The bboxes, masks, and + segmentations are not modified. + + Args: + level (int | float): Should be in range [0,_MAX_LEVEL]. + prob (float): The probability for performing Color transformation. + """ + + def __init__(self, level, prob=0.5): + assert isinstance(level, (int, float)), \ + 'The level must be type int or float.' + assert 0 <= level <= _MAX_LEVEL, \ + 'The level should be in range [0,_MAX_LEVEL].' + assert 0 <= prob <= 1.0, \ + 'The probability should be in range [0,1].' + self.level = level + self.prob = prob + self.factor = enhance_level_to_value(level) + + def _adjust_color_img(self, results, factor=1.0): + """Apply Color transformation to image.""" + for key in results.get('img_fields', ['img']): + # NOTE defaultly the image should be BGR format + img = results[key] + results[key] = mmcv.adjust_color(img, factor).astype(img.dtype) + + def __call__(self, results): + """Call function for Color transformation. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Colored results. + """ + if np.random.rand() > self.prob: + return results + self._adjust_color_img(results, self.factor) + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(level={self.level}, ' + repr_str += f'prob={self.prob})' + return repr_str + + +@PIPELINES.register_module() +class EqualizeTransform(object): + """Apply Equalize transformation to image. The bboxes, masks and + segmentations are not modified. + + Args: + prob (float): The probability for performing Equalize transformation. + """ + + def __init__(self, prob=0.5): + assert 0 <= prob <= 1.0, \ + 'The probability should be in range [0,1].' + self.prob = prob + + def _imequalize(self, results): + """Equalizes the histogram of one image.""" + for key in results.get('img_fields', ['img']): + img = results[key] + results[key] = mmcv.imequalize(img).astype(img.dtype) + + def __call__(self, results): + """Call function for Equalize transformation. + + Args: + results (dict): Results dict from loading pipeline. + + Returns: + dict: Results after the transformation. + """ + if np.random.rand() > self.prob: + return results + self._imequalize(results) + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(prob={self.prob})' + + +@PIPELINES.register_module() +class BrightnessTransform(object): + """Apply Brightness transformation to image. The bboxes, masks and + segmentations are not modified. + + Args: + level (int | float): Should be in range [0,_MAX_LEVEL]. + prob (float): The probability for performing Brightness transformation. + """ + + def __init__(self, level, prob=0.5): + assert isinstance(level, (int, float)), \ + 'The level must be type int or float.' + assert 0 <= level <= _MAX_LEVEL, \ + 'The level should be in range [0,_MAX_LEVEL].' + assert 0 <= prob <= 1.0, \ + 'The probability should be in range [0,1].' + self.level = level + self.prob = prob + self.factor = enhance_level_to_value(level) + + def _adjust_brightness_img(self, results, factor=1.0): + """Adjust the brightness of image.""" + for key in results.get('img_fields', ['img']): + img = results[key] + results[key] = mmcv.adjust_brightness(img, + factor).astype(img.dtype) + + def __call__(self, results): + """Call function for Brightness transformation. + + Args: + results (dict): Results dict from loading pipeline. + + Returns: + dict: Results after the transformation. + """ + if np.random.rand() > self.prob: + return results + self._adjust_brightness_img(results, self.factor) + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(level={self.level}, ' + repr_str += f'prob={self.prob})' + return repr_str + + +@PIPELINES.register_module() +class ContrastTransform(object): + """Apply Contrast transformation to image. The bboxes, masks and + segmentations are not modified. + + Args: + level (int | float): Should be in range [0,_MAX_LEVEL]. + prob (float): The probability for performing Contrast transformation. + """ + + def __init__(self, level, prob=0.5): + assert isinstance(level, (int, float)), \ + 'The level must be type int or float.' + assert 0 <= level <= _MAX_LEVEL, \ + 'The level should be in range [0,_MAX_LEVEL].' + assert 0 <= prob <= 1.0, \ + 'The probability should be in range [0,1].' + self.level = level + self.prob = prob + self.factor = enhance_level_to_value(level) + + def _adjust_contrast_img(self, results, factor=1.0): + """Adjust the image contrast.""" + for key in results.get('img_fields', ['img']): + img = results[key] + results[key] = mmcv.adjust_contrast(img, factor).astype(img.dtype) + + def __call__(self, results): + """Call function for Contrast transformation. + + Args: + results (dict): Results dict from loading pipeline. + + Returns: + dict: Results after the transformation. + """ + if np.random.rand() > self.prob: + return results + self._adjust_contrast_img(results, self.factor) + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(level={self.level}, ' + repr_str += f'prob={self.prob})' + return repr_str diff --git a/detection/mmdet/datasets/pipelines/compose.py b/detection/mmdet/datasets/pipelines/compose.py new file mode 100644 index 0000000..ca48f1c --- /dev/null +++ b/detection/mmdet/datasets/pipelines/compose.py @@ -0,0 +1,51 @@ +import collections + +from mmcv.utils import build_from_cfg + +from ..builder import PIPELINES + + +@PIPELINES.register_module() +class Compose(object): + """Compose multiple transforms sequentially. + + Args: + transforms (Sequence[dict | callable]): Sequence of transform object or + config dict to be composed. + """ + + def __init__(self, transforms): + assert isinstance(transforms, collections.abc.Sequence) + self.transforms = [] + for transform in transforms: + if isinstance(transform, dict): + transform = build_from_cfg(transform, PIPELINES) + self.transforms.append(transform) + elif callable(transform): + self.transforms.append(transform) + else: + raise TypeError('transform must be callable or a dict') + + def __call__(self, data): + """Call function to apply transforms sequentially. + + Args: + data (dict): A result dict contains the data to transform. + + Returns: + dict: Transformed data. + """ + + for t in self.transforms: + data = t(data) + if data is None: + return None + return data + + def __repr__(self): + format_string = self.__class__.__name__ + '(' + for t in self.transforms: + format_string += '\n' + format_string += f' {t}' + format_string += '\n)' + return format_string diff --git a/detection/mmdet/datasets/pipelines/formating.py b/detection/mmdet/datasets/pipelines/formating.py new file mode 100644 index 0000000..5781341 --- /dev/null +++ b/detection/mmdet/datasets/pipelines/formating.py @@ -0,0 +1,364 @@ +from collections.abc import Sequence + +import mmcv +import numpy as np +import torch +from mmcv.parallel import DataContainer as DC + +from ..builder import PIPELINES + + +def to_tensor(data): + """Convert objects of various python types to :obj:`torch.Tensor`. + + Supported types are: :class:`numpy.ndarray`, :class:`torch.Tensor`, + :class:`Sequence`, :class:`int` and :class:`float`. + + Args: + data (torch.Tensor | numpy.ndarray | Sequence | int | float): Data to + be converted. + """ + + if isinstance(data, torch.Tensor): + return data + elif isinstance(data, np.ndarray): + return torch.from_numpy(data) + elif isinstance(data, Sequence) and not mmcv.is_str(data): + return torch.tensor(data) + elif isinstance(data, int): + return torch.LongTensor([data]) + elif isinstance(data, float): + return torch.FloatTensor([data]) + else: + raise TypeError(f'type {type(data)} cannot be converted to tensor.') + + +@PIPELINES.register_module() +class ToTensor(object): + """Convert some results to :obj:`torch.Tensor` by given keys. + + Args: + keys (Sequence[str]): Keys that need to be converted to Tensor. + """ + + def __init__(self, keys): + self.keys = keys + + def __call__(self, results): + """Call function to convert data in results to :obj:`torch.Tensor`. + + Args: + results (dict): Result dict contains the data to convert. + + Returns: + dict: The result dict contains the data converted + to :obj:`torch.Tensor`. + """ + for key in self.keys: + results[key] = to_tensor(results[key]) + return results + + def __repr__(self): + return self.__class__.__name__ + f'(keys={self.keys})' + + +@PIPELINES.register_module() +class ImageToTensor(object): + """Convert image to :obj:`torch.Tensor` by given keys. + + The dimension order of input image is (H, W, C). The pipeline will convert + it to (C, H, W). If only 2 dimension (H, W) is given, the output would be + (1, H, W). + + Args: + keys (Sequence[str]): Key of images to be converted to Tensor. + """ + + def __init__(self, keys): + self.keys = keys + + def __call__(self, results): + """Call function to convert image in results to :obj:`torch.Tensor` and + transpose the channel order. + + Args: + results (dict): Result dict contains the image data to convert. + + Returns: + dict: The result dict contains the image converted + to :obj:`torch.Tensor` and transposed to (C, H, W) order. + """ + for key in self.keys: + img = results[key] + if len(img.shape) < 3: + img = np.expand_dims(img, -1) + results[key] = to_tensor(img.transpose(2, 0, 1)) + return results + + def __repr__(self): + return self.__class__.__name__ + f'(keys={self.keys})' + + +@PIPELINES.register_module() +class Transpose(object): + """Transpose some results by given keys. + + Args: + keys (Sequence[str]): Keys of results to be transposed. + order (Sequence[int]): Order of transpose. + """ + + def __init__(self, keys, order): + self.keys = keys + self.order = order + + def __call__(self, results): + """Call function to transpose the channel order of data in results. + + Args: + results (dict): Result dict contains the data to transpose. + + Returns: + dict: The result dict contains the data transposed to \ + ``self.order``. + """ + for key in self.keys: + results[key] = results[key].transpose(self.order) + return results + + def __repr__(self): + return self.__class__.__name__ + \ + f'(keys={self.keys}, order={self.order})' + + +@PIPELINES.register_module() +class ToDataContainer(object): + """Convert results to :obj:`mmcv.DataContainer` by given fields. + + Args: + fields (Sequence[dict]): Each field is a dict like + ``dict(key='xxx', **kwargs)``. The ``key`` in result will + be converted to :obj:`mmcv.DataContainer` with ``**kwargs``. + Default: ``(dict(key='img', stack=True), dict(key='gt_bboxes'), + dict(key='gt_labels'))``. + """ + + def __init__(self, + fields=(dict(key='img', stack=True), dict(key='gt_bboxes'), + dict(key='gt_labels'))): + self.fields = fields + + def __call__(self, results): + """Call function to convert data in results to + :obj:`mmcv.DataContainer`. + + Args: + results (dict): Result dict contains the data to convert. + + Returns: + dict: The result dict contains the data converted to \ + :obj:`mmcv.DataContainer`. + """ + + for field in self.fields: + field = field.copy() + key = field.pop('key') + results[key] = DC(results[key], **field) + return results + + def __repr__(self): + return self.__class__.__name__ + f'(fields={self.fields})' + + +@PIPELINES.register_module() +class DefaultFormatBundle(object): + """Default formatting bundle. + + It simplifies the pipeline of formatting common fields, including "img", + "proposals", "gt_bboxes", "gt_labels", "gt_masks" and "gt_semantic_seg". + These fields are formatted as follows. + + - img: (1)transpose, (2)to tensor, (3)to DataContainer (stack=True) + - proposals: (1)to tensor, (2)to DataContainer + - gt_bboxes: (1)to tensor, (2)to DataContainer + - gt_bboxes_ignore: (1)to tensor, (2)to DataContainer + - gt_labels: (1)to tensor, (2)to DataContainer + - gt_masks: (1)to tensor, (2)to DataContainer (cpu_only=True) + - gt_semantic_seg: (1)unsqueeze dim-0 (2)to tensor, \ + (3)to DataContainer (stack=True) + """ + + def __call__(self, results): + """Call function to transform and format common fields in results. + + Args: + results (dict): Result dict contains the data to convert. + + Returns: + dict: The result dict contains the data that is formatted with \ + default bundle. + """ + + if 'img' in results: + img = results['img'] + # add default meta keys + results = self._add_default_meta_keys(results) + if len(img.shape) < 3: + img = np.expand_dims(img, -1) + img = np.ascontiguousarray(img.transpose(2, 0, 1)) + results['img'] = DC(to_tensor(img), stack=True) + for key in ['proposals', 'gt_bboxes', 'gt_bboxes_ignore', 'gt_labels']: + if key not in results: + continue + results[key] = DC(to_tensor(results[key])) + if 'gt_masks' in results: + results['gt_masks'] = DC(results['gt_masks'], cpu_only=True) + if 'gt_semantic_seg' in results: + results['gt_semantic_seg'] = DC( + to_tensor(results['gt_semantic_seg'][None, ...]), stack=True) + return results + + def _add_default_meta_keys(self, results): + """Add default meta keys. + + We set default meta keys including `pad_shape`, `scale_factor` and + `img_norm_cfg` to avoid the case where no `Resize`, `Normalize` and + `Pad` are implemented during the whole pipeline. + + Args: + results (dict): Result dict contains the data to convert. + + Returns: + results (dict): Updated result dict contains the data to convert. + """ + img = results['img'] + results.setdefault('pad_shape', img.shape) + results.setdefault('scale_factor', 1.0) + num_channels = 1 if len(img.shape) < 3 else img.shape[2] + results.setdefault( + 'img_norm_cfg', + dict( + mean=np.zeros(num_channels, dtype=np.float32), + std=np.ones(num_channels, dtype=np.float32), + to_rgb=False)) + return results + + def __repr__(self): + return self.__class__.__name__ + + +@PIPELINES.register_module() +class Collect(object): + """Collect data from the loader relevant to the specific task. + + This is usually the last stage of the data loader pipeline. Typically keys + is set to some subset of "img", "proposals", "gt_bboxes", + "gt_bboxes_ignore", "gt_labels", and/or "gt_masks". + + The "img_meta" item is always populated. The contents of the "img_meta" + dictionary depends on "meta_keys". By default this includes: + + - "img_shape": shape of the image input to the network as a tuple \ + (h, w, c). Note that images may be zero padded on the \ + bottom/right if the batch tensor is larger than this shape. + + - "scale_factor": a float indicating the preprocessing scale + + - "flip": a boolean indicating if image flip transform was used + + - "filename": path to the image file + + - "ori_shape": original shape of the image as a tuple (h, w, c) + + - "pad_shape": image shape after padding + + - "img_norm_cfg": a dict of normalization information: + + - mean - per channel mean subtraction + - std - per channel std divisor + - to_rgb - bool indicating if bgr was converted to rgb + + Args: + keys (Sequence[str]): Keys of results to be collected in ``data``. + meta_keys (Sequence[str], optional): Meta keys to be converted to + ``mmcv.DataContainer`` and collected in ``data[img_metas]``. + Default: ``('filename', 'ori_filename', 'ori_shape', 'img_shape', + 'pad_shape', 'scale_factor', 'flip', 'flip_direction', + 'img_norm_cfg')`` + """ + + def __init__(self, + keys, + meta_keys=('filename', 'ori_filename', 'ori_shape', + 'img_shape', 'pad_shape', 'scale_factor', 'flip', + 'flip_direction', 'img_norm_cfg')): + self.keys = keys + self.meta_keys = meta_keys + + def __call__(self, results): + """Call function to collect keys in results. The keys in ``meta_keys`` + will be converted to :obj:mmcv.DataContainer. + + Args: + results (dict): Result dict contains the data to collect. + + Returns: + dict: The result dict contains the following keys + + - keys in``self.keys`` + - ``img_metas`` + """ + + data = {} + img_meta = {} + for key in self.meta_keys: + img_meta[key] = results[key] + data['img_metas'] = DC(img_meta, cpu_only=True) + for key in self.keys: + data[key] = results[key] + return data + + def __repr__(self): + return self.__class__.__name__ + \ + f'(keys={self.keys}, meta_keys={self.meta_keys})' + + +@PIPELINES.register_module() +class WrapFieldsToLists(object): + """Wrap fields of the data dictionary into lists for evaluation. + + This class can be used as a last step of a test or validation + pipeline for single image evaluation or inference. + + Example: + >>> test_pipeline = [ + >>> dict(type='LoadImageFromFile'), + >>> dict(type='Normalize', + mean=[123.675, 116.28, 103.53], + std=[58.395, 57.12, 57.375], + to_rgb=True), + >>> dict(type='Pad', size_divisor=32), + >>> dict(type='ImageToTensor', keys=['img']), + >>> dict(type='Collect', keys=['img']), + >>> dict(type='WrapFieldsToLists') + >>> ] + """ + + def __call__(self, results): + """Call function to wrap fields into lists. + + Args: + results (dict): Result dict contains the data to wrap. + + Returns: + dict: The result dict where value of ``self.keys`` are wrapped \ + into list. + """ + + # Wrap dict fields into lists + for key, val in results.items(): + results[key] = [val] + return results + + def __repr__(self): + return f'{self.__class__.__name__}()' diff --git a/detection/mmdet/datasets/pipelines/instaboost.py b/detection/mmdet/datasets/pipelines/instaboost.py new file mode 100644 index 0000000..38b6819 --- /dev/null +++ b/detection/mmdet/datasets/pipelines/instaboost.py @@ -0,0 +1,98 @@ +import numpy as np + +from ..builder import PIPELINES + + +@PIPELINES.register_module() +class InstaBoost(object): + r"""Data augmentation method in `InstaBoost: Boosting Instance + Segmentation Via Probability Map Guided Copy-Pasting + `_. + + Refer to https://github.com/GothicAi/Instaboost for implementation details. + """ + + def __init__(self, + action_candidate=('normal', 'horizontal', 'skip'), + action_prob=(1, 0, 0), + scale=(0.8, 1.2), + dx=15, + dy=15, + theta=(-1, 1), + color_prob=0.5, + hflag=False, + aug_ratio=0.5): + try: + import instaboostfast as instaboost + except ImportError: + raise ImportError( + 'Please run "pip install instaboostfast" ' + 'to install instaboostfast first for instaboost augmentation.') + self.cfg = instaboost.InstaBoostConfig(action_candidate, action_prob, + scale, dx, dy, theta, + color_prob, hflag) + self.aug_ratio = aug_ratio + + def _load_anns(self, results): + labels = results['ann_info']['labels'] + masks = results['ann_info']['masks'] + bboxes = results['ann_info']['bboxes'] + n = len(labels) + + anns = [] + for i in range(n): + label = labels[i] + bbox = bboxes[i] + mask = masks[i] + x1, y1, x2, y2 = bbox + # assert (x2 - x1) >= 1 and (y2 - y1) >= 1 + bbox = [x1, y1, x2 - x1, y2 - y1] + anns.append({ + 'category_id': label, + 'segmentation': mask, + 'bbox': bbox + }) + + return anns + + def _parse_anns(self, results, anns, img): + gt_bboxes = [] + gt_labels = [] + gt_masks_ann = [] + for ann in anns: + x1, y1, w, h = ann['bbox'] + # TODO: more essential bug need to be fixed in instaboost + if w <= 0 or h <= 0: + continue + bbox = [x1, y1, x1 + w, y1 + h] + gt_bboxes.append(bbox) + gt_labels.append(ann['category_id']) + gt_masks_ann.append(ann['segmentation']) + gt_bboxes = np.array(gt_bboxes, dtype=np.float32) + gt_labels = np.array(gt_labels, dtype=np.int64) + results['ann_info']['labels'] = gt_labels + results['ann_info']['bboxes'] = gt_bboxes + results['ann_info']['masks'] = gt_masks_ann + results['img'] = img + return results + + def __call__(self, results): + img = results['img'] + orig_type = img.dtype + anns = self._load_anns(results) + if np.random.choice([0, 1], p=[1 - self.aug_ratio, self.aug_ratio]): + try: + import instaboostfast as instaboost + except ImportError: + raise ImportError('Please run "pip install instaboostfast" ' + 'to install instaboostfast first.') + anns, img = instaboost.get_new_data( + anns, img.astype(np.uint8), self.cfg, background=None) + + results = self._parse_anns(results, anns, img.astype(orig_type)) + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(cfg={self.cfg}, aug_ratio={self.aug_ratio})' + return repr_str diff --git a/detection/mmdet/datasets/pipelines/loading.py b/detection/mmdet/datasets/pipelines/loading.py new file mode 100644 index 0000000..6922594 --- /dev/null +++ b/detection/mmdet/datasets/pipelines/loading.py @@ -0,0 +1,458 @@ +import os.path as osp + +import mmcv +import numpy as np +import pycocotools.mask as maskUtils + +from mmdet.core import BitmapMasks, PolygonMasks +from ..builder import PIPELINES + + +@PIPELINES.register_module() +class LoadImageFromFile(object): + """Load an image from file. + + Required keys are "img_prefix" and "img_info" (a dict that must contain the + key "filename"). Added or updated keys are "filename", "img", "img_shape", + "ori_shape" (same as `img_shape`), "pad_shape" (same as `img_shape`), + "scale_factor" (1.0) and "img_norm_cfg" (means=0 and stds=1). + + Args: + to_float32 (bool): Whether to convert the loaded image to a float32 + numpy array. If set to False, the loaded image is an uint8 array. + Defaults to False. + color_type (str): The flag argument for :func:`mmcv.imfrombytes`. + Defaults to 'color'. + file_client_args (dict): Arguments to instantiate a FileClient. + See :class:`mmcv.fileio.FileClient` for details. + Defaults to ``dict(backend='disk')``. + """ + + def __init__(self, + to_float32=False, + color_type='color', + file_client_args=dict(backend='disk')): + self.to_float32 = to_float32 + self.color_type = color_type + self.file_client_args = file_client_args.copy() + self.file_client = None + + def __call__(self, results): + """Call functions to load image and get image meta information. + + Args: + results (dict): Result dict from :obj:`mmdet.CustomDataset`. + + Returns: + dict: The dict contains loaded image and meta information. + """ + + if self.file_client is None: + self.file_client = mmcv.FileClient(**self.file_client_args) + + if results['img_prefix'] is not None: + filename = osp.join(results['img_prefix'], + results['img_info']['filename']) + else: + filename = results['img_info']['filename'] + + img_bytes = self.file_client.get(filename) + img = mmcv.imfrombytes(img_bytes, flag=self.color_type) + if self.to_float32: + img = img.astype(np.float32) + + results['filename'] = filename + results['ori_filename'] = results['img_info']['filename'] + results['img'] = img + results['img_shape'] = img.shape + results['ori_shape'] = img.shape + results['img_fields'] = ['img'] + return results + + def __repr__(self): + repr_str = (f'{self.__class__.__name__}(' + f'to_float32={self.to_float32}, ' + f"color_type='{self.color_type}', " + f'file_client_args={self.file_client_args})') + return repr_str + + +@PIPELINES.register_module() +class LoadImageFromWebcam(LoadImageFromFile): + """Load an image from webcam. + + Similar with :obj:`LoadImageFromFile`, but the image read from webcam is in + ``results['img']``. + """ + + def __call__(self, results): + """Call functions to add image meta information. + + Args: + results (dict): Result dict with Webcam read image in + ``results['img']``. + + Returns: + dict: The dict contains loaded image and meta information. + """ + + img = results['img'] + if self.to_float32: + img = img.astype(np.float32) + + results['filename'] = None + results['ori_filename'] = None + results['img'] = img + results['img_shape'] = img.shape + results['ori_shape'] = img.shape + results['img_fields'] = ['img'] + return results + + +@PIPELINES.register_module() +class LoadMultiChannelImageFromFiles(object): + """Load multi-channel images from a list of separate channel files. + + Required keys are "img_prefix" and "img_info" (a dict that must contain the + key "filename", which is expected to be a list of filenames). + Added or updated keys are "filename", "img", "img_shape", + "ori_shape" (same as `img_shape`), "pad_shape" (same as `img_shape`), + "scale_factor" (1.0) and "img_norm_cfg" (means=0 and stds=1). + + Args: + to_float32 (bool): Whether to convert the loaded image to a float32 + numpy array. If set to False, the loaded image is an uint8 array. + Defaults to False. + color_type (str): The flag argument for :func:`mmcv.imfrombytes`. + Defaults to 'color'. + file_client_args (dict): Arguments to instantiate a FileClient. + See :class:`mmcv.fileio.FileClient` for details. + Defaults to ``dict(backend='disk')``. + """ + + def __init__(self, + to_float32=False, + color_type='unchanged', + file_client_args=dict(backend='disk')): + self.to_float32 = to_float32 + self.color_type = color_type + self.file_client_args = file_client_args.copy() + self.file_client = None + + def __call__(self, results): + """Call functions to load multiple images and get images meta + information. + + Args: + results (dict): Result dict from :obj:`mmdet.CustomDataset`. + + Returns: + dict: The dict contains loaded images and meta information. + """ + + if self.file_client is None: + self.file_client = mmcv.FileClient(**self.file_client_args) + + if results['img_prefix'] is not None: + filename = [ + osp.join(results['img_prefix'], fname) + for fname in results['img_info']['filename'] + ] + else: + filename = results['img_info']['filename'] + + img = [] + for name in filename: + img_bytes = self.file_client.get(name) + img.append(mmcv.imfrombytes(img_bytes, flag=self.color_type)) + img = np.stack(img, axis=-1) + if self.to_float32: + img = img.astype(np.float32) + + results['filename'] = filename + results['ori_filename'] = results['img_info']['filename'] + results['img'] = img + results['img_shape'] = img.shape + results['ori_shape'] = img.shape + # Set initial values for default meta_keys + results['pad_shape'] = img.shape + results['scale_factor'] = 1.0 + num_channels = 1 if len(img.shape) < 3 else img.shape[2] + results['img_norm_cfg'] = dict( + mean=np.zeros(num_channels, dtype=np.float32), + std=np.ones(num_channels, dtype=np.float32), + to_rgb=False) + return results + + def __repr__(self): + repr_str = (f'{self.__class__.__name__}(' + f'to_float32={self.to_float32}, ' + f"color_type='{self.color_type}', " + f'file_client_args={self.file_client_args})') + return repr_str + + +@PIPELINES.register_module() +class LoadAnnotations(object): + """Load mutiple types of annotations. + + Args: + with_bbox (bool): Whether to parse and load the bbox annotation. + Default: True. + with_label (bool): Whether to parse and load the label annotation. + Default: True. + with_mask (bool): Whether to parse and load the mask annotation. + Default: False. + with_seg (bool): Whether to parse and load the semantic segmentation + annotation. Default: False. + poly2mask (bool): Whether to convert the instance masks from polygons + to bitmaps. Default: True. + file_client_args (dict): Arguments to instantiate a FileClient. + See :class:`mmcv.fileio.FileClient` for details. + Defaults to ``dict(backend='disk')``. + """ + + def __init__(self, + with_bbox=True, + with_label=True, + with_mask=False, + with_seg=False, + poly2mask=True, + file_client_args=dict(backend='disk')): + self.with_bbox = with_bbox + self.with_label = with_label + self.with_mask = with_mask + self.with_seg = with_seg + self.poly2mask = poly2mask + self.file_client_args = file_client_args.copy() + self.file_client = None + + def _load_bboxes(self, results): + """Private function to load bounding box annotations. + + Args: + results (dict): Result dict from :obj:`mmdet.CustomDataset`. + + Returns: + dict: The dict contains loaded bounding box annotations. + """ + + ann_info = results['ann_info'] + results['gt_bboxes'] = ann_info['bboxes'].copy() + + gt_bboxes_ignore = ann_info.get('bboxes_ignore', None) + if gt_bboxes_ignore is not None: + results['gt_bboxes_ignore'] = gt_bboxes_ignore.copy() + results['bbox_fields'].append('gt_bboxes_ignore') + results['bbox_fields'].append('gt_bboxes') + return results + + def _load_labels(self, results): + """Private function to load label annotations. + + Args: + results (dict): Result dict from :obj:`mmdet.CustomDataset`. + + Returns: + dict: The dict contains loaded label annotations. + """ + + results['gt_labels'] = results['ann_info']['labels'].copy() + return results + + def _poly2mask(self, mask_ann, img_h, img_w): + """Private function to convert masks represented with polygon to + bitmaps. + + Args: + mask_ann (list | dict): Polygon mask annotation input. + img_h (int): The height of output mask. + img_w (int): The width of output mask. + + Returns: + numpy.ndarray: The decode bitmap mask of shape (img_h, img_w). + """ + + if isinstance(mask_ann, list): + # polygon -- a single object might consist of multiple parts + # we merge all parts into one mask rle code + rles = maskUtils.frPyObjects(mask_ann, img_h, img_w) + rle = maskUtils.merge(rles) + elif isinstance(mask_ann['counts'], list): + # uncompressed RLE + rle = maskUtils.frPyObjects(mask_ann, img_h, img_w) + else: + # rle + rle = mask_ann + mask = maskUtils.decode(rle) + return mask + + def process_polygons(self, polygons): + """Convert polygons to list of ndarray and filter invalid polygons. + + Args: + polygons (list[list]): Polygons of one instance. + + Returns: + list[numpy.ndarray]: Processed polygons. + """ + + polygons = [np.array(p) for p in polygons] + valid_polygons = [] + for polygon in polygons: + if len(polygon) % 2 == 0 and len(polygon) >= 6: + valid_polygons.append(polygon) + return valid_polygons + + def _load_masks(self, results): + """Private function to load mask annotations. + + Args: + results (dict): Result dict from :obj:`mmdet.CustomDataset`. + + Returns: + dict: The dict contains loaded mask annotations. + If ``self.poly2mask`` is set ``True``, `gt_mask` will contain + :obj:`PolygonMasks`. Otherwise, :obj:`BitmapMasks` is used. + """ + + h, w = results['img_info']['height'], results['img_info']['width'] + gt_masks = results['ann_info']['masks'] + if self.poly2mask: + gt_masks = BitmapMasks( + [self._poly2mask(mask, h, w) for mask in gt_masks], h, w) + else: + gt_masks = PolygonMasks( + [self.process_polygons(polygons) for polygons in gt_masks], h, + w) + results['gt_masks'] = gt_masks + results['mask_fields'].append('gt_masks') + return results + + def _load_semantic_seg(self, results): + """Private function to load semantic segmentation annotations. + + Args: + results (dict): Result dict from :obj:`dataset`. + + Returns: + dict: The dict contains loaded semantic segmentation annotations. + """ + + if self.file_client is None: + self.file_client = mmcv.FileClient(**self.file_client_args) + + filename = osp.join(results['seg_prefix'], + results['ann_info']['seg_map']) + img_bytes = self.file_client.get(filename) + results['gt_semantic_seg'] = mmcv.imfrombytes( + img_bytes, flag='unchanged').squeeze() + results['seg_fields'].append('gt_semantic_seg') + return results + + def __call__(self, results): + """Call function to load multiple types annotations. + + Args: + results (dict): Result dict from :obj:`mmdet.CustomDataset`. + + Returns: + dict: The dict contains loaded bounding box, label, mask and + semantic segmentation annotations. + """ + + if self.with_bbox: + results = self._load_bboxes(results) + if results is None: + return None + if self.with_label: + results = self._load_labels(results) + if self.with_mask: + results = self._load_masks(results) + if self.with_seg: + results = self._load_semantic_seg(results) + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(with_bbox={self.with_bbox}, ' + repr_str += f'with_label={self.with_label}, ' + repr_str += f'with_mask={self.with_mask}, ' + repr_str += f'with_seg={self.with_seg}, ' + repr_str += f'poly2mask={self.poly2mask}, ' + repr_str += f'poly2mask={self.file_client_args})' + return repr_str + + +@PIPELINES.register_module() +class LoadProposals(object): + """Load proposal pipeline. + + Required key is "proposals". Updated keys are "proposals", "bbox_fields". + + Args: + num_max_proposals (int, optional): Maximum number of proposals to load. + If not specified, all proposals will be loaded. + """ + + def __init__(self, num_max_proposals=None): + self.num_max_proposals = num_max_proposals + + def __call__(self, results): + """Call function to load proposals from file. + + Args: + results (dict): Result dict from :obj:`mmdet.CustomDataset`. + + Returns: + dict: The dict contains loaded proposal annotations. + """ + + proposals = results['proposals'] + if proposals.shape[1] not in (4, 5): + raise AssertionError( + 'proposals should have shapes (n, 4) or (n, 5), ' + f'but found {proposals.shape}') + proposals = proposals[:, :4] + + if self.num_max_proposals is not None: + proposals = proposals[:self.num_max_proposals] + + if len(proposals) == 0: + proposals = np.array([[0, 0, 0, 0]], dtype=np.float32) + results['proposals'] = proposals + results['bbox_fields'].append('proposals') + return results + + def __repr__(self): + return self.__class__.__name__ + \ + f'(num_max_proposals={self.num_max_proposals})' + + +@PIPELINES.register_module() +class FilterAnnotations(object): + """Filter invalid annotations. + + Args: + min_gt_bbox_wh (tuple[int]): Minimum width and height of ground truth + boxes. + """ + + def __init__(self, min_gt_bbox_wh): + # TODO: add more filter options + self.min_gt_bbox_wh = min_gt_bbox_wh + + def __call__(self, results): + assert 'gt_bboxes' in results + gt_bboxes = results['gt_bboxes'] + w = gt_bboxes[:, 2] - gt_bboxes[:, 0] + h = gt_bboxes[:, 3] - gt_bboxes[:, 1] + keep = (w > self.min_gt_bbox_wh[0]) & (h > self.min_gt_bbox_wh[1]) + if not keep.any(): + return None + else: + keys = ('gt_bboxes', 'gt_labels', 'gt_masks', 'gt_semantic_seg') + for key in keys: + if key in results: + results[key] = results[key][keep] + return results diff --git a/detection/mmdet/datasets/pipelines/test_time_aug.py b/detection/mmdet/datasets/pipelines/test_time_aug.py new file mode 100644 index 0000000..b6226e0 --- /dev/null +++ b/detection/mmdet/datasets/pipelines/test_time_aug.py @@ -0,0 +1,119 @@ +import warnings + +import mmcv + +from ..builder import PIPELINES +from .compose import Compose + + +@PIPELINES.register_module() +class MultiScaleFlipAug(object): + """Test-time augmentation with multiple scales and flipping. + + An example configuration is as followed: + + .. code-block:: + + img_scale=[(1333, 400), (1333, 800)], + flip=True, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size_divisor=32), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ] + + After MultiScaleFLipAug with above configuration, the results are wrapped + into lists of the same length as followed: + + .. code-block:: + + dict( + img=[...], + img_shape=[...], + scale=[(1333, 400), (1333, 400), (1333, 800), (1333, 800)] + flip=[False, True, False, True] + ... + ) + + Args: + transforms (list[dict]): Transforms to apply in each augmentation. + img_scale (tuple | list[tuple] | None): Images scales for resizing. + scale_factor (float | list[float] | None): Scale factors for resizing. + flip (bool): Whether apply flip augmentation. Default: False. + flip_direction (str | list[str]): Flip augmentation directions, + options are "horizontal" and "vertical". If flip_direction is list, + multiple flip augmentations will be applied. + It has no effect when flip == False. Default: "horizontal". + """ + + def __init__(self, + transforms, + img_scale=None, + scale_factor=None, + flip=False, + flip_direction='horizontal'): + self.transforms = Compose(transforms) + assert (img_scale is None) ^ (scale_factor is None), ( + 'Must have but only one variable can be setted') + if img_scale is not None: + self.img_scale = img_scale if isinstance(img_scale, + list) else [img_scale] + self.scale_key = 'scale' + assert mmcv.is_list_of(self.img_scale, tuple) + else: + self.img_scale = scale_factor if isinstance( + scale_factor, list) else [scale_factor] + self.scale_key = 'scale_factor' + + self.flip = flip + self.flip_direction = flip_direction if isinstance( + flip_direction, list) else [flip_direction] + assert mmcv.is_list_of(self.flip_direction, str) + if not self.flip and self.flip_direction != ['horizontal']: + warnings.warn( + 'flip_direction has no effect when flip is set to False') + if (self.flip + and not any([t['type'] == 'RandomFlip' for t in transforms])): + warnings.warn( + 'flip has no effect when RandomFlip is not in transforms') + + def __call__(self, results): + """Call function to apply test time augment transforms on results. + + Args: + results (dict): Result dict contains the data to transform. + + Returns: + dict[str: list]: The augmented data, where each value is wrapped + into a list. + """ + + aug_data = [] + flip_args = [(False, None)] + if self.flip: + flip_args += [(True, direction) + for direction in self.flip_direction] + for scale in self.img_scale: + for flip, direction in flip_args: + _results = results.copy() + _results[self.scale_key] = scale + _results['flip'] = flip + _results['flip_direction'] = direction + data = self.transforms(_results) + aug_data.append(data) + # list of dict to dict of list + aug_data_dict = {key: [] for key in aug_data[0]} + for data in aug_data: + for key, val in data.items(): + aug_data_dict[key].append(val) + return aug_data_dict + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(transforms={self.transforms}, ' + repr_str += f'img_scale={self.img_scale}, flip={self.flip}, ' + repr_str += f'flip_direction={self.flip_direction})' + return repr_str diff --git a/detection/mmdet/datasets/pipelines/transforms.py b/detection/mmdet/datasets/pipelines/transforms.py new file mode 100644 index 0000000..caed51d --- /dev/null +++ b/detection/mmdet/datasets/pipelines/transforms.py @@ -0,0 +1,1811 @@ +import copy +import inspect + +import mmcv +import numpy as np +from numpy import random + +from mmdet.core import PolygonMasks +from mmdet.core.evaluation.bbox_overlaps import bbox_overlaps +from ..builder import PIPELINES + +try: + from imagecorruptions import corrupt +except ImportError: + corrupt = None + +try: + import albumentations + from albumentations import Compose +except ImportError: + albumentations = None + Compose = None + + +@PIPELINES.register_module() +class Resize(object): + """Resize images & bbox & mask. + + This transform resizes the input image to some scale. Bboxes and masks are + then resized with the same scale factor. If the input dict contains the key + "scale", then the scale in the input dict is used, otherwise the specified + scale in the init method is used. If the input dict contains the key + "scale_factor" (if MultiScaleFlipAug does not give img_scale but + scale_factor), the actual scale will be computed by image shape and + scale_factor. + + `img_scale` can either be a tuple (single-scale) or a list of tuple + (multi-scale). There are 3 multiscale modes: + + - ``ratio_range is not None``: randomly sample a ratio from the ratio \ + range and multiply it with the image scale. + - ``ratio_range is None`` and ``multiscale_mode == "range"``: randomly \ + sample a scale from the multiscale range. + - ``ratio_range is None`` and ``multiscale_mode == "value"``: randomly \ + sample a scale from multiple scales. + + Args: + img_scale (tuple or list[tuple]): Images scales for resizing. + multiscale_mode (str): Either "range" or "value". + ratio_range (tuple[float]): (min_ratio, max_ratio) + keep_ratio (bool): Whether to keep the aspect ratio when resizing the + image. + bbox_clip_border (bool, optional): Whether clip the objects outside + the border of the image. Defaults to True. + backend (str): Image resize backend, choices are 'cv2' and 'pillow'. + These two backends generates slightly different results. Defaults + to 'cv2'. + override (bool, optional): Whether to override `scale` and + `scale_factor` so as to call resize twice. Default False. If True, + after the first resizing, the existed `scale` and `scale_factor` + will be ignored so the second resizing can be allowed. + This option is a work-around for multiple times of resize in DETR. + Defaults to False. + """ + + def __init__(self, + img_scale=None, + multiscale_mode='range', + ratio_range=None, + keep_ratio=True, + bbox_clip_border=True, + backend='cv2', + override=False): + if img_scale is None: + self.img_scale = None + else: + if isinstance(img_scale, list): + self.img_scale = img_scale + else: + self.img_scale = [img_scale] + assert mmcv.is_list_of(self.img_scale, tuple) + + if ratio_range is not None: + # mode 1: given a scale and a range of image ratio + assert len(self.img_scale) == 1 + else: + # mode 2: given multiple scales or a range of scales + assert multiscale_mode in ['value', 'range'] + + self.backend = backend + self.multiscale_mode = multiscale_mode + self.ratio_range = ratio_range + self.keep_ratio = keep_ratio + # TODO: refactor the override option in Resize + self.override = override + self.bbox_clip_border = bbox_clip_border + + @staticmethod + def random_select(img_scales): + """Randomly select an img_scale from given candidates. + + Args: + img_scales (list[tuple]): Images scales for selection. + + Returns: + (tuple, int): Returns a tuple ``(img_scale, scale_dix)``, \ + where ``img_scale`` is the selected image scale and \ + ``scale_idx`` is the selected index in the given candidates. + """ + + assert mmcv.is_list_of(img_scales, tuple) + scale_idx = np.random.randint(len(img_scales)) + img_scale = img_scales[scale_idx] + return img_scale, scale_idx + + @staticmethod + def random_sample(img_scales): + """Randomly sample an img_scale when ``multiscale_mode=='range'``. + + Args: + img_scales (list[tuple]): Images scale range for sampling. + There must be two tuples in img_scales, which specify the lower + and upper bound of image scales. + + Returns: + (tuple, None): Returns a tuple ``(img_scale, None)``, where \ + ``img_scale`` is sampled scale and None is just a placeholder \ + to be consistent with :func:`random_select`. + """ + + assert mmcv.is_list_of(img_scales, tuple) and len(img_scales) == 2 + img_scale_long = [max(s) for s in img_scales] + img_scale_short = [min(s) for s in img_scales] + long_edge = np.random.randint( + min(img_scale_long), + max(img_scale_long) + 1) + short_edge = np.random.randint( + min(img_scale_short), + max(img_scale_short) + 1) + img_scale = (long_edge, short_edge) + return img_scale, None + + @staticmethod + def random_sample_ratio(img_scale, ratio_range): + """Randomly sample an img_scale when ``ratio_range`` is specified. + + A ratio will be randomly sampled from the range specified by + ``ratio_range``. Then it would be multiplied with ``img_scale`` to + generate sampled scale. + + Args: + img_scale (tuple): Images scale base to multiply with ratio. + ratio_range (tuple[float]): The minimum and maximum ratio to scale + the ``img_scale``. + + Returns: + (tuple, None): Returns a tuple ``(scale, None)``, where \ + ``scale`` is sampled ratio multiplied with ``img_scale`` and \ + None is just a placeholder to be consistent with \ + :func:`random_select`. + """ + + assert isinstance(img_scale, tuple) and len(img_scale) == 2 + min_ratio, max_ratio = ratio_range + assert min_ratio <= max_ratio + ratio = np.random.random_sample() * (max_ratio - min_ratio) + min_ratio + scale = int(img_scale[0] * ratio), int(img_scale[1] * ratio) + return scale, None + + def _random_scale(self, results): + """Randomly sample an img_scale according to ``ratio_range`` and + ``multiscale_mode``. + + If ``ratio_range`` is specified, a ratio will be sampled and be + multiplied with ``img_scale``. + If multiple scales are specified by ``img_scale``, a scale will be + sampled according to ``multiscale_mode``. + Otherwise, single scale will be used. + + Args: + results (dict): Result dict from :obj:`dataset`. + + Returns: + dict: Two new keys 'scale` and 'scale_idx` are added into \ + ``results``, which would be used by subsequent pipelines. + """ + + if self.ratio_range is not None: + scale, scale_idx = self.random_sample_ratio( + self.img_scale[0], self.ratio_range) + elif len(self.img_scale) == 1: + scale, scale_idx = self.img_scale[0], 0 + elif self.multiscale_mode == 'range': + scale, scale_idx = self.random_sample(self.img_scale) + elif self.multiscale_mode == 'value': + scale, scale_idx = self.random_select(self.img_scale) + else: + raise NotImplementedError + + results['scale'] = scale + results['scale_idx'] = scale_idx + + def _resize_img(self, results): + """Resize images with ``results['scale']``.""" + for key in results.get('img_fields', ['img']): + if self.keep_ratio: + img, scale_factor = mmcv.imrescale( + results[key], + results['scale'], + return_scale=True, + backend=self.backend) + # the w_scale and h_scale has minor difference + # a real fix should be done in the mmcv.imrescale in the future + new_h, new_w = img.shape[:2] + h, w = results[key].shape[:2] + w_scale = new_w / w + h_scale = new_h / h + else: + img, w_scale, h_scale = mmcv.imresize( + results[key], + results['scale'], + return_scale=True, + backend=self.backend) + results[key] = img + + scale_factor = np.array([w_scale, h_scale, w_scale, h_scale], + dtype=np.float32) + results['img_shape'] = img.shape + # in case that there is no padding + results['pad_shape'] = img.shape + results['scale_factor'] = scale_factor + results['keep_ratio'] = self.keep_ratio + + def _resize_bboxes(self, results): + """Resize bounding boxes with ``results['scale_factor']``.""" + for key in results.get('bbox_fields', []): + bboxes = results[key] * results['scale_factor'] + if self.bbox_clip_border: + img_shape = results['img_shape'] + bboxes[:, 0::2] = np.clip(bboxes[:, 0::2], 0, img_shape[1]) + bboxes[:, 1::2] = np.clip(bboxes[:, 1::2], 0, img_shape[0]) + results[key] = bboxes + + def _resize_masks(self, results): + """Resize masks with ``results['scale']``""" + for key in results.get('mask_fields', []): + if results[key] is None: + continue + if self.keep_ratio: + results[key] = results[key].rescale(results['scale']) + else: + results[key] = results[key].resize(results['img_shape'][:2]) + + def _resize_seg(self, results): + """Resize semantic segmentation map with ``results['scale']``.""" + for key in results.get('seg_fields', []): + if self.keep_ratio: + gt_seg = mmcv.imrescale( + results[key], + results['scale'], + interpolation='nearest', + backend=self.backend) + else: + gt_seg = mmcv.imresize( + results[key], + results['scale'], + interpolation='nearest', + backend=self.backend) + results['gt_semantic_seg'] = gt_seg + + def __call__(self, results): + """Call function to resize images, bounding boxes, masks, semantic + segmentation map. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Resized results, 'img_shape', 'pad_shape', 'scale_factor', \ + 'keep_ratio' keys are added into result dict. + """ + + if 'scale' not in results: + if 'scale_factor' in results: + img_shape = results['img'].shape[:2] + scale_factor = results['scale_factor'] + assert isinstance(scale_factor, float) + results['scale'] = tuple( + [int(x * scale_factor) for x in img_shape][::-1]) + else: + self._random_scale(results) + else: + if not self.override: + assert 'scale_factor' not in results, ( + 'scale and scale_factor cannot be both set.') + else: + results.pop('scale') + if 'scale_factor' in results: + results.pop('scale_factor') + self._random_scale(results) + + self._resize_img(results) + self._resize_bboxes(results) + self._resize_masks(results) + self._resize_seg(results) + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(img_scale={self.img_scale}, ' + repr_str += f'multiscale_mode={self.multiscale_mode}, ' + repr_str += f'ratio_range={self.ratio_range}, ' + repr_str += f'keep_ratio={self.keep_ratio}, ' + repr_str += f'bbox_clip_border={self.bbox_clip_border})' + return repr_str + + +@PIPELINES.register_module() +class RandomFlip(object): + """Flip the image & bbox & mask. + + If the input dict contains the key "flip", then the flag will be used, + otherwise it will be randomly decided by a ratio specified in the init + method. + + When random flip is enabled, ``flip_ratio``/``direction`` can either be a + float/string or tuple of float/string. There are 3 flip modes: + + - ``flip_ratio`` is float, ``direction`` is string: the image will be + ``direction``ly flipped with probability of ``flip_ratio`` . + E.g., ``flip_ratio=0.5``, ``direction='horizontal'``, + then image will be horizontally flipped with probability of 0.5. + - ``flip_ratio`` is float, ``direction`` is list of string: the image wil + be ``direction[i]``ly flipped with probability of + ``flip_ratio/len(direction)``. + E.g., ``flip_ratio=0.5``, ``direction=['horizontal', 'vertical']``, + then image will be horizontally flipped with probability of 0.25, + vertically with probability of 0.25. + - ``flip_ratio`` is list of float, ``direction`` is list of string: + given ``len(flip_ratio) == len(direction)``, the image wil + be ``direction[i]``ly flipped with probability of ``flip_ratio[i]``. + E.g., ``flip_ratio=[0.3, 0.5]``, ``direction=['horizontal', + 'vertical']``, then image will be horizontally flipped with probability + of 0.3, vertically with probability of 0.5 + + Args: + flip_ratio (float | list[float], optional): The flipping probability. + Default: None. + direction(str | list[str], optional): The flipping direction. Options + are 'horizontal', 'vertical', 'diagonal'. Default: 'horizontal'. + If input is a list, the length must equal ``flip_ratio``. Each + element in ``flip_ratio`` indicates the flip probability of + corresponding direction. + """ + + def __init__(self, flip_ratio=None, direction='horizontal'): + if isinstance(flip_ratio, list): + assert mmcv.is_list_of(flip_ratio, float) + assert 0 <= sum(flip_ratio) <= 1 + elif isinstance(flip_ratio, float): + assert 0 <= flip_ratio <= 1 + elif flip_ratio is None: + pass + else: + raise ValueError('flip_ratios must be None, float, ' + 'or list of float') + self.flip_ratio = flip_ratio + + valid_directions = ['horizontal', 'vertical', 'diagonal'] + if isinstance(direction, str): + assert direction in valid_directions + elif isinstance(direction, list): + assert mmcv.is_list_of(direction, str) + assert set(direction).issubset(set(valid_directions)) + else: + raise ValueError('direction must be either str or list of str') + self.direction = direction + + if isinstance(flip_ratio, list): + assert len(self.flip_ratio) == len(self.direction) + + def bbox_flip(self, bboxes, img_shape, direction): + """Flip bboxes horizontally. + + Args: + bboxes (numpy.ndarray): Bounding boxes, shape (..., 4*k) + img_shape (tuple[int]): Image shape (height, width) + direction (str): Flip direction. Options are 'horizontal', + 'vertical'. + + Returns: + numpy.ndarray: Flipped bounding boxes. + """ + + assert bboxes.shape[-1] % 4 == 0 + flipped = bboxes.copy() + if direction == 'horizontal': + w = img_shape[1] + flipped[..., 0::4] = w - bboxes[..., 2::4] + flipped[..., 2::4] = w - bboxes[..., 0::4] + elif direction == 'vertical': + h = img_shape[0] + flipped[..., 1::4] = h - bboxes[..., 3::4] + flipped[..., 3::4] = h - bboxes[..., 1::4] + elif direction == 'diagonal': + w = img_shape[1] + h = img_shape[0] + flipped[..., 0::4] = w - bboxes[..., 2::4] + flipped[..., 1::4] = h - bboxes[..., 3::4] + flipped[..., 2::4] = w - bboxes[..., 0::4] + flipped[..., 3::4] = h - bboxes[..., 1::4] + else: + raise ValueError(f"Invalid flipping direction '{direction}'") + return flipped + + def __call__(self, results): + """Call function to flip bounding boxes, masks, semantic segmentation + maps. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Flipped results, 'flip', 'flip_direction' keys are added \ + into result dict. + """ + + if 'flip' not in results: + if isinstance(self.direction, list): + # None means non-flip + direction_list = self.direction + [None] + else: + # None means non-flip + direction_list = [self.direction, None] + + if isinstance(self.flip_ratio, list): + non_flip_ratio = 1 - sum(self.flip_ratio) + flip_ratio_list = self.flip_ratio + [non_flip_ratio] + else: + non_flip_ratio = 1 - self.flip_ratio + # exclude non-flip + single_ratio = self.flip_ratio / (len(direction_list) - 1) + flip_ratio_list = [single_ratio] * (len(direction_list) - + 1) + [non_flip_ratio] + + cur_dir = np.random.choice(direction_list, p=flip_ratio_list) + + results['flip'] = cur_dir is not None + if 'flip_direction' not in results: + results['flip_direction'] = cur_dir + if results['flip']: + # flip image + for key in results.get('img_fields', ['img']): + results[key] = mmcv.imflip( + results[key], direction=results['flip_direction']) + # flip bboxes + for key in results.get('bbox_fields', []): + results[key] = self.bbox_flip(results[key], + results['img_shape'], + results['flip_direction']) + # flip masks + for key in results.get('mask_fields', []): + results[key] = results[key].flip(results['flip_direction']) + + # flip segs + for key in results.get('seg_fields', []): + results[key] = mmcv.imflip( + results[key], direction=results['flip_direction']) + return results + + def __repr__(self): + return self.__class__.__name__ + f'(flip_ratio={self.flip_ratio})' + + +@PIPELINES.register_module() +class Pad(object): + """Pad the image & mask. + + There are two padding modes: (1) pad to a fixed size and (2) pad to the + minimum size that is divisible by some number. + Added keys are "pad_shape", "pad_fixed_size", "pad_size_divisor", + + Args: + size (tuple, optional): Fixed padding size. + size_divisor (int, optional): The divisor of padded size. + pad_val (float, optional): Padding value, 0 by default. + """ + + def __init__(self, size=None, size_divisor=None, pad_val=0): + self.size = size + self.size_divisor = size_divisor + self.pad_val = pad_val + # only one of size and size_divisor should be valid + assert size is not None or size_divisor is not None + assert size is None or size_divisor is None + + def _pad_img(self, results): + """Pad images according to ``self.size``.""" + for key in results.get('img_fields', ['img']): + if self.size is not None: + padded_img = mmcv.impad( + results[key], shape=self.size, pad_val=self.pad_val) + elif self.size_divisor is not None: + padded_img = mmcv.impad_to_multiple( + results[key], self.size_divisor, pad_val=self.pad_val) + results[key] = padded_img + results['pad_shape'] = padded_img.shape + results['pad_fixed_size'] = self.size + results['pad_size_divisor'] = self.size_divisor + + def _pad_masks(self, results): + """Pad masks according to ``results['pad_shape']``.""" + pad_shape = results['pad_shape'][:2] + for key in results.get('mask_fields', []): + results[key] = results[key].pad(pad_shape, pad_val=self.pad_val) + + def _pad_seg(self, results): + """Pad semantic segmentation map according to + ``results['pad_shape']``.""" + for key in results.get('seg_fields', []): + results[key] = mmcv.impad( + results[key], shape=results['pad_shape'][:2]) + + def __call__(self, results): + """Call function to pad images, masks, semantic segmentation maps. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Updated result dict. + """ + self._pad_img(results) + self._pad_masks(results) + self._pad_seg(results) + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(size={self.size}, ' + repr_str += f'size_divisor={self.size_divisor}, ' + repr_str += f'pad_val={self.pad_val})' + return repr_str + + +@PIPELINES.register_module() +class Normalize(object): + """Normalize the image. + + Added key is "img_norm_cfg". + + Args: + mean (sequence): Mean values of 3 channels. + std (sequence): Std values of 3 channels. + to_rgb (bool): Whether to convert the image from BGR to RGB, + default is true. + """ + + def __init__(self, mean, std, to_rgb=True): + self.mean = np.array(mean, dtype=np.float32) + self.std = np.array(std, dtype=np.float32) + self.to_rgb = to_rgb + + def __call__(self, results): + """Call function to normalize images. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Normalized results, 'img_norm_cfg' key is added into + result dict. + """ + for key in results.get('img_fields', ['img']): + results[key] = mmcv.imnormalize(results[key], self.mean, self.std, + self.to_rgb) + results['img_norm_cfg'] = dict( + mean=self.mean, std=self.std, to_rgb=self.to_rgb) + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(mean={self.mean}, std={self.std}, to_rgb={self.to_rgb})' + return repr_str + + +@PIPELINES.register_module() +class RandomCrop(object): + """Random crop the image & bboxes & masks. + + The absolute `crop_size` is sampled based on `crop_type` and `image_size`, + then the cropped results are generated. + + Args: + crop_size (tuple): The relative ratio or absolute pixels of + height and width. + crop_type (str, optional): one of "relative_range", "relative", + "absolute", "absolute_range". "relative" randomly crops + (h * crop_size[0], w * crop_size[1]) part from an input of size + (h, w). "relative_range" uniformly samples relative crop size from + range [crop_size[0], 1] and [crop_size[1], 1] for height and width + respectively. "absolute" crops from an input with absolute size + (crop_size[0], crop_size[1]). "absolute_range" uniformly samples + crop_h in range [crop_size[0], min(h, crop_size[1])] and crop_w + in range [crop_size[0], min(w, crop_size[1])]. Default "absolute". + allow_negative_crop (bool, optional): Whether to allow a crop that does + not contain any bbox area. Default False. + bbox_clip_border (bool, optional): Whether clip the objects outside + the border of the image. Defaults to True. + + Note: + - If the image is smaller than the absolute crop size, return the + original image. + - The keys for bboxes, labels and masks must be aligned. That is, + `gt_bboxes` corresponds to `gt_labels` and `gt_masks`, and + `gt_bboxes_ignore` corresponds to `gt_labels_ignore` and + `gt_masks_ignore`. + - If the crop does not contain any gt-bbox region and + `allow_negative_crop` is set to False, skip this image. + """ + + def __init__(self, + crop_size, + crop_type='absolute', + allow_negative_crop=False, + bbox_clip_border=True): + if crop_type not in [ + 'relative_range', 'relative', 'absolute', 'absolute_range' + ]: + raise ValueError(f'Invalid crop_type {crop_type}.') + if crop_type in ['absolute', 'absolute_range']: + assert crop_size[0] > 0 and crop_size[1] > 0 + assert isinstance(crop_size[0], int) and isinstance( + crop_size[1], int) + else: + assert 0 < crop_size[0] <= 1 and 0 < crop_size[1] <= 1 + self.crop_size = crop_size + self.crop_type = crop_type + self.allow_negative_crop = allow_negative_crop + self.bbox_clip_border = bbox_clip_border + # The key correspondence from bboxes to labels and masks. + self.bbox2label = { + 'gt_bboxes': 'gt_labels', + 'gt_bboxes_ignore': 'gt_labels_ignore' + } + self.bbox2mask = { + 'gt_bboxes': 'gt_masks', + 'gt_bboxes_ignore': 'gt_masks_ignore' + } + + def _crop_data(self, results, crop_size, allow_negative_crop): + """Function to randomly crop images, bounding boxes, masks, semantic + segmentation maps. + + Args: + results (dict): Result dict from loading pipeline. + crop_size (tuple): Expected absolute size after cropping, (h, w). + allow_negative_crop (bool): Whether to allow a crop that does not + contain any bbox area. Default to False. + + Returns: + dict: Randomly cropped results, 'img_shape' key in result dict is + updated according to crop size. + """ + assert crop_size[0] > 0 and crop_size[1] > 0 + for key in results.get('img_fields', ['img']): + img = results[key] + margin_h = max(img.shape[0] - crop_size[0], 0) + margin_w = max(img.shape[1] - crop_size[1], 0) + offset_h = np.random.randint(0, margin_h + 1) + offset_w = np.random.randint(0, margin_w + 1) + crop_y1, crop_y2 = offset_h, offset_h + crop_size[0] + crop_x1, crop_x2 = offset_w, offset_w + crop_size[1] + + # crop the image + img = img[crop_y1:crop_y2, crop_x1:crop_x2, ...] + img_shape = img.shape + results[key] = img + results['img_shape'] = img_shape + + # crop bboxes accordingly and clip to the image boundary + for key in results.get('bbox_fields', []): + # e.g. gt_bboxes and gt_bboxes_ignore + bbox_offset = np.array([offset_w, offset_h, offset_w, offset_h], + dtype=np.float32) + bboxes = results[key] - bbox_offset + if self.bbox_clip_border: + bboxes[:, 0::2] = np.clip(bboxes[:, 0::2], 0, img_shape[1]) + bboxes[:, 1::2] = np.clip(bboxes[:, 1::2], 0, img_shape[0]) + valid_inds = (bboxes[:, 2] > bboxes[:, 0]) & ( + bboxes[:, 3] > bboxes[:, 1]) + # If the crop does not contain any gt-bbox area and + # allow_negative_crop is False, skip this image. + if (key == 'gt_bboxes' and not valid_inds.any() + and not allow_negative_crop): + return None + results[key] = bboxes[valid_inds, :] + # label fields. e.g. gt_labels and gt_labels_ignore + label_key = self.bbox2label.get(key) + if label_key in results: + results[label_key] = results[label_key][valid_inds] + + # mask fields, e.g. gt_masks and gt_masks_ignore + mask_key = self.bbox2mask.get(key) + if mask_key in results: + results[mask_key] = results[mask_key][ + valid_inds.nonzero()[0]].crop( + np.asarray([crop_x1, crop_y1, crop_x2, crop_y2])) + + # crop semantic seg + for key in results.get('seg_fields', []): + results[key] = results[key][crop_y1:crop_y2, crop_x1:crop_x2] + + return results + + def _get_crop_size(self, image_size): + """Randomly generates the absolute crop size based on `crop_type` and + `image_size`. + + Args: + image_size (tuple): (h, w). + + Returns: + crop_size (tuple): (crop_h, crop_w) in absolute pixels. + """ + h, w = image_size + if self.crop_type == 'absolute': + return (min(self.crop_size[0], h), min(self.crop_size[1], w)) + elif self.crop_type == 'absolute_range': + assert self.crop_size[0] <= self.crop_size[1] + crop_h = np.random.randint( + min(h, self.crop_size[0]), + min(h, self.crop_size[1]) + 1) + crop_w = np.random.randint( + min(w, self.crop_size[0]), + min(w, self.crop_size[1]) + 1) + return crop_h, crop_w + elif self.crop_type == 'relative': + crop_h, crop_w = self.crop_size + return int(h * crop_h + 0.5), int(w * crop_w + 0.5) + elif self.crop_type == 'relative_range': + crop_size = np.asarray(self.crop_size, dtype=np.float32) + crop_h, crop_w = crop_size + np.random.rand(2) * (1 - crop_size) + return int(h * crop_h + 0.5), int(w * crop_w + 0.5) + + def __call__(self, results): + """Call function to randomly crop images, bounding boxes, masks, + semantic segmentation maps. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Randomly cropped results, 'img_shape' key in result dict is + updated according to crop size. + """ + image_size = results['img'].shape[:2] + crop_size = self._get_crop_size(image_size) + results = self._crop_data(results, crop_size, self.allow_negative_crop) + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(crop_size={self.crop_size}, ' + repr_str += f'crop_type={self.crop_type}, ' + repr_str += f'allow_negative_crop={self.allow_negative_crop}, ' + repr_str += f'bbox_clip_border={self.bbox_clip_border})' + return repr_str + + +@PIPELINES.register_module() +class SegRescale(object): + """Rescale semantic segmentation maps. + + Args: + scale_factor (float): The scale factor of the final output. + backend (str): Image rescale backend, choices are 'cv2' and 'pillow'. + These two backends generates slightly different results. Defaults + to 'cv2'. + """ + + def __init__(self, scale_factor=1, backend='cv2'): + self.scale_factor = scale_factor + self.backend = backend + + def __call__(self, results): + """Call function to scale the semantic segmentation map. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Result dict with semantic segmentation map scaled. + """ + + for key in results.get('seg_fields', []): + if self.scale_factor != 1: + results[key] = mmcv.imrescale( + results[key], + self.scale_factor, + interpolation='nearest', + backend=self.backend) + return results + + def __repr__(self): + return self.__class__.__name__ + f'(scale_factor={self.scale_factor})' + + +@PIPELINES.register_module() +class PhotoMetricDistortion(object): + """Apply photometric distortion to image sequentially, every transformation + is applied with a probability of 0.5. The position of random contrast is in + second or second to last. + + 1. random brightness + 2. random contrast (mode 0) + 3. convert color from BGR to HSV + 4. random saturation + 5. random hue + 6. convert color from HSV to BGR + 7. random contrast (mode 1) + 8. randomly swap channels + + Args: + brightness_delta (int): delta of brightness. + contrast_range (tuple): range of contrast. + saturation_range (tuple): range of saturation. + hue_delta (int): delta of hue. + """ + + def __init__(self, + brightness_delta=32, + contrast_range=(0.5, 1.5), + saturation_range=(0.5, 1.5), + hue_delta=18): + self.brightness_delta = brightness_delta + self.contrast_lower, self.contrast_upper = contrast_range + self.saturation_lower, self.saturation_upper = saturation_range + self.hue_delta = hue_delta + + def __call__(self, results): + """Call function to perform photometric distortion on images. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Result dict with images distorted. + """ + + if 'img_fields' in results: + assert results['img_fields'] == ['img'], \ + 'Only single img_fields is allowed' + img = results['img'] + assert img.dtype == np.float32, \ + 'PhotoMetricDistortion needs the input image of dtype np.float32,'\ + ' please set "to_float32=True" in "LoadImageFromFile" pipeline' + # random brightness + if random.randint(2): + delta = random.uniform(-self.brightness_delta, + self.brightness_delta) + img += delta + + # mode == 0 --> do random contrast first + # mode == 1 --> do random contrast last + mode = random.randint(2) + if mode == 1: + if random.randint(2): + alpha = random.uniform(self.contrast_lower, + self.contrast_upper) + img *= alpha + + # convert color from BGR to HSV + img = mmcv.bgr2hsv(img) + + # random saturation + if random.randint(2): + img[..., 1] *= random.uniform(self.saturation_lower, + self.saturation_upper) + + # random hue + if random.randint(2): + img[..., 0] += random.uniform(-self.hue_delta, self.hue_delta) + img[..., 0][img[..., 0] > 360] -= 360 + img[..., 0][img[..., 0] < 0] += 360 + + # convert color from HSV to BGR + img = mmcv.hsv2bgr(img) + + # random contrast + if mode == 0: + if random.randint(2): + alpha = random.uniform(self.contrast_lower, + self.contrast_upper) + img *= alpha + + # randomly swap channels + if random.randint(2): + img = img[..., random.permutation(3)] + + results['img'] = img + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(\nbrightness_delta={self.brightness_delta},\n' + repr_str += 'contrast_range=' + repr_str += f'{(self.contrast_lower, self.contrast_upper)},\n' + repr_str += 'saturation_range=' + repr_str += f'{(self.saturation_lower, self.saturation_upper)},\n' + repr_str += f'hue_delta={self.hue_delta})' + return repr_str + + +@PIPELINES.register_module() +class Expand(object): + """Random expand the image & bboxes. + + Randomly place the original image on a canvas of 'ratio' x original image + size filled with mean values. The ratio is in the range of ratio_range. + + Args: + mean (tuple): mean value of dataset. + to_rgb (bool): if need to convert the order of mean to align with RGB. + ratio_range (tuple): range of expand ratio. + prob (float): probability of applying this transformation + """ + + def __init__(self, + mean=(0, 0, 0), + to_rgb=True, + ratio_range=(1, 4), + seg_ignore_label=None, + prob=0.5): + self.to_rgb = to_rgb + self.ratio_range = ratio_range + if to_rgb: + self.mean = mean[::-1] + else: + self.mean = mean + self.min_ratio, self.max_ratio = ratio_range + self.seg_ignore_label = seg_ignore_label + self.prob = prob + + def __call__(self, results): + """Call function to expand images, bounding boxes. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Result dict with images, bounding boxes expanded + """ + + if random.uniform(0, 1) > self.prob: + return results + + if 'img_fields' in results: + assert results['img_fields'] == ['img'], \ + 'Only single img_fields is allowed' + img = results['img'] + + h, w, c = img.shape + ratio = random.uniform(self.min_ratio, self.max_ratio) + # speedup expand when meets large image + if np.all(self.mean == self.mean[0]): + expand_img = np.empty((int(h * ratio), int(w * ratio), c), + img.dtype) + expand_img.fill(self.mean[0]) + else: + expand_img = np.full((int(h * ratio), int(w * ratio), c), + self.mean, + dtype=img.dtype) + left = int(random.uniform(0, w * ratio - w)) + top = int(random.uniform(0, h * ratio - h)) + expand_img[top:top + h, left:left + w] = img + + results['img'] = expand_img + # expand bboxes + for key in results.get('bbox_fields', []): + results[key] = results[key] + np.tile( + (left, top), 2).astype(results[key].dtype) + + # expand masks + for key in results.get('mask_fields', []): + results[key] = results[key].expand( + int(h * ratio), int(w * ratio), top, left) + + # expand segs + for key in results.get('seg_fields', []): + gt_seg = results[key] + expand_gt_seg = np.full((int(h * ratio), int(w * ratio)), + self.seg_ignore_label, + dtype=gt_seg.dtype) + expand_gt_seg[top:top + h, left:left + w] = gt_seg + results[key] = expand_gt_seg + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(mean={self.mean}, to_rgb={self.to_rgb}, ' + repr_str += f'ratio_range={self.ratio_range}, ' + repr_str += f'seg_ignore_label={self.seg_ignore_label})' + return repr_str + + +@PIPELINES.register_module() +class MinIoURandomCrop(object): + """Random crop the image & bboxes, the cropped patches have minimum IoU + requirement with original image & bboxes, the IoU threshold is randomly + selected from min_ious. + + Args: + min_ious (tuple): minimum IoU threshold for all intersections with + bounding boxes + min_crop_size (float): minimum crop's size (i.e. h,w := a*h, a*w, + where a >= min_crop_size). + bbox_clip_border (bool, optional): Whether clip the objects outside + the border of the image. Defaults to True. + + Note: + The keys for bboxes, labels and masks should be paired. That is, \ + `gt_bboxes` corresponds to `gt_labels` and `gt_masks`, and \ + `gt_bboxes_ignore` to `gt_labels_ignore` and `gt_masks_ignore`. + """ + + def __init__(self, + min_ious=(0.1, 0.3, 0.5, 0.7, 0.9), + min_crop_size=0.3, + bbox_clip_border=True): + # 1: return ori img + self.min_ious = min_ious + self.sample_mode = (1, *min_ious, 0) + self.min_crop_size = min_crop_size + self.bbox_clip_border = bbox_clip_border + self.bbox2label = { + 'gt_bboxes': 'gt_labels', + 'gt_bboxes_ignore': 'gt_labels_ignore' + } + self.bbox2mask = { + 'gt_bboxes': 'gt_masks', + 'gt_bboxes_ignore': 'gt_masks_ignore' + } + + def __call__(self, results): + """Call function to crop images and bounding boxes with minimum IoU + constraint. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Result dict with images and bounding boxes cropped, \ + 'img_shape' key is updated. + """ + + if 'img_fields' in results: + assert results['img_fields'] == ['img'], \ + 'Only single img_fields is allowed' + img = results['img'] + assert 'bbox_fields' in results + boxes = [results[key] for key in results['bbox_fields']] + boxes = np.concatenate(boxes, 0) + h, w, c = img.shape + while True: + mode = random.choice(self.sample_mode) + self.mode = mode + if mode == 1: + return results + + min_iou = mode + for i in range(50): + new_w = random.uniform(self.min_crop_size * w, w) + new_h = random.uniform(self.min_crop_size * h, h) + + # h / w in [0.5, 2] + if new_h / new_w < 0.5 or new_h / new_w > 2: + continue + + left = random.uniform(w - new_w) + top = random.uniform(h - new_h) + + patch = np.array( + (int(left), int(top), int(left + new_w), int(top + new_h))) + # Line or point crop is not allowed + if patch[2] == patch[0] or patch[3] == patch[1]: + continue + overlaps = bbox_overlaps( + patch.reshape(-1, 4), boxes.reshape(-1, 4)).reshape(-1) + if len(overlaps) > 0 and overlaps.min() < min_iou: + continue + + # center of boxes should inside the crop img + # only adjust boxes and instance masks when the gt is not empty + if len(overlaps) > 0: + # adjust boxes + def is_center_of_bboxes_in_patch(boxes, patch): + center = (boxes[:, :2] + boxes[:, 2:]) / 2 + mask = ((center[:, 0] > patch[0]) * + (center[:, 1] > patch[1]) * + (center[:, 0] < patch[2]) * + (center[:, 1] < patch[3])) + return mask + + mask = is_center_of_bboxes_in_patch(boxes, patch) + if not mask.any(): + continue + for key in results.get('bbox_fields', []): + boxes = results[key].copy() + mask = is_center_of_bboxes_in_patch(boxes, patch) + boxes = boxes[mask] + if self.bbox_clip_border: + boxes[:, 2:] = boxes[:, 2:].clip(max=patch[2:]) + boxes[:, :2] = boxes[:, :2].clip(min=patch[:2]) + boxes -= np.tile(patch[:2], 2) + + results[key] = boxes + # labels + label_key = self.bbox2label.get(key) + if label_key in results: + results[label_key] = results[label_key][mask] + + # mask fields + mask_key = self.bbox2mask.get(key) + if mask_key in results: + results[mask_key] = results[mask_key][ + mask.nonzero()[0]].crop(patch) + # adjust the img no matter whether the gt is empty before crop + img = img[patch[1]:patch[3], patch[0]:patch[2]] + results['img'] = img + results['img_shape'] = img.shape + + # seg fields + for key in results.get('seg_fields', []): + results[key] = results[key][patch[1]:patch[3], + patch[0]:patch[2]] + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(min_ious={self.min_ious}, ' + repr_str += f'min_crop_size={self.min_crop_size}, ' + repr_str += f'bbox_clip_border={self.bbox_clip_border})' + return repr_str + + +@PIPELINES.register_module() +class Corrupt(object): + """Corruption augmentation. + + Corruption transforms implemented based on + `imagecorruptions `_. + + Args: + corruption (str): Corruption name. + severity (int, optional): The severity of corruption. Default: 1. + """ + + def __init__(self, corruption, severity=1): + self.corruption = corruption + self.severity = severity + + def __call__(self, results): + """Call function to corrupt image. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Result dict with images corrupted. + """ + + if corrupt is None: + raise RuntimeError('imagecorruptions is not installed') + if 'img_fields' in results: + assert results['img_fields'] == ['img'], \ + 'Only single img_fields is allowed' + results['img'] = corrupt( + results['img'].astype(np.uint8), + corruption_name=self.corruption, + severity=self.severity) + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(corruption={self.corruption}, ' + repr_str += f'severity={self.severity})' + return repr_str + + +@PIPELINES.register_module() +class Albu(object): + """Albumentation augmentation. + + Adds custom transformations from Albumentations library. + Please, visit `https://albumentations.readthedocs.io` + to get more information. + + An example of ``transforms`` is as followed: + + .. code-block:: + + [ + dict( + type='ShiftScaleRotate', + shift_limit=0.0625, + scale_limit=0.0, + rotate_limit=0, + interpolation=1, + p=0.5), + dict( + type='RandomBrightnessContrast', + brightness_limit=[0.1, 0.3], + contrast_limit=[0.1, 0.3], + p=0.2), + dict(type='ChannelShuffle', p=0.1), + dict( + type='OneOf', + transforms=[ + dict(type='Blur', blur_limit=3, p=1.0), + dict(type='MedianBlur', blur_limit=3, p=1.0) + ], + p=0.1), + ] + + Args: + transforms (list[dict]): A list of albu transformations + bbox_params (dict): Bbox_params for albumentation `Compose` + keymap (dict): Contains {'input key':'albumentation-style key'} + skip_img_without_anno (bool): Whether to skip the image if no ann left + after aug + """ + + def __init__(self, + transforms, + bbox_params=None, + keymap=None, + update_pad_shape=False, + skip_img_without_anno=False): + if Compose is None: + raise RuntimeError('albumentations is not installed') + + # Args will be modified later, copying it will be safer + transforms = copy.deepcopy(transforms) + if bbox_params is not None: + bbox_params = copy.deepcopy(bbox_params) + if keymap is not None: + keymap = copy.deepcopy(keymap) + self.transforms = transforms + self.filter_lost_elements = False + self.update_pad_shape = update_pad_shape + self.skip_img_without_anno = skip_img_without_anno + + # A simple workaround to remove masks without boxes + if (isinstance(bbox_params, dict) and 'label_fields' in bbox_params + and 'filter_lost_elements' in bbox_params): + self.filter_lost_elements = True + self.origin_label_fields = bbox_params['label_fields'] + bbox_params['label_fields'] = ['idx_mapper'] + del bbox_params['filter_lost_elements'] + + self.bbox_params = ( + self.albu_builder(bbox_params) if bbox_params else None) + self.aug = Compose([self.albu_builder(t) for t in self.transforms], + bbox_params=self.bbox_params) + + if not keymap: + self.keymap_to_albu = { + 'img': 'image', + 'gt_masks': 'masks', + 'gt_bboxes': 'bboxes' + } + else: + self.keymap_to_albu = keymap + self.keymap_back = {v: k for k, v in self.keymap_to_albu.items()} + + def albu_builder(self, cfg): + """Import a module from albumentations. + + It inherits some of :func:`build_from_cfg` logic. + + Args: + cfg (dict): Config dict. It should at least contain the key "type". + + Returns: + obj: The constructed object. + """ + + assert isinstance(cfg, dict) and 'type' in cfg + args = cfg.copy() + + obj_type = args.pop('type') + if mmcv.is_str(obj_type): + if albumentations is None: + raise RuntimeError('albumentations is not installed') + obj_cls = getattr(albumentations, obj_type) + elif inspect.isclass(obj_type): + obj_cls = obj_type + else: + raise TypeError( + f'type must be a str or valid type, but got {type(obj_type)}') + + if 'transforms' in args: + args['transforms'] = [ + self.albu_builder(transform) + for transform in args['transforms'] + ] + + return obj_cls(**args) + + @staticmethod + def mapper(d, keymap): + """Dictionary mapper. Renames keys according to keymap provided. + + Args: + d (dict): old dict + keymap (dict): {'old_key':'new_key'} + Returns: + dict: new dict. + """ + + updated_dict = {} + for k, v in zip(d.keys(), d.values()): + new_k = keymap.get(k, k) + updated_dict[new_k] = d[k] + return updated_dict + + def __call__(self, results): + # dict to albumentations format + results = self.mapper(results, self.keymap_to_albu) + # TODO: add bbox_fields + if 'bboxes' in results: + # to list of boxes + if isinstance(results['bboxes'], np.ndarray): + results['bboxes'] = [x for x in results['bboxes']] + # add pseudo-field for filtration + if self.filter_lost_elements: + results['idx_mapper'] = np.arange(len(results['bboxes'])) + + # TODO: Support mask structure in albu + if 'masks' in results: + if isinstance(results['masks'], PolygonMasks): + raise NotImplementedError( + 'Albu only supports BitMap masks now') + ori_masks = results['masks'] + if albumentations.__version__ < '0.5': + results['masks'] = results['masks'].masks + else: + results['masks'] = [mask for mask in results['masks'].masks] + + results = self.aug(**results) + + if 'bboxes' in results: + if isinstance(results['bboxes'], list): + results['bboxes'] = np.array( + results['bboxes'], dtype=np.float32) + results['bboxes'] = results['bboxes'].reshape(-1, 4) + + # filter label_fields + if self.filter_lost_elements: + + for label in self.origin_label_fields: + results[label] = np.array( + [results[label][i] for i in results['idx_mapper']]) + if 'masks' in results: + results['masks'] = np.array( + [results['masks'][i] for i in results['idx_mapper']]) + results['masks'] = ori_masks.__class__( + results['masks'], results['image'].shape[0], + results['image'].shape[1]) + + if (not len(results['idx_mapper']) + and self.skip_img_without_anno): + return None + + if 'gt_labels' in results: + if isinstance(results['gt_labels'], list): + results['gt_labels'] = np.array(results['gt_labels']) + results['gt_labels'] = results['gt_labels'].astype(np.int64) + + # back to the original format + results = self.mapper(results, self.keymap_back) + + # update final shape + if self.update_pad_shape: + results['pad_shape'] = results['img'].shape + + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + f'(transforms={self.transforms})' + return repr_str + + +@PIPELINES.register_module() +class RandomCenterCropPad(object): + """Random center crop and random around padding for CornerNet. + + This operation generates randomly cropped image from the original image and + pads it simultaneously. Different from :class:`RandomCrop`, the output + shape may not equal to ``crop_size`` strictly. We choose a random value + from ``ratios`` and the output shape could be larger or smaller than + ``crop_size``. The padding operation is also different from :class:`Pad`, + here we use around padding instead of right-bottom padding. + + The relation between output image (padding image) and original image: + + .. code:: text + + output image + + +----------------------------+ + | padded area | + +------|----------------------------|----------+ + | | cropped area | | + | | +---------------+ | | + | | | . center | | | original image + | | | range | | | + | | +---------------+ | | + +------|----------------------------|----------+ + | padded area | + +----------------------------+ + + There are 5 main areas in the figure: + + - output image: output image of this operation, also called padding + image in following instruction. + - original image: input image of this operation. + - padded area: non-intersect area of output image and original image. + - cropped area: the overlap of output image and original image. + - center range: a smaller area where random center chosen from. + center range is computed by ``border`` and original image's shape + to avoid our random center is too close to original image's border. + + Also this operation act differently in train and test mode, the summary + pipeline is listed below. + + Train pipeline: + + 1. Choose a ``random_ratio`` from ``ratios``, the shape of padding image + will be ``random_ratio * crop_size``. + 2. Choose a ``random_center`` in center range. + 3. Generate padding image with center matches the ``random_center``. + 4. Initialize the padding image with pixel value equals to ``mean``. + 5. Copy the cropped area to padding image. + 6. Refine annotations. + + Test pipeline: + + 1. Compute output shape according to ``test_pad_mode``. + 2. Generate padding image with center matches the original image + center. + 3. Initialize the padding image with pixel value equals to ``mean``. + 4. Copy the ``cropped area`` to padding image. + + Args: + crop_size (tuple | None): expected size after crop, final size will + computed according to ratio. Requires (h, w) in train mode, and + None in test mode. + ratios (tuple): random select a ratio from tuple and crop image to + (crop_size[0] * ratio) * (crop_size[1] * ratio). + Only available in train mode. + border (int): max distance from center select area to image border. + Only available in train mode. + mean (sequence): Mean values of 3 channels. + std (sequence): Std values of 3 channels. + to_rgb (bool): Whether to convert the image from BGR to RGB. + test_mode (bool): whether involve random variables in transform. + In train mode, crop_size is fixed, center coords and ratio is + random selected from predefined lists. In test mode, crop_size + is image's original shape, center coords and ratio is fixed. + test_pad_mode (tuple): padding method and padding shape value, only + available in test mode. Default is using 'logical_or' with + 127 as padding shape value. + + - 'logical_or': final_shape = input_shape | padding_shape_value + - 'size_divisor': final_shape = int( + ceil(input_shape / padding_shape_value) * padding_shape_value) + bbox_clip_border (bool, optional): Whether clip the objects outside + the border of the image. Defaults to True. + """ + + def __init__(self, + crop_size=None, + ratios=(0.9, 1.0, 1.1), + border=128, + mean=None, + std=None, + to_rgb=None, + test_mode=False, + test_pad_mode=('logical_or', 127), + bbox_clip_border=True): + if test_mode: + assert crop_size is None, 'crop_size must be None in test mode' + assert ratios is None, 'ratios must be None in test mode' + assert border is None, 'border must be None in test mode' + assert isinstance(test_pad_mode, (list, tuple)) + assert test_pad_mode[0] in ['logical_or', 'size_divisor'] + else: + assert isinstance(crop_size, (list, tuple)) + assert crop_size[0] > 0 and crop_size[1] > 0, ( + 'crop_size must > 0 in train mode') + assert isinstance(ratios, (list, tuple)) + assert test_pad_mode is None, ( + 'test_pad_mode must be None in train mode') + + self.crop_size = crop_size + self.ratios = ratios + self.border = border + # We do not set default value to mean, std and to_rgb because these + # hyper-parameters are easy to forget but could affect the performance. + # Please use the same setting as Normalize for performance assurance. + assert mean is not None and std is not None and to_rgb is not None + self.to_rgb = to_rgb + self.input_mean = mean + self.input_std = std + if to_rgb: + self.mean = mean[::-1] + self.std = std[::-1] + else: + self.mean = mean + self.std = std + self.test_mode = test_mode + self.test_pad_mode = test_pad_mode + self.bbox_clip_border = bbox_clip_border + + def _get_border(self, border, size): + """Get final border for the target size. + + This function generates a ``final_border`` according to image's shape. + The area between ``final_border`` and ``size - final_border`` is the + ``center range``. We randomly choose center from the ``center range`` + to avoid our random center is too close to original image's border. + Also ``center range`` should be larger than 0. + + Args: + border (int): The initial border, default is 128. + size (int): The width or height of original image. + Returns: + int: The final border. + """ + k = 2 * border / size + i = pow(2, np.ceil(np.log2(np.ceil(k))) + (k == int(k))) + return border // i + + def _filter_boxes(self, patch, boxes): + """Check whether the center of each box is in the patch. + + Args: + patch (list[int]): The cropped area, [left, top, right, bottom]. + boxes (numpy array, (N x 4)): Ground truth boxes. + + Returns: + mask (numpy array, (N,)): Each box is inside or outside the patch. + """ + center = (boxes[:, :2] + boxes[:, 2:]) / 2 + mask = (center[:, 0] > patch[0]) * (center[:, 1] > patch[1]) * ( + center[:, 0] < patch[2]) * ( + center[:, 1] < patch[3]) + return mask + + def _crop_image_and_paste(self, image, center, size): + """Crop image with a given center and size, then paste the cropped + image to a blank image with two centers align. + + This function is equivalent to generating a blank image with ``size`` + as its shape. Then cover it on the original image with two centers ( + the center of blank image and the random center of original image) + aligned. The overlap area is paste from the original image and the + outside area is filled with ``mean pixel``. + + Args: + image (np array, H x W x C): Original image. + center (list[int]): Target crop center coord. + size (list[int]): Target crop size. [target_h, target_w] + + Returns: + cropped_img (np array, target_h x target_w x C): Cropped image. + border (np array, 4): The distance of four border of + ``cropped_img`` to the original image area, [top, bottom, + left, right] + patch (list[int]): The cropped area, [left, top, right, bottom]. + """ + center_y, center_x = center + target_h, target_w = size + img_h, img_w, img_c = image.shape + + x0 = max(0, center_x - target_w // 2) + x1 = min(center_x + target_w // 2, img_w) + y0 = max(0, center_y - target_h // 2) + y1 = min(center_y + target_h // 2, img_h) + patch = np.array((int(x0), int(y0), int(x1), int(y1))) + + left, right = center_x - x0, x1 - center_x + top, bottom = center_y - y0, y1 - center_y + + cropped_center_y, cropped_center_x = target_h // 2, target_w // 2 + cropped_img = np.zeros((target_h, target_w, img_c), dtype=image.dtype) + for i in range(img_c): + cropped_img[:, :, i] += self.mean[i] + y_slice = slice(cropped_center_y - top, cropped_center_y + bottom) + x_slice = slice(cropped_center_x - left, cropped_center_x + right) + cropped_img[y_slice, x_slice, :] = image[y0:y1, x0:x1, :] + + border = np.array([ + cropped_center_y - top, cropped_center_y + bottom, + cropped_center_x - left, cropped_center_x + right + ], + dtype=np.float32) + + return cropped_img, border, patch + + def _train_aug(self, results): + """Random crop and around padding the original image. + + Args: + results (dict): Image infomations in the augment pipeline. + + Returns: + results (dict): The updated dict. + """ + img = results['img'] + h, w, c = img.shape + boxes = results['gt_bboxes'] + while True: + scale = random.choice(self.ratios) + new_h = int(self.crop_size[0] * scale) + new_w = int(self.crop_size[1] * scale) + h_border = self._get_border(self.border, h) + w_border = self._get_border(self.border, w) + + for i in range(50): + center_x = random.randint(low=w_border, high=w - w_border) + center_y = random.randint(low=h_border, high=h - h_border) + + cropped_img, border, patch = self._crop_image_and_paste( + img, [center_y, center_x], [new_h, new_w]) + + mask = self._filter_boxes(patch, boxes) + # if image do not have valid bbox, any crop patch is valid. + if not mask.any() and len(boxes) > 0: + continue + + results['img'] = cropped_img + results['img_shape'] = cropped_img.shape + results['pad_shape'] = cropped_img.shape + + x0, y0, x1, y1 = patch + + left_w, top_h = center_x - x0, center_y - y0 + cropped_center_x, cropped_center_y = new_w // 2, new_h // 2 + + # crop bboxes accordingly and clip to the image boundary + for key in results.get('bbox_fields', []): + mask = self._filter_boxes(patch, results[key]) + bboxes = results[key][mask] + bboxes[:, 0:4:2] += cropped_center_x - left_w - x0 + bboxes[:, 1:4:2] += cropped_center_y - top_h - y0 + if self.bbox_clip_border: + bboxes[:, 0:4:2] = np.clip(bboxes[:, 0:4:2], 0, new_w) + bboxes[:, 1:4:2] = np.clip(bboxes[:, 1:4:2], 0, new_h) + keep = (bboxes[:, 2] > bboxes[:, 0]) & ( + bboxes[:, 3] > bboxes[:, 1]) + bboxes = bboxes[keep] + results[key] = bboxes + if key in ['gt_bboxes']: + if 'gt_labels' in results: + labels = results['gt_labels'][mask] + labels = labels[keep] + results['gt_labels'] = labels + if 'gt_masks' in results: + raise NotImplementedError( + 'RandomCenterCropPad only supports bbox.') + + # crop semantic seg + for key in results.get('seg_fields', []): + raise NotImplementedError( + 'RandomCenterCropPad only supports bbox.') + return results + + def _test_aug(self, results): + """Around padding the original image without cropping. + + The padding mode and value are from ``test_pad_mode``. + + Args: + results (dict): Image infomations in the augment pipeline. + + Returns: + results (dict): The updated dict. + """ + img = results['img'] + h, w, c = img.shape + results['img_shape'] = img.shape + if self.test_pad_mode[0] in ['logical_or']: + target_h = h | self.test_pad_mode[1] + target_w = w | self.test_pad_mode[1] + elif self.test_pad_mode[0] in ['size_divisor']: + divisor = self.test_pad_mode[1] + target_h = int(np.ceil(h / divisor)) * divisor + target_w = int(np.ceil(w / divisor)) * divisor + else: + raise NotImplementedError( + 'RandomCenterCropPad only support two testing pad mode:' + 'logical-or and size_divisor.') + + cropped_img, border, _ = self._crop_image_and_paste( + img, [h // 2, w // 2], [target_h, target_w]) + results['img'] = cropped_img + results['pad_shape'] = cropped_img.shape + results['border'] = border + return results + + def __call__(self, results): + img = results['img'] + assert img.dtype == np.float32, ( + 'RandomCenterCropPad needs the input image of dtype np.float32,' + ' please set "to_float32=True" in "LoadImageFromFile" pipeline') + h, w, c = img.shape + assert c == len(self.mean) + if self.test_mode: + return self._test_aug(results) + else: + return self._train_aug(results) + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(crop_size={self.crop_size}, ' + repr_str += f'ratios={self.ratios}, ' + repr_str += f'border={self.border}, ' + repr_str += f'mean={self.input_mean}, ' + repr_str += f'std={self.input_std}, ' + repr_str += f'to_rgb={self.to_rgb}, ' + repr_str += f'test_mode={self.test_mode}, ' + repr_str += f'test_pad_mode={self.test_pad_mode}, ' + repr_str += f'bbox_clip_border={self.bbox_clip_border})' + return repr_str + + +@PIPELINES.register_module() +class CutOut(object): + """CutOut operation. + + Randomly drop some regions of image used in + `Cutout `_. + + Args: + n_holes (int | tuple[int, int]): Number of regions to be dropped. + If it is given as a list, number of holes will be randomly + selected from the closed interval [`n_holes[0]`, `n_holes[1]`]. + cutout_shape (tuple[int, int] | list[tuple[int, int]]): The candidate + shape of dropped regions. It can be `tuple[int, int]` to use a + fixed cutout shape, or `list[tuple[int, int]]` to randomly choose + shape from the list. + cutout_ratio (tuple[float, float] | list[tuple[float, float]]): The + candidate ratio of dropped regions. It can be `tuple[float, float]` + to use a fixed ratio or `list[tuple[float, float]]` to randomly + choose ratio from the list. Please note that `cutout_shape` + and `cutout_ratio` cannot be both given at the same time. + fill_in (tuple[float, float, float] | tuple[int, int, int]): The value + of pixel to fill in the dropped regions. Default: (0, 0, 0). + """ + + def __init__(self, + n_holes, + cutout_shape=None, + cutout_ratio=None, + fill_in=(0, 0, 0)): + + assert (cutout_shape is None) ^ (cutout_ratio is None), \ + 'Either cutout_shape or cutout_ratio should be specified.' + assert (isinstance(cutout_shape, (list, tuple)) + or isinstance(cutout_ratio, (list, tuple))) + if isinstance(n_holes, tuple): + assert len(n_holes) == 2 and 0 <= n_holes[0] < n_holes[1] + else: + n_holes = (n_holes, n_holes) + self.n_holes = n_holes + self.fill_in = fill_in + self.with_ratio = cutout_ratio is not None + self.candidates = cutout_ratio if self.with_ratio else cutout_shape + if not isinstance(self.candidates, list): + self.candidates = [self.candidates] + + def __call__(self, results): + """Call function to drop some regions of image.""" + h, w, c = results['img'].shape + n_holes = np.random.randint(self.n_holes[0], self.n_holes[1] + 1) + for _ in range(n_holes): + x1 = np.random.randint(0, w) + y1 = np.random.randint(0, h) + index = np.random.randint(0, len(self.candidates)) + if not self.with_ratio: + cutout_w, cutout_h = self.candidates[index] + else: + cutout_w = int(self.candidates[index][0] * w) + cutout_h = int(self.candidates[index][1] * h) + + x2 = np.clip(x1 + cutout_w, 0, w) + y2 = np.clip(y1 + cutout_h, 0, h) + results['img'][y1:y2, x1:x2, :] = self.fill_in + + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(n_holes={self.n_holes}, ' + repr_str += (f'cutout_ratio={self.candidates}, ' if self.with_ratio + else f'cutout_shape={self.candidates}, ') + repr_str += f'fill_in={self.fill_in})' + return repr_str diff --git a/detection/mmdet/datasets/samplers/__init__.py b/detection/mmdet/datasets/samplers/__init__.py new file mode 100644 index 0000000..2596aeb --- /dev/null +++ b/detection/mmdet/datasets/samplers/__init__.py @@ -0,0 +1,4 @@ +from .distributed_sampler import DistributedSampler +from .group_sampler import DistributedGroupSampler, GroupSampler + +__all__ = ['DistributedSampler', 'DistributedGroupSampler', 'GroupSampler'] diff --git a/detection/mmdet/datasets/samplers/distributed_sampler.py b/detection/mmdet/datasets/samplers/distributed_sampler.py new file mode 100644 index 0000000..cc61019 --- /dev/null +++ b/detection/mmdet/datasets/samplers/distributed_sampler.py @@ -0,0 +1,39 @@ +import math + +import torch +from torch.utils.data import DistributedSampler as _DistributedSampler + + +class DistributedSampler(_DistributedSampler): + + def __init__(self, + dataset, + num_replicas=None, + rank=None, + shuffle=True, + seed=0): + super().__init__( + dataset, num_replicas=num_replicas, rank=rank, shuffle=shuffle) + # for the compatibility from PyTorch 1.3+ + self.seed = seed if seed is not None else 0 + + def __iter__(self): + # deterministically shuffle based on epoch + if self.shuffle: + g = torch.Generator() + g.manual_seed(self.epoch + self.seed) + indices = torch.randperm(len(self.dataset), generator=g).tolist() + else: + indices = torch.arange(len(self.dataset)).tolist() + + # add extra samples to make it evenly divisible + # in case that indices is shorter than half of total_size + indices = (indices * + math.ceil(self.total_size / len(indices)))[:self.total_size] + assert len(indices) == self.total_size + + # subsample + indices = indices[self.rank:self.total_size:self.num_replicas] + assert len(indices) == self.num_samples + + return iter(indices) diff --git a/detection/mmdet/datasets/samplers/group_sampler.py b/detection/mmdet/datasets/samplers/group_sampler.py new file mode 100644 index 0000000..f88cf34 --- /dev/null +++ b/detection/mmdet/datasets/samplers/group_sampler.py @@ -0,0 +1,148 @@ +from __future__ import division +import math + +import numpy as np +import torch +from mmcv.runner import get_dist_info +from torch.utils.data import Sampler + + +class GroupSampler(Sampler): + + def __init__(self, dataset, samples_per_gpu=1): + assert hasattr(dataset, 'flag') + self.dataset = dataset + self.samples_per_gpu = samples_per_gpu + self.flag = dataset.flag.astype(np.int64) + self.group_sizes = np.bincount(self.flag) + self.num_samples = 0 + for i, size in enumerate(self.group_sizes): + self.num_samples += int(np.ceil( + size / self.samples_per_gpu)) * self.samples_per_gpu + + def __iter__(self): + indices = [] + for i, size in enumerate(self.group_sizes): + if size == 0: + continue + indice = np.where(self.flag == i)[0] + assert len(indice) == size + np.random.shuffle(indice) + num_extra = int(np.ceil(size / self.samples_per_gpu) + ) * self.samples_per_gpu - len(indice) + indice = np.concatenate( + [indice, np.random.choice(indice, num_extra)]) + indices.append(indice) + indices = np.concatenate(indices) + indices = [ + indices[i * self.samples_per_gpu:(i + 1) * self.samples_per_gpu] + for i in np.random.permutation( + range(len(indices) // self.samples_per_gpu)) + ] + indices = np.concatenate(indices) + indices = indices.astype(np.int64).tolist() + assert len(indices) == self.num_samples + return iter(indices) + + def __len__(self): + return self.num_samples + + +class DistributedGroupSampler(Sampler): + """Sampler that restricts data loading to a subset of the dataset. + + It is especially useful in conjunction with + :class:`torch.nn.parallel.DistributedDataParallel`. In such case, each + process can pass a DistributedSampler instance as a DataLoader sampler, + and load a subset of the original dataset that is exclusive to it. + + .. note:: + Dataset is assumed to be of constant size. + + Arguments: + dataset: Dataset used for sampling. + num_replicas (optional): Number of processes participating in + distributed training. + rank (optional): Rank of the current process within num_replicas. + seed (int, optional): random seed used to shuffle the sampler if + ``shuffle=True``. This number should be identical across all + processes in the distributed group. Default: 0. + """ + + def __init__(self, + dataset, + samples_per_gpu=1, + num_replicas=None, + rank=None, + seed=0): + _rank, _num_replicas = get_dist_info() + if num_replicas is None: + num_replicas = _num_replicas + if rank is None: + rank = _rank + self.dataset = dataset + self.samples_per_gpu = samples_per_gpu + self.num_replicas = num_replicas + self.rank = rank + self.epoch = 0 + self.seed = seed if seed is not None else 0 + + assert hasattr(self.dataset, 'flag') + self.flag = self.dataset.flag + self.group_sizes = np.bincount(self.flag) + + self.num_samples = 0 + for i, j in enumerate(self.group_sizes): + self.num_samples += int( + math.ceil(self.group_sizes[i] * 1.0 / self.samples_per_gpu / + self.num_replicas)) * self.samples_per_gpu + self.total_size = self.num_samples * self.num_replicas + + def __iter__(self): + # deterministically shuffle based on epoch + g = torch.Generator() + g.manual_seed(self.epoch + self.seed) + + indices = [] + for i, size in enumerate(self.group_sizes): + if size > 0: + indice = np.where(self.flag == i)[0] + assert len(indice) == size + # add .numpy() to avoid bug when selecting indice in parrots. + # TODO: check whether torch.randperm() can be replaced by + # numpy.random.permutation(). + indice = indice[list( + torch.randperm(int(size), generator=g).numpy())].tolist() + extra = int( + math.ceil( + size * 1.0 / self.samples_per_gpu / self.num_replicas) + ) * self.samples_per_gpu * self.num_replicas - len(indice) + # pad indice + tmp = indice.copy() + for _ in range(extra // size): + indice.extend(tmp) + indice.extend(tmp[:extra % size]) + indices.extend(indice) + + assert len(indices) == self.total_size + + indices = [ + indices[j] for i in list( + torch.randperm( + len(indices) // self.samples_per_gpu, generator=g)) + for j in range(i * self.samples_per_gpu, (i + 1) * + self.samples_per_gpu) + ] + + # subsample + offset = self.num_samples * self.rank + indices = indices[offset:offset + self.num_samples] + assert len(indices) == self.num_samples + + return iter(indices) + + def __len__(self): + return self.num_samples + + def set_epoch(self, epoch): + self.epoch = epoch diff --git a/detection/mmdet/datasets/utils.py b/detection/mmdet/datasets/utils.py new file mode 100644 index 0000000..157c9a2 --- /dev/null +++ b/detection/mmdet/datasets/utils.py @@ -0,0 +1,158 @@ +import copy +import warnings + +from mmcv.cnn import VGG +from mmcv.runner.hooks import HOOKS, Hook + +from mmdet.datasets.builder import PIPELINES +from mmdet.datasets.pipelines import LoadAnnotations, LoadImageFromFile +from mmdet.models.dense_heads import GARPNHead, RPNHead +from mmdet.models.roi_heads.mask_heads import FusedSemanticHead + + +def replace_ImageToTensor(pipelines): + """Replace the ImageToTensor transform in a data pipeline to + DefaultFormatBundle, which is normally useful in batch inference. + + Args: + pipelines (list[dict]): Data pipeline configs. + + Returns: + list: The new pipeline list with all ImageToTensor replaced by + DefaultFormatBundle. + + Examples: + >>> pipelines = [ + ... dict(type='LoadImageFromFile'), + ... dict( + ... type='MultiScaleFlipAug', + ... img_scale=(1333, 800), + ... flip=False, + ... transforms=[ + ... dict(type='Resize', keep_ratio=True), + ... dict(type='RandomFlip'), + ... dict(type='Normalize', mean=[0, 0, 0], std=[1, 1, 1]), + ... dict(type='Pad', size_divisor=32), + ... dict(type='ImageToTensor', keys=['img']), + ... dict(type='Collect', keys=['img']), + ... ]) + ... ] + >>> expected_pipelines = [ + ... dict(type='LoadImageFromFile'), + ... dict( + ... type='MultiScaleFlipAug', + ... img_scale=(1333, 800), + ... flip=False, + ... transforms=[ + ... dict(type='Resize', keep_ratio=True), + ... dict(type='RandomFlip'), + ... dict(type='Normalize', mean=[0, 0, 0], std=[1, 1, 1]), + ... dict(type='Pad', size_divisor=32), + ... dict(type='DefaultFormatBundle'), + ... dict(type='Collect', keys=['img']), + ... ]) + ... ] + >>> assert expected_pipelines == replace_ImageToTensor(pipelines) + """ + pipelines = copy.deepcopy(pipelines) + for i, pipeline in enumerate(pipelines): + if pipeline['type'] == 'MultiScaleFlipAug': + assert 'transforms' in pipeline + pipeline['transforms'] = replace_ImageToTensor( + pipeline['transforms']) + elif pipeline['type'] == 'ImageToTensor': + warnings.warn( + '"ImageToTensor" pipeline is replaced by ' + '"DefaultFormatBundle" for batch inference. It is ' + 'recommended to manually replace it in the test ' + 'data pipeline in your config file.', UserWarning) + pipelines[i] = {'type': 'DefaultFormatBundle'} + return pipelines + + +def get_loading_pipeline(pipeline): + """Only keep loading image and annotations related configuration. + + Args: + pipeline (list[dict]): Data pipeline configs. + + Returns: + list[dict]: The new pipeline list with only keep + loading image and annotations related configuration. + + Examples: + >>> pipelines = [ + ... dict(type='LoadImageFromFile'), + ... dict(type='LoadAnnotations', with_bbox=True), + ... dict(type='Resize', img_scale=(1333, 800), keep_ratio=True), + ... dict(type='RandomFlip', flip_ratio=0.5), + ... dict(type='Normalize', **img_norm_cfg), + ... dict(type='Pad', size_divisor=32), + ... dict(type='DefaultFormatBundle'), + ... dict(type='Collect', keys=['img', 'gt_bboxes', 'gt_labels']) + ... ] + >>> expected_pipelines = [ + ... dict(type='LoadImageFromFile'), + ... dict(type='LoadAnnotations', with_bbox=True) + ... ] + >>> assert expected_pipelines ==\ + ... get_loading_pipeline(pipelines) + """ + loading_pipeline_cfg = [] + for cfg in pipeline: + obj_cls = PIPELINES.get(cfg['type']) + # TODO:use more elegant way to distinguish loading modules + if obj_cls is not None and obj_cls in (LoadImageFromFile, + LoadAnnotations): + loading_pipeline_cfg.append(cfg) + assert len(loading_pipeline_cfg) == 2, \ + 'The data pipeline in your config file must include ' \ + 'loading image and annotations related pipeline.' + return loading_pipeline_cfg + + +@HOOKS.register_module() +class NumClassCheckHook(Hook): + + def _check_head(self, runner): + """Check whether the `num_classes` in head matches the length of + `CLASSSES` in `dataset`. + + Args: + runner (obj:`EpochBasedRunner`): Epoch based Runner. + """ + model = runner.model + dataset = runner.data_loader.dataset + if dataset.CLASSES is None: + runner.logger.warning( + f'Please set `CLASSES` ' + f'in the {dataset.__class__.__name__} and' + f'check if it is consistent with the `num_classes` ' + f'of head') + else: + for name, module in model.named_modules(): + if hasattr(module, 'num_classes') and not isinstance( + module, (RPNHead, VGG, FusedSemanticHead, GARPNHead)): + assert module.num_classes == len(dataset.CLASSES), \ + (f'The `num_classes` ({module.num_classes}) in ' + f'{module.__class__.__name__} of ' + f'{model.__class__.__name__} does not matches ' + f'the length of `CLASSES` ' + f'{len(dataset.CLASSES)}) in ' + f'{dataset.__class__.__name__}') + + def before_train_epoch(self, runner): + """Check whether the training dataset is compatible with head. + + Args: + runner (obj:`EpochBasedRunner`): Epoch based Runner. + """ + self._check_head(runner) + + def before_val_epoch(self, runner): + """Check whether the dataset in val epoch is compatible with head. + + Args: + runner (obj:`EpochBasedRunner`): Epoch based Runner. + """ + self._check_head(runner) diff --git a/detection/mmdet/datasets/voc.py b/detection/mmdet/datasets/voc.py new file mode 100644 index 0000000..abd4cb8 --- /dev/null +++ b/detection/mmdet/datasets/voc.py @@ -0,0 +1,93 @@ +from collections import OrderedDict + +from mmcv.utils import print_log + +from mmdet.core import eval_map, eval_recalls +from .builder import DATASETS +from .xml_style import XMLDataset + + +@DATASETS.register_module() +class VOCDataset(XMLDataset): + + CLASSES = ('aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', + 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', + 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', + 'tvmonitor') + + def __init__(self, **kwargs): + super(VOCDataset, self).__init__(**kwargs) + if 'VOC2007' in self.img_prefix: + self.year = 2007 + elif 'VOC2012' in self.img_prefix: + self.year = 2012 + else: + raise ValueError('Cannot infer dataset year from img_prefix') + + def evaluate(self, + results, + metric='mAP', + logger=None, + proposal_nums=(100, 300, 1000), + iou_thr=0.5, + scale_ranges=None): + """Evaluate in VOC protocol. + + Args: + results (list[list | tuple]): Testing results of the dataset. + metric (str | list[str]): Metrics to be evaluated. Options are + 'mAP', 'recall'. + logger (logging.Logger | str, optional): Logger used for printing + related information during evaluation. Default: None. + proposal_nums (Sequence[int]): Proposal number used for evaluating + recalls, such as recall@100, recall@1000. + Default: (100, 300, 1000). + iou_thr (float | list[float]): IoU threshold. Default: 0.5. + scale_ranges (list[tuple], optional): Scale ranges for evaluating + mAP. If not specified, all bounding boxes would be included in + evaluation. Default: None. + + Returns: + dict[str, float]: AP/recall metrics. + """ + + if not isinstance(metric, str): + assert len(metric) == 1 + metric = metric[0] + allowed_metrics = ['mAP', 'recall'] + if metric not in allowed_metrics: + raise KeyError(f'metric {metric} is not supported') + annotations = [self.get_ann_info(i) for i in range(len(self))] + eval_results = OrderedDict() + iou_thrs = [iou_thr] if isinstance(iou_thr, float) else iou_thr + if metric == 'mAP': + assert isinstance(iou_thrs, list) + if self.year == 2007: + ds_name = 'voc07' + else: + ds_name = self.CLASSES + mean_aps = [] + for iou_thr in iou_thrs: + print_log(f'\n{"-" * 15}iou_thr: {iou_thr}{"-" * 15}') + mean_ap, _ = eval_map( + results, + annotations, + scale_ranges=None, + iou_thr=iou_thr, + dataset=ds_name, + logger=logger) + mean_aps.append(mean_ap) + eval_results[f'AP{int(iou_thr * 100):02d}'] = round(mean_ap, 3) + eval_results['mAP'] = sum(mean_aps) / len(mean_aps) + elif metric == 'recall': + gt_bboxes = [ann['bboxes'] for ann in annotations] + recalls = eval_recalls( + gt_bboxes, results, proposal_nums, iou_thr, logger=logger) + for i, num in enumerate(proposal_nums): + for j, iou in enumerate(iou_thr): + eval_results[f'recall@{num}@{iou}'] = recalls[i, j] + if recalls.shape[1] > 1: + ar = recalls.mean(axis=1) + for i, num in enumerate(proposal_nums): + eval_results[f'AR@{num}'] = ar[i] + return eval_results diff --git a/detection/mmdet/datasets/wider_face.py b/detection/mmdet/datasets/wider_face.py new file mode 100644 index 0000000..3a13907 --- /dev/null +++ b/detection/mmdet/datasets/wider_face.py @@ -0,0 +1,51 @@ +import os.path as osp +import xml.etree.ElementTree as ET + +import mmcv + +from .builder import DATASETS +from .xml_style import XMLDataset + + +@DATASETS.register_module() +class WIDERFaceDataset(XMLDataset): + """Reader for the WIDER Face dataset in PASCAL VOC format. + + Conversion scripts can be found in + https://github.com/sovrasov/wider-face-pascal-voc-annotations + """ + CLASSES = ('face', ) + + def __init__(self, **kwargs): + super(WIDERFaceDataset, self).__init__(**kwargs) + + def load_annotations(self, ann_file): + """Load annotation from WIDERFace XML style annotation file. + + Args: + ann_file (str): Path of XML file. + + Returns: + list[dict]: Annotation info from XML file. + """ + + data_infos = [] + img_ids = mmcv.list_from_file(ann_file) + for img_id in img_ids: + filename = f'{img_id}.jpg' + xml_path = osp.join(self.img_prefix, 'Annotations', + f'{img_id}.xml') + tree = ET.parse(xml_path) + root = tree.getroot() + size = root.find('size') + width = int(size.find('width').text) + height = int(size.find('height').text) + folder = root.find('folder').text + data_infos.append( + dict( + id=img_id, + filename=osp.join(folder, filename), + width=width, + height=height)) + + return data_infos diff --git a/detection/mmdet/datasets/xml_style.py b/detection/mmdet/datasets/xml_style.py new file mode 100644 index 0000000..7106948 --- /dev/null +++ b/detection/mmdet/datasets/xml_style.py @@ -0,0 +1,170 @@ +import os.path as osp +import xml.etree.ElementTree as ET + +import mmcv +import numpy as np +from PIL import Image + +from .builder import DATASETS +from .custom import CustomDataset + + +@DATASETS.register_module() +class XMLDataset(CustomDataset): + """XML dataset for detection. + + Args: + min_size (int | float, optional): The minimum size of bounding + boxes in the images. If the size of a bounding box is less than + ``min_size``, it would be add to ignored field. + """ + + def __init__(self, min_size=None, **kwargs): + assert self.CLASSES or kwargs.get( + 'classes', None), 'CLASSES in `XMLDataset` can not be None.' + super(XMLDataset, self).__init__(**kwargs) + self.cat2label = {cat: i for i, cat in enumerate(self.CLASSES)} + self.min_size = min_size + + def load_annotations(self, ann_file): + """Load annotation from XML style ann_file. + + Args: + ann_file (str): Path of XML file. + + Returns: + list[dict]: Annotation info from XML file. + """ + + data_infos = [] + img_ids = mmcv.list_from_file(ann_file) + for img_id in img_ids: + filename = f'JPEGImages/{img_id}.jpg' + xml_path = osp.join(self.img_prefix, 'Annotations', + f'{img_id}.xml') + tree = ET.parse(xml_path) + root = tree.getroot() + size = root.find('size') + if size is not None: + width = int(size.find('width').text) + height = int(size.find('height').text) + else: + img_path = osp.join(self.img_prefix, 'JPEGImages', + '{}.jpg'.format(img_id)) + img = Image.open(img_path) + width, height = img.size + data_infos.append( + dict(id=img_id, filename=filename, width=width, height=height)) + + return data_infos + + def _filter_imgs(self, min_size=32): + """Filter images too small or without annotation.""" + valid_inds = [] + for i, img_info in enumerate(self.data_infos): + if min(img_info['width'], img_info['height']) < min_size: + continue + if self.filter_empty_gt: + img_id = img_info['id'] + xml_path = osp.join(self.img_prefix, 'Annotations', + f'{img_id}.xml') + tree = ET.parse(xml_path) + root = tree.getroot() + for obj in root.findall('object'): + name = obj.find('name').text + if name in self.CLASSES: + valid_inds.append(i) + break + else: + valid_inds.append(i) + return valid_inds + + def get_ann_info(self, idx): + """Get annotation from XML file by index. + + Args: + idx (int): Index of data. + + Returns: + dict: Annotation info of specified index. + """ + + img_id = self.data_infos[idx]['id'] + xml_path = osp.join(self.img_prefix, 'Annotations', f'{img_id}.xml') + tree = ET.parse(xml_path) + root = tree.getroot() + bboxes = [] + labels = [] + bboxes_ignore = [] + labels_ignore = [] + for obj in root.findall('object'): + name = obj.find('name').text + if name not in self.CLASSES: + continue + label = self.cat2label[name] + difficult = obj.find('difficult') + difficult = 0 if difficult is None else int(difficult.text) + bnd_box = obj.find('bndbox') + # TODO: check whether it is necessary to use int + # Coordinates may be float type + bbox = [ + int(float(bnd_box.find('xmin').text)), + int(float(bnd_box.find('ymin').text)), + int(float(bnd_box.find('xmax').text)), + int(float(bnd_box.find('ymax').text)) + ] + ignore = False + if self.min_size: + assert not self.test_mode + w = bbox[2] - bbox[0] + h = bbox[3] - bbox[1] + if w < self.min_size or h < self.min_size: + ignore = True + if difficult or ignore: + bboxes_ignore.append(bbox) + labels_ignore.append(label) + else: + bboxes.append(bbox) + labels.append(label) + if not bboxes: + bboxes = np.zeros((0, 4)) + labels = np.zeros((0, )) + else: + bboxes = np.array(bboxes, ndmin=2) - 1 + labels = np.array(labels) + if not bboxes_ignore: + bboxes_ignore = np.zeros((0, 4)) + labels_ignore = np.zeros((0, )) + else: + bboxes_ignore = np.array(bboxes_ignore, ndmin=2) - 1 + labels_ignore = np.array(labels_ignore) + ann = dict( + bboxes=bboxes.astype(np.float32), + labels=labels.astype(np.int64), + bboxes_ignore=bboxes_ignore.astype(np.float32), + labels_ignore=labels_ignore.astype(np.int64)) + return ann + + def get_cat_ids(self, idx): + """Get category ids in XML file by index. + + Args: + idx (int): Index of data. + + Returns: + list[int]: All categories in the image of specified index. + """ + + cat_ids = [] + img_id = self.data_infos[idx]['id'] + xml_path = osp.join(self.img_prefix, 'Annotations', f'{img_id}.xml') + tree = ET.parse(xml_path) + root = tree.getroot() + for obj in root.findall('object'): + name = obj.find('name').text + if name not in self.CLASSES: + continue + label = self.cat2label[name] + cat_ids.append(label) + + return cat_ids diff --git a/detection/mmdet/models/__init__.py b/detection/mmdet/models/__init__.py new file mode 100644 index 0000000..44ac998 --- /dev/null +++ b/detection/mmdet/models/__init__.py @@ -0,0 +1,16 @@ +from .backbones import * # noqa: F401,F403 +from .builder import (BACKBONES, DETECTORS, HEADS, LOSSES, NECKS, + ROI_EXTRACTORS, SHARED_HEADS, build_backbone, + build_detector, build_head, build_loss, build_neck, + build_roi_extractor, build_shared_head) +from .dense_heads import * # noqa: F401,F403 +from .detectors import * # noqa: F401,F403 +from .losses import * # noqa: F401,F403 +from .necks import * # noqa: F401,F403 +from .roi_heads import * # noqa: F401,F403 + +__all__ = [ + 'BACKBONES', 'NECKS', 'ROI_EXTRACTORS', 'SHARED_HEADS', 'HEADS', 'LOSSES', + 'DETECTORS', 'build_backbone', 'build_neck', 'build_roi_extractor', + 'build_shared_head', 'build_head', 'build_loss', 'build_detector' +] diff --git a/detection/mmdet/models/backbones/__init__.py b/detection/mmdet/models/backbones/__init__.py new file mode 100644 index 0000000..91f415b --- /dev/null +++ b/detection/mmdet/models/backbones/__init__.py @@ -0,0 +1,19 @@ +from .darknet import Darknet +from .detectors_resnet import DetectoRS_ResNet +from .detectors_resnext import DetectoRS_ResNeXt +from .hourglass import HourglassNet +from .hrnet import HRNet +from .regnet import RegNet +from .res2net import Res2Net +from .resnest import ResNeSt +from .resnet import ResNet, ResNetV1d +from .resnext import ResNeXt +from .ssd_vgg import SSDVGG +from .trident_resnet import TridentResNet +from .litv2 import LITv2 + +__all__ = [ + 'RegNet', 'ResNet', 'ResNetV1d', 'ResNeXt', 'SSDVGG', 'HRNet', 'Res2Net', + 'HourglassNet', 'DetectoRS_ResNet', 'DetectoRS_ResNeXt', 'Darknet', + 'ResNeSt', 'TridentResNet', 'LITv2' +] diff --git a/detection/mmdet/models/backbones/darknet.py b/detection/mmdet/models/backbones/darknet.py new file mode 100644 index 0000000..517fe26 --- /dev/null +++ b/detection/mmdet/models/backbones/darknet.py @@ -0,0 +1,199 @@ +# Copyright (c) 2019 Western Digital Corporation or its affiliates. + +import logging + +import torch.nn as nn +from mmcv.cnn import ConvModule, constant_init, kaiming_init +from mmcv.runner import load_checkpoint +from torch.nn.modules.batchnorm import _BatchNorm + +from ..builder import BACKBONES + + +class ResBlock(nn.Module): + """The basic residual block used in Darknet. Each ResBlock consists of two + ConvModules and the input is added to the final output. Each ConvModule is + composed of Conv, BN, and LeakyReLU. In YoloV3 paper, the first convLayer + has half of the number of the filters as much as the second convLayer. The + first convLayer has filter size of 1x1 and the second one has the filter + size of 3x3. + + Args: + in_channels (int): The input channels. Must be even. + conv_cfg (dict): Config dict for convolution layer. Default: None. + norm_cfg (dict): Dictionary to construct and config norm layer. + Default: dict(type='BN', requires_grad=True) + act_cfg (dict): Config dict for activation layer. + Default: dict(type='LeakyReLU', negative_slope=0.1). + """ + + def __init__(self, + in_channels, + conv_cfg=None, + norm_cfg=dict(type='BN', requires_grad=True), + act_cfg=dict(type='LeakyReLU', negative_slope=0.1)): + super(ResBlock, self).__init__() + assert in_channels % 2 == 0 # ensure the in_channels is even + half_in_channels = in_channels // 2 + + # shortcut + cfg = dict(conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg) + + self.conv1 = ConvModule(in_channels, half_in_channels, 1, **cfg) + self.conv2 = ConvModule( + half_in_channels, in_channels, 3, padding=1, **cfg) + + def forward(self, x): + residual = x + out = self.conv1(x) + out = self.conv2(out) + out = out + residual + + return out + + +@BACKBONES.register_module() +class Darknet(nn.Module): + """Darknet backbone. + + Args: + depth (int): Depth of Darknet. Currently only support 53. + out_indices (Sequence[int]): Output from which stages. + frozen_stages (int): Stages to be frozen (stop grad and set eval mode). + -1 means not freezing any parameters. Default: -1. + conv_cfg (dict): Config dict for convolution layer. Default: None. + norm_cfg (dict): Dictionary to construct and config norm layer. + Default: dict(type='BN', requires_grad=True) + act_cfg (dict): Config dict for activation layer. + Default: dict(type='LeakyReLU', negative_slope=0.1). + norm_eval (bool): Whether to set norm layers to eval mode, namely, + freeze running stats (mean and var). Note: Effect on Batch Norm + and its variants only. + + Example: + >>> from mmdet.models import Darknet + >>> import torch + >>> self = Darknet(depth=53) + >>> self.eval() + >>> inputs = torch.rand(1, 3, 416, 416) + >>> level_outputs = self.forward(inputs) + >>> for level_out in level_outputs: + ... print(tuple(level_out.shape)) + ... + (1, 256, 52, 52) + (1, 512, 26, 26) + (1, 1024, 13, 13) + """ + + # Dict(depth: (layers, channels)) + arch_settings = { + 53: ((1, 2, 8, 8, 4), ((32, 64), (64, 128), (128, 256), (256, 512), + (512, 1024))) + } + + def __init__(self, + depth=53, + out_indices=(3, 4, 5), + frozen_stages=-1, + conv_cfg=None, + norm_cfg=dict(type='BN', requires_grad=True), + act_cfg=dict(type='LeakyReLU', negative_slope=0.1), + norm_eval=True): + super(Darknet, self).__init__() + if depth not in self.arch_settings: + raise KeyError(f'invalid depth {depth} for darknet') + self.depth = depth + self.out_indices = out_indices + self.frozen_stages = frozen_stages + self.layers, self.channels = self.arch_settings[depth] + + cfg = dict(conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg) + + self.conv1 = ConvModule(3, 32, 3, padding=1, **cfg) + + self.cr_blocks = ['conv1'] + for i, n_layers in enumerate(self.layers): + layer_name = f'conv_res_block{i + 1}' + in_c, out_c = self.channels[i] + self.add_module( + layer_name, + self.make_conv_res_block(in_c, out_c, n_layers, **cfg)) + self.cr_blocks.append(layer_name) + + self.norm_eval = norm_eval + + def forward(self, x): + outs = [] + for i, layer_name in enumerate(self.cr_blocks): + cr_block = getattr(self, layer_name) + x = cr_block(x) + if i in self.out_indices: + outs.append(x) + + return tuple(outs) + + def init_weights(self, pretrained=None): + if isinstance(pretrained, str): + logger = logging.getLogger() + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + for m in self.modules(): + if isinstance(m, nn.Conv2d): + kaiming_init(m) + elif isinstance(m, (_BatchNorm, nn.GroupNorm)): + constant_init(m, 1) + + else: + raise TypeError('pretrained must be a str or None') + + def _freeze_stages(self): + if self.frozen_stages >= 0: + for i in range(self.frozen_stages): + m = getattr(self, self.cr_blocks[i]) + m.eval() + for param in m.parameters(): + param.requires_grad = False + + def train(self, mode=True): + super(Darknet, self).train(mode) + self._freeze_stages() + if mode and self.norm_eval: + for m in self.modules(): + if isinstance(m, _BatchNorm): + m.eval() + + @staticmethod + def make_conv_res_block(in_channels, + out_channels, + res_repeat, + conv_cfg=None, + norm_cfg=dict(type='BN', requires_grad=True), + act_cfg=dict(type='LeakyReLU', + negative_slope=0.1)): + """In Darknet backbone, ConvLayer is usually followed by ResBlock. This + function will make that. The Conv layers always have 3x3 filters with + stride=2. The number of the filters in Conv layer is the same as the + out channels of the ResBlock. + + Args: + in_channels (int): The number of input channels. + out_channels (int): The number of output channels. + res_repeat (int): The number of ResBlocks. + conv_cfg (dict): Config dict for convolution layer. Default: None. + norm_cfg (dict): Dictionary to construct and config norm layer. + Default: dict(type='BN', requires_grad=True) + act_cfg (dict): Config dict for activation layer. + Default: dict(type='LeakyReLU', negative_slope=0.1). + """ + + cfg = dict(conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg) + + model = nn.Sequential() + model.add_module( + 'conv', + ConvModule( + in_channels, out_channels, 3, stride=2, padding=1, **cfg)) + for idx in range(res_repeat): + model.add_module('res{}'.format(idx), + ResBlock(out_channels, **cfg)) + return model diff --git a/detection/mmdet/models/backbones/detectors_resnet.py b/detection/mmdet/models/backbones/detectors_resnet.py new file mode 100644 index 0000000..519db46 --- /dev/null +++ b/detection/mmdet/models/backbones/detectors_resnet.py @@ -0,0 +1,305 @@ +import torch.nn as nn +import torch.utils.checkpoint as cp +from mmcv.cnn import build_conv_layer, build_norm_layer, constant_init + +from ..builder import BACKBONES +from .resnet import Bottleneck as _Bottleneck +from .resnet import ResNet + + +class Bottleneck(_Bottleneck): + r"""Bottleneck for the ResNet backbone in `DetectoRS + `_. + + This bottleneck allows the users to specify whether to use + SAC (Switchable Atrous Convolution) and RFP (Recursive Feature Pyramid). + + Args: + inplanes (int): The number of input channels. + planes (int): The number of output channels before expansion. + rfp_inplanes (int, optional): The number of channels from RFP. + Default: None. If specified, an additional conv layer will be + added for ``rfp_feat``. Otherwise, the structure is the same as + base class. + sac (dict, optional): Dictionary to construct SAC. Default: None. + """ + expansion = 4 + + def __init__(self, + inplanes, + planes, + rfp_inplanes=None, + sac=None, + **kwargs): + super(Bottleneck, self).__init__(inplanes, planes, **kwargs) + + assert sac is None or isinstance(sac, dict) + self.sac = sac + self.with_sac = sac is not None + if self.with_sac: + self.conv2 = build_conv_layer( + self.sac, + planes, + planes, + kernel_size=3, + stride=self.conv2_stride, + padding=self.dilation, + dilation=self.dilation, + bias=False) + + self.rfp_inplanes = rfp_inplanes + if self.rfp_inplanes: + self.rfp_conv = build_conv_layer( + None, + self.rfp_inplanes, + planes * self.expansion, + 1, + stride=1, + bias=True) + self.init_weights() + + def init_weights(self): + """Initialize the weights.""" + if self.rfp_inplanes: + constant_init(self.rfp_conv, 0) + + def rfp_forward(self, x, rfp_feat): + """The forward function that also takes the RFP features as input.""" + + def _inner_forward(x): + identity = x + + out = self.conv1(x) + out = self.norm1(out) + out = self.relu(out) + + if self.with_plugins: + out = self.forward_plugin(out, self.after_conv1_plugin_names) + + out = self.conv2(out) + out = self.norm2(out) + out = self.relu(out) + + if self.with_plugins: + out = self.forward_plugin(out, self.after_conv2_plugin_names) + + out = self.conv3(out) + out = self.norm3(out) + + if self.with_plugins: + out = self.forward_plugin(out, self.after_conv3_plugin_names) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + + return out + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x) + else: + out = _inner_forward(x) + + if self.rfp_inplanes: + rfp_feat = self.rfp_conv(rfp_feat) + out = out + rfp_feat + + out = self.relu(out) + + return out + + +class ResLayer(nn.Sequential): + """ResLayer to build ResNet style backbone for RPF in detectoRS. + + The difference between this module and base class is that we pass + ``rfp_inplanes`` to the first block. + + Args: + block (nn.Module): block used to build ResLayer. + inplanes (int): inplanes of block. + planes (int): planes of block. + num_blocks (int): number of blocks. + stride (int): stride of the first block. Default: 1 + avg_down (bool): Use AvgPool instead of stride conv when + downsampling in the bottleneck. Default: False + conv_cfg (dict): dictionary to construct and config conv layer. + Default: None + norm_cfg (dict): dictionary to construct and config norm layer. + Default: dict(type='BN') + downsample_first (bool): Downsample at the first block or last block. + False for Hourglass, True for ResNet. Default: True + rfp_inplanes (int, optional): The number of channels from RFP. + Default: None. If specified, an additional conv layer will be + added for ``rfp_feat``. Otherwise, the structure is the same as + base class. + """ + + def __init__(self, + block, + inplanes, + planes, + num_blocks, + stride=1, + avg_down=False, + conv_cfg=None, + norm_cfg=dict(type='BN'), + downsample_first=True, + rfp_inplanes=None, + **kwargs): + self.block = block + assert downsample_first, f'downsample_first={downsample_first} is ' \ + 'not supported in DetectoRS' + + downsample = None + if stride != 1 or inplanes != planes * block.expansion: + downsample = [] + conv_stride = stride + if avg_down and stride != 1: + conv_stride = 1 + downsample.append( + nn.AvgPool2d( + kernel_size=stride, + stride=stride, + ceil_mode=True, + count_include_pad=False)) + downsample.extend([ + build_conv_layer( + conv_cfg, + inplanes, + planes * block.expansion, + kernel_size=1, + stride=conv_stride, + bias=False), + build_norm_layer(norm_cfg, planes * block.expansion)[1] + ]) + downsample = nn.Sequential(*downsample) + + layers = [] + layers.append( + block( + inplanes=inplanes, + planes=planes, + stride=stride, + downsample=downsample, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + rfp_inplanes=rfp_inplanes, + **kwargs)) + inplanes = planes * block.expansion + for _ in range(1, num_blocks): + layers.append( + block( + inplanes=inplanes, + planes=planes, + stride=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + **kwargs)) + + super(ResLayer, self).__init__(*layers) + + +@BACKBONES.register_module() +class DetectoRS_ResNet(ResNet): + """ResNet backbone for DetectoRS. + + Args: + sac (dict, optional): Dictionary to construct SAC (Switchable Atrous + Convolution). Default: None. + stage_with_sac (list): Which stage to use sac. Default: (False, False, + False, False). + rfp_inplanes (int, optional): The number of channels from RFP. + Default: None. If specified, an additional conv layer will be + added for ``rfp_feat``. Otherwise, the structure is the same as + base class. + output_img (bool): If ``True``, the input image will be inserted into + the starting position of output. Default: False. + pretrained (str, optional): The pretrained model to load. + """ + + arch_settings = { + 50: (Bottleneck, (3, 4, 6, 3)), + 101: (Bottleneck, (3, 4, 23, 3)), + 152: (Bottleneck, (3, 8, 36, 3)) + } + + def __init__(self, + sac=None, + stage_with_sac=(False, False, False, False), + rfp_inplanes=None, + output_img=False, + pretrained=None, + **kwargs): + self.sac = sac + self.stage_with_sac = stage_with_sac + self.rfp_inplanes = rfp_inplanes + self.output_img = output_img + self.pretrained = pretrained + super(DetectoRS_ResNet, self).__init__(**kwargs) + + self.inplanes = self.stem_channels + self.res_layers = [] + for i, num_blocks in enumerate(self.stage_blocks): + stride = self.strides[i] + dilation = self.dilations[i] + dcn = self.dcn if self.stage_with_dcn[i] else None + sac = self.sac if self.stage_with_sac[i] else None + if self.plugins is not None: + stage_plugins = self.make_stage_plugins(self.plugins, i) + else: + stage_plugins = None + planes = self.base_channels * 2**i + res_layer = self.make_res_layer( + block=self.block, + inplanes=self.inplanes, + planes=planes, + num_blocks=num_blocks, + stride=stride, + dilation=dilation, + style=self.style, + avg_down=self.avg_down, + with_cp=self.with_cp, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + dcn=dcn, + sac=sac, + rfp_inplanes=rfp_inplanes if i > 0 else None, + plugins=stage_plugins) + self.inplanes = planes * self.block.expansion + layer_name = f'layer{i + 1}' + self.add_module(layer_name, res_layer) + self.res_layers.append(layer_name) + + self._freeze_stages() + + def make_res_layer(self, **kwargs): + """Pack all blocks in a stage into a ``ResLayer`` for DetectoRS.""" + return ResLayer(**kwargs) + + def forward(self, x): + """Forward function.""" + outs = list(super(DetectoRS_ResNet, self).forward(x)) + if self.output_img: + outs.insert(0, x) + return tuple(outs) + + def rfp_forward(self, x, rfp_feats): + """Forward function for RFP.""" + if self.deep_stem: + x = self.stem(x) + else: + x = self.conv1(x) + x = self.norm1(x) + x = self.relu(x) + x = self.maxpool(x) + outs = [] + for i, layer_name in enumerate(self.res_layers): + res_layer = getattr(self, layer_name) + rfp_feat = rfp_feats[i] if i > 0 else None + for layer in res_layer: + x = layer.rfp_forward(x, rfp_feat) + if i in self.out_indices: + outs.append(x) + return tuple(outs) diff --git a/detection/mmdet/models/backbones/detectors_resnext.py b/detection/mmdet/models/backbones/detectors_resnext.py new file mode 100644 index 0000000..57d032f --- /dev/null +++ b/detection/mmdet/models/backbones/detectors_resnext.py @@ -0,0 +1,122 @@ +import math + +from mmcv.cnn import build_conv_layer, build_norm_layer + +from ..builder import BACKBONES +from .detectors_resnet import Bottleneck as _Bottleneck +from .detectors_resnet import DetectoRS_ResNet + + +class Bottleneck(_Bottleneck): + expansion = 4 + + def __init__(self, + inplanes, + planes, + groups=1, + base_width=4, + base_channels=64, + **kwargs): + """Bottleneck block for ResNeXt. + + If style is "pytorch", the stride-two layer is the 3x3 conv layer, if + it is "caffe", the stride-two layer is the first 1x1 conv layer. + """ + super(Bottleneck, self).__init__(inplanes, planes, **kwargs) + + if groups == 1: + width = self.planes + else: + width = math.floor(self.planes * + (base_width / base_channels)) * groups + + self.norm1_name, norm1 = build_norm_layer( + self.norm_cfg, width, postfix=1) + self.norm2_name, norm2 = build_norm_layer( + self.norm_cfg, width, postfix=2) + self.norm3_name, norm3 = build_norm_layer( + self.norm_cfg, self.planes * self.expansion, postfix=3) + + self.conv1 = build_conv_layer( + self.conv_cfg, + self.inplanes, + width, + kernel_size=1, + stride=self.conv1_stride, + bias=False) + self.add_module(self.norm1_name, norm1) + fallback_on_stride = False + self.with_modulated_dcn = False + if self.with_dcn: + fallback_on_stride = self.dcn.pop('fallback_on_stride', False) + if self.with_sac: + self.conv2 = build_conv_layer( + self.sac, + width, + width, + kernel_size=3, + stride=self.conv2_stride, + padding=self.dilation, + dilation=self.dilation, + groups=groups, + bias=False) + elif not self.with_dcn or fallback_on_stride: + self.conv2 = build_conv_layer( + self.conv_cfg, + width, + width, + kernel_size=3, + stride=self.conv2_stride, + padding=self.dilation, + dilation=self.dilation, + groups=groups, + bias=False) + else: + assert self.conv_cfg is None, 'conv_cfg must be None for DCN' + self.conv2 = build_conv_layer( + self.dcn, + width, + width, + kernel_size=3, + stride=self.conv2_stride, + padding=self.dilation, + dilation=self.dilation, + groups=groups, + bias=False) + + self.add_module(self.norm2_name, norm2) + self.conv3 = build_conv_layer( + self.conv_cfg, + width, + self.planes * self.expansion, + kernel_size=1, + bias=False) + self.add_module(self.norm3_name, norm3) + + +@BACKBONES.register_module() +class DetectoRS_ResNeXt(DetectoRS_ResNet): + """ResNeXt backbone for DetectoRS. + + Args: + groups (int): The number of groups in ResNeXt. + base_width (int): The base width of ResNeXt. + """ + + arch_settings = { + 50: (Bottleneck, (3, 4, 6, 3)), + 101: (Bottleneck, (3, 4, 23, 3)), + 152: (Bottleneck, (3, 8, 36, 3)) + } + + def __init__(self, groups=1, base_width=4, **kwargs): + self.groups = groups + self.base_width = base_width + super(DetectoRS_ResNeXt, self).__init__(**kwargs) + + def make_res_layer(self, **kwargs): + return super().make_res_layer( + groups=self.groups, + base_width=self.base_width, + base_channels=self.base_channels, + **kwargs) diff --git a/detection/mmdet/models/backbones/hourglass.py b/detection/mmdet/models/backbones/hourglass.py new file mode 100644 index 0000000..3422ace --- /dev/null +++ b/detection/mmdet/models/backbones/hourglass.py @@ -0,0 +1,198 @@ +import torch.nn as nn +from mmcv.cnn import ConvModule + +from ..builder import BACKBONES +from ..utils import ResLayer +from .resnet import BasicBlock + + +class HourglassModule(nn.Module): + """Hourglass Module for HourglassNet backbone. + + Generate module recursively and use BasicBlock as the base unit. + + Args: + depth (int): Depth of current HourglassModule. + stage_channels (list[int]): Feature channels of sub-modules in current + and follow-up HourglassModule. + stage_blocks (list[int]): Number of sub-modules stacked in current and + follow-up HourglassModule. + norm_cfg (dict): Dictionary to construct and config norm layer. + """ + + def __init__(self, + depth, + stage_channels, + stage_blocks, + norm_cfg=dict(type='BN', requires_grad=True)): + super(HourglassModule, self).__init__() + + self.depth = depth + + cur_block = stage_blocks[0] + next_block = stage_blocks[1] + + cur_channel = stage_channels[0] + next_channel = stage_channels[1] + + self.up1 = ResLayer( + BasicBlock, cur_channel, cur_channel, cur_block, norm_cfg=norm_cfg) + + self.low1 = ResLayer( + BasicBlock, + cur_channel, + next_channel, + cur_block, + stride=2, + norm_cfg=norm_cfg) + + if self.depth > 1: + self.low2 = HourglassModule(depth - 1, stage_channels[1:], + stage_blocks[1:]) + else: + self.low2 = ResLayer( + BasicBlock, + next_channel, + next_channel, + next_block, + norm_cfg=norm_cfg) + + self.low3 = ResLayer( + BasicBlock, + next_channel, + cur_channel, + cur_block, + norm_cfg=norm_cfg, + downsample_first=False) + + self.up2 = nn.Upsample(scale_factor=2) + + def forward(self, x): + """Forward function.""" + up1 = self.up1(x) + low1 = self.low1(x) + low2 = self.low2(low1) + low3 = self.low3(low2) + up2 = self.up2(low3) + return up1 + up2 + + +@BACKBONES.register_module() +class HourglassNet(nn.Module): + """HourglassNet backbone. + + Stacked Hourglass Networks for Human Pose Estimation. + More details can be found in the `paper + `_ . + + Args: + downsample_times (int): Downsample times in a HourglassModule. + num_stacks (int): Number of HourglassModule modules stacked, + 1 for Hourglass-52, 2 for Hourglass-104. + stage_channels (list[int]): Feature channel of each sub-module in a + HourglassModule. + stage_blocks (list[int]): Number of sub-modules stacked in a + HourglassModule. + feat_channel (int): Feature channel of conv after a HourglassModule. + norm_cfg (dict): Dictionary to construct and config norm layer. + + Example: + >>> from mmdet.models import HourglassNet + >>> import torch + >>> self = HourglassNet() + >>> self.eval() + >>> inputs = torch.rand(1, 3, 511, 511) + >>> level_outputs = self.forward(inputs) + >>> for level_output in level_outputs: + ... print(tuple(level_output.shape)) + (1, 256, 128, 128) + (1, 256, 128, 128) + """ + + def __init__(self, + downsample_times=5, + num_stacks=2, + stage_channels=(256, 256, 384, 384, 384, 512), + stage_blocks=(2, 2, 2, 2, 2, 4), + feat_channel=256, + norm_cfg=dict(type='BN', requires_grad=True)): + super(HourglassNet, self).__init__() + + self.num_stacks = num_stacks + assert self.num_stacks >= 1 + assert len(stage_channels) == len(stage_blocks) + assert len(stage_channels) > downsample_times + + cur_channel = stage_channels[0] + + self.stem = nn.Sequential( + ConvModule(3, 128, 7, padding=3, stride=2, norm_cfg=norm_cfg), + ResLayer(BasicBlock, 128, 256, 1, stride=2, norm_cfg=norm_cfg)) + + self.hourglass_modules = nn.ModuleList([ + HourglassModule(downsample_times, stage_channels, stage_blocks) + for _ in range(num_stacks) + ]) + + self.inters = ResLayer( + BasicBlock, + cur_channel, + cur_channel, + num_stacks - 1, + norm_cfg=norm_cfg) + + self.conv1x1s = nn.ModuleList([ + ConvModule( + cur_channel, cur_channel, 1, norm_cfg=norm_cfg, act_cfg=None) + for _ in range(num_stacks - 1) + ]) + + self.out_convs = nn.ModuleList([ + ConvModule( + cur_channel, feat_channel, 3, padding=1, norm_cfg=norm_cfg) + for _ in range(num_stacks) + ]) + + self.remap_convs = nn.ModuleList([ + ConvModule( + feat_channel, cur_channel, 1, norm_cfg=norm_cfg, act_cfg=None) + for _ in range(num_stacks - 1) + ]) + + self.relu = nn.ReLU(inplace=True) + + def init_weights(self, pretrained=None): + """Init module weights. + + We do nothing in this function because all modules we used + (ConvModule, BasicBlock and etc.) have default initialization, and + currently we don't provide pretrained model of HourglassNet. + + Detector's __init__() will call backbone's init_weights() with + pretrained as input, so we keep this function. + """ + # Training Centripetal Model needs to reset parameters for Conv2d + for m in self.modules(): + if isinstance(m, nn.Conv2d): + m.reset_parameters() + + def forward(self, x): + """Forward function.""" + inter_feat = self.stem(x) + out_feats = [] + + for ind in range(self.num_stacks): + single_hourglass = self.hourglass_modules[ind] + out_conv = self.out_convs[ind] + + hourglass_feat = single_hourglass(inter_feat) + out_feat = out_conv(hourglass_feat) + out_feats.append(out_feat) + + if ind < self.num_stacks - 1: + inter_feat = self.conv1x1s[ind]( + inter_feat) + self.remap_convs[ind]( + out_feat) + inter_feat = self.inters[ind](self.relu(inter_feat)) + + return out_feats diff --git a/detection/mmdet/models/backbones/hrnet.py b/detection/mmdet/models/backbones/hrnet.py new file mode 100644 index 0000000..c0fd0a9 --- /dev/null +++ b/detection/mmdet/models/backbones/hrnet.py @@ -0,0 +1,537 @@ +import torch.nn as nn +from mmcv.cnn import (build_conv_layer, build_norm_layer, constant_init, + kaiming_init) +from mmcv.runner import load_checkpoint +from torch.nn.modules.batchnorm import _BatchNorm + +from mmdet.utils import get_root_logger +from ..builder import BACKBONES +from .resnet import BasicBlock, Bottleneck + + +class HRModule(nn.Module): + """High-Resolution Module for HRNet. + + In this module, every branch has 4 BasicBlocks/Bottlenecks. Fusion/Exchange + is in this module. + """ + + def __init__(self, + num_branches, + blocks, + num_blocks, + in_channels, + num_channels, + multiscale_output=True, + with_cp=False, + conv_cfg=None, + norm_cfg=dict(type='BN')): + super(HRModule, self).__init__() + self._check_branches(num_branches, num_blocks, in_channels, + num_channels) + + self.in_channels = in_channels + self.num_branches = num_branches + + self.multiscale_output = multiscale_output + self.norm_cfg = norm_cfg + self.conv_cfg = conv_cfg + self.with_cp = with_cp + self.branches = self._make_branches(num_branches, blocks, num_blocks, + num_channels) + self.fuse_layers = self._make_fuse_layers() + self.relu = nn.ReLU(inplace=False) + + def _check_branches(self, num_branches, num_blocks, in_channels, + num_channels): + if num_branches != len(num_blocks): + error_msg = f'NUM_BRANCHES({num_branches}) ' \ + f'!= NUM_BLOCKS({len(num_blocks)})' + raise ValueError(error_msg) + + if num_branches != len(num_channels): + error_msg = f'NUM_BRANCHES({num_branches}) ' \ + f'!= NUM_CHANNELS({len(num_channels)})' + raise ValueError(error_msg) + + if num_branches != len(in_channels): + error_msg = f'NUM_BRANCHES({num_branches}) ' \ + f'!= NUM_INCHANNELS({len(in_channels)})' + raise ValueError(error_msg) + + def _make_one_branch(self, + branch_index, + block, + num_blocks, + num_channels, + stride=1): + downsample = None + if stride != 1 or \ + self.in_channels[branch_index] != \ + num_channels[branch_index] * block.expansion: + downsample = nn.Sequential( + build_conv_layer( + self.conv_cfg, + self.in_channels[branch_index], + num_channels[branch_index] * block.expansion, + kernel_size=1, + stride=stride, + bias=False), + build_norm_layer(self.norm_cfg, num_channels[branch_index] * + block.expansion)[1]) + + layers = [] + layers.append( + block( + self.in_channels[branch_index], + num_channels[branch_index], + stride, + downsample=downsample, + with_cp=self.with_cp, + norm_cfg=self.norm_cfg, + conv_cfg=self.conv_cfg)) + self.in_channels[branch_index] = \ + num_channels[branch_index] * block.expansion + for i in range(1, num_blocks[branch_index]): + layers.append( + block( + self.in_channels[branch_index], + num_channels[branch_index], + with_cp=self.with_cp, + norm_cfg=self.norm_cfg, + conv_cfg=self.conv_cfg)) + + return nn.Sequential(*layers) + + def _make_branches(self, num_branches, block, num_blocks, num_channels): + branches = [] + + for i in range(num_branches): + branches.append( + self._make_one_branch(i, block, num_blocks, num_channels)) + + return nn.ModuleList(branches) + + def _make_fuse_layers(self): + if self.num_branches == 1: + return None + + num_branches = self.num_branches + in_channels = self.in_channels + fuse_layers = [] + num_out_branches = num_branches if self.multiscale_output else 1 + for i in range(num_out_branches): + fuse_layer = [] + for j in range(num_branches): + if j > i: + fuse_layer.append( + nn.Sequential( + build_conv_layer( + self.conv_cfg, + in_channels[j], + in_channels[i], + kernel_size=1, + stride=1, + padding=0, + bias=False), + build_norm_layer(self.norm_cfg, in_channels[i])[1], + nn.Upsample( + scale_factor=2**(j - i), mode='nearest'))) + elif j == i: + fuse_layer.append(None) + else: + conv_downsamples = [] + for k in range(i - j): + if k == i - j - 1: + conv_downsamples.append( + nn.Sequential( + build_conv_layer( + self.conv_cfg, + in_channels[j], + in_channels[i], + kernel_size=3, + stride=2, + padding=1, + bias=False), + build_norm_layer(self.norm_cfg, + in_channels[i])[1])) + else: + conv_downsamples.append( + nn.Sequential( + build_conv_layer( + self.conv_cfg, + in_channels[j], + in_channels[j], + kernel_size=3, + stride=2, + padding=1, + bias=False), + build_norm_layer(self.norm_cfg, + in_channels[j])[1], + nn.ReLU(inplace=False))) + fuse_layer.append(nn.Sequential(*conv_downsamples)) + fuse_layers.append(nn.ModuleList(fuse_layer)) + + return nn.ModuleList(fuse_layers) + + def forward(self, x): + """Forward function.""" + if self.num_branches == 1: + return [self.branches[0](x[0])] + + for i in range(self.num_branches): + x[i] = self.branches[i](x[i]) + + x_fuse = [] + for i in range(len(self.fuse_layers)): + y = 0 + for j in range(self.num_branches): + if i == j: + y += x[j] + else: + y += self.fuse_layers[i][j](x[j]) + x_fuse.append(self.relu(y)) + return x_fuse + + +@BACKBONES.register_module() +class HRNet(nn.Module): + """HRNet backbone. + + High-Resolution Representations for Labeling Pixels and Regions + arXiv: https://arxiv.org/abs/1904.04514 + + Args: + extra (dict): detailed configuration for each stage of HRNet. + in_channels (int): Number of input image channels. Default: 3. + conv_cfg (dict): dictionary to construct and config conv layer. + norm_cfg (dict): dictionary to construct and config norm layer. + norm_eval (bool): Whether to set norm layers to eval mode, namely, + freeze running stats (mean and var). Note: Effect on Batch Norm + and its variants only. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. + zero_init_residual (bool): whether to use zero init for last norm layer + in resblocks to let them behave as identity. + + Example: + >>> from mmdet.models import HRNet + >>> import torch + >>> extra = dict( + >>> stage1=dict( + >>> num_modules=1, + >>> num_branches=1, + >>> block='BOTTLENECK', + >>> num_blocks=(4, ), + >>> num_channels=(64, )), + >>> stage2=dict( + >>> num_modules=1, + >>> num_branches=2, + >>> block='BASIC', + >>> num_blocks=(4, 4), + >>> num_channels=(32, 64)), + >>> stage3=dict( + >>> num_modules=4, + >>> num_branches=3, + >>> block='BASIC', + >>> num_blocks=(4, 4, 4), + >>> num_channels=(32, 64, 128)), + >>> stage4=dict( + >>> num_modules=3, + >>> num_branches=4, + >>> block='BASIC', + >>> num_blocks=(4, 4, 4, 4), + >>> num_channels=(32, 64, 128, 256))) + >>> self = HRNet(extra, in_channels=1) + >>> self.eval() + >>> inputs = torch.rand(1, 1, 32, 32) + >>> level_outputs = self.forward(inputs) + >>> for level_out in level_outputs: + ... print(tuple(level_out.shape)) + (1, 32, 8, 8) + (1, 64, 4, 4) + (1, 128, 2, 2) + (1, 256, 1, 1) + """ + + blocks_dict = {'BASIC': BasicBlock, 'BOTTLENECK': Bottleneck} + + def __init__(self, + extra, + in_channels=3, + conv_cfg=None, + norm_cfg=dict(type='BN'), + norm_eval=True, + with_cp=False, + zero_init_residual=False): + super(HRNet, self).__init__() + self.extra = extra + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.norm_eval = norm_eval + self.with_cp = with_cp + self.zero_init_residual = zero_init_residual + + # stem net + self.norm1_name, norm1 = build_norm_layer(self.norm_cfg, 64, postfix=1) + self.norm2_name, norm2 = build_norm_layer(self.norm_cfg, 64, postfix=2) + + self.conv1 = build_conv_layer( + self.conv_cfg, + in_channels, + 64, + kernel_size=3, + stride=2, + padding=1, + bias=False) + + self.add_module(self.norm1_name, norm1) + self.conv2 = build_conv_layer( + self.conv_cfg, + 64, + 64, + kernel_size=3, + stride=2, + padding=1, + bias=False) + + self.add_module(self.norm2_name, norm2) + self.relu = nn.ReLU(inplace=True) + + # stage 1 + self.stage1_cfg = self.extra['stage1'] + num_channels = self.stage1_cfg['num_channels'][0] + block_type = self.stage1_cfg['block'] + num_blocks = self.stage1_cfg['num_blocks'][0] + + block = self.blocks_dict[block_type] + stage1_out_channels = num_channels * block.expansion + self.layer1 = self._make_layer(block, 64, num_channels, num_blocks) + + # stage 2 + self.stage2_cfg = self.extra['stage2'] + num_channels = self.stage2_cfg['num_channels'] + block_type = self.stage2_cfg['block'] + + block = self.blocks_dict[block_type] + num_channels = [channel * block.expansion for channel in num_channels] + self.transition1 = self._make_transition_layer([stage1_out_channels], + num_channels) + self.stage2, pre_stage_channels = self._make_stage( + self.stage2_cfg, num_channels) + + # stage 3 + self.stage3_cfg = self.extra['stage3'] + num_channels = self.stage3_cfg['num_channels'] + block_type = self.stage3_cfg['block'] + + block = self.blocks_dict[block_type] + num_channels = [channel * block.expansion for channel in num_channels] + self.transition2 = self._make_transition_layer(pre_stage_channels, + num_channels) + self.stage3, pre_stage_channels = self._make_stage( + self.stage3_cfg, num_channels) + + # stage 4 + self.stage4_cfg = self.extra['stage4'] + num_channels = self.stage4_cfg['num_channels'] + block_type = self.stage4_cfg['block'] + + block = self.blocks_dict[block_type] + num_channels = [channel * block.expansion for channel in num_channels] + self.transition3 = self._make_transition_layer(pre_stage_channels, + num_channels) + self.stage4, pre_stage_channels = self._make_stage( + self.stage4_cfg, num_channels) + + @property + def norm1(self): + """nn.Module: the normalization layer named "norm1" """ + return getattr(self, self.norm1_name) + + @property + def norm2(self): + """nn.Module: the normalization layer named "norm2" """ + return getattr(self, self.norm2_name) + + def _make_transition_layer(self, num_channels_pre_layer, + num_channels_cur_layer): + num_branches_cur = len(num_channels_cur_layer) + num_branches_pre = len(num_channels_pre_layer) + + transition_layers = [] + for i in range(num_branches_cur): + if i < num_branches_pre: + if num_channels_cur_layer[i] != num_channels_pre_layer[i]: + transition_layers.append( + nn.Sequential( + build_conv_layer( + self.conv_cfg, + num_channels_pre_layer[i], + num_channels_cur_layer[i], + kernel_size=3, + stride=1, + padding=1, + bias=False), + build_norm_layer(self.norm_cfg, + num_channels_cur_layer[i])[1], + nn.ReLU(inplace=True))) + else: + transition_layers.append(None) + else: + conv_downsamples = [] + for j in range(i + 1 - num_branches_pre): + in_channels = num_channels_pre_layer[-1] + out_channels = num_channels_cur_layer[i] \ + if j == i - num_branches_pre else in_channels + conv_downsamples.append( + nn.Sequential( + build_conv_layer( + self.conv_cfg, + in_channels, + out_channels, + kernel_size=3, + stride=2, + padding=1, + bias=False), + build_norm_layer(self.norm_cfg, out_channels)[1], + nn.ReLU(inplace=True))) + transition_layers.append(nn.Sequential(*conv_downsamples)) + + return nn.ModuleList(transition_layers) + + def _make_layer(self, block, inplanes, planes, blocks, stride=1): + downsample = None + if stride != 1 or inplanes != planes * block.expansion: + downsample = nn.Sequential( + build_conv_layer( + self.conv_cfg, + inplanes, + planes * block.expansion, + kernel_size=1, + stride=stride, + bias=False), + build_norm_layer(self.norm_cfg, planes * block.expansion)[1]) + + layers = [] + layers.append( + block( + inplanes, + planes, + stride, + downsample=downsample, + with_cp=self.with_cp, + norm_cfg=self.norm_cfg, + conv_cfg=self.conv_cfg)) + inplanes = planes * block.expansion + for i in range(1, blocks): + layers.append( + block( + inplanes, + planes, + with_cp=self.with_cp, + norm_cfg=self.norm_cfg, + conv_cfg=self.conv_cfg)) + + return nn.Sequential(*layers) + + def _make_stage(self, layer_config, in_channels, multiscale_output=True): + num_modules = layer_config['num_modules'] + num_branches = layer_config['num_branches'] + num_blocks = layer_config['num_blocks'] + num_channels = layer_config['num_channels'] + block = self.blocks_dict[layer_config['block']] + + hr_modules = [] + for i in range(num_modules): + # multi_scale_output is only used for the last module + if not multiscale_output and i == num_modules - 1: + reset_multiscale_output = False + else: + reset_multiscale_output = True + + hr_modules.append( + HRModule( + num_branches, + block, + num_blocks, + in_channels, + num_channels, + reset_multiscale_output, + with_cp=self.with_cp, + norm_cfg=self.norm_cfg, + conv_cfg=self.conv_cfg)) + + return nn.Sequential(*hr_modules), in_channels + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + if isinstance(pretrained, str): + logger = get_root_logger() + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + for m in self.modules(): + if isinstance(m, nn.Conv2d): + kaiming_init(m) + elif isinstance(m, (_BatchNorm, nn.GroupNorm)): + constant_init(m, 1) + + if self.zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + constant_init(m.norm3, 0) + elif isinstance(m, BasicBlock): + constant_init(m.norm2, 0) + else: + raise TypeError('pretrained must be a str or None') + + def forward(self, x): + """Forward function.""" + x = self.conv1(x) + x = self.norm1(x) + x = self.relu(x) + x = self.conv2(x) + x = self.norm2(x) + x = self.relu(x) + x = self.layer1(x) + + x_list = [] + for i in range(self.stage2_cfg['num_branches']): + if self.transition1[i] is not None: + x_list.append(self.transition1[i](x)) + else: + x_list.append(x) + y_list = self.stage2(x_list) + + x_list = [] + for i in range(self.stage3_cfg['num_branches']): + if self.transition2[i] is not None: + x_list.append(self.transition2[i](y_list[-1])) + else: + x_list.append(y_list[i]) + y_list = self.stage3(x_list) + + x_list = [] + for i in range(self.stage4_cfg['num_branches']): + if self.transition3[i] is not None: + x_list.append(self.transition3[i](y_list[-1])) + else: + x_list.append(y_list[i]) + y_list = self.stage4(x_list) + + return y_list + + def train(self, mode=True): + """Convert the model into training mode will keeping the normalization + layer freezed.""" + super(HRNet, self).train(mode) + if mode and self.norm_eval: + for m in self.modules(): + # trick: eval have effect on BatchNorm only + if isinstance(m, _BatchNorm): + m.eval() diff --git a/detection/mmdet/models/backbones/litv2.py b/detection/mmdet/models/backbones/litv2.py new file mode 100644 index 0000000..054e741 --- /dev/null +++ b/detection/mmdet/models/backbones/litv2.py @@ -0,0 +1,646 @@ +from traceback import print_tb +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as checkpoint +import numpy as np +from timm.models.layers import DropPath, to_2tuple, trunc_normal_ + +from mmcv_custom import load_checkpoint +from mmdet.utils import get_root_logger +from ..builder import BACKBONES +from mm_modules.DCN.modules.deform_conv2d import DeformConv2dPack +import math + +class DWConv(nn.Module): + def __init__(self, dim=768): + super(DWConv, self).__init__() + self.dwconv = nn.Conv2d(dim, dim, 3, 1, 1, bias=True, groups=dim) + + def forward(self, x): + x = self.dwconv(x) + x = x.flatten(2).transpose(1, 2) + return x + +class DWMlp(nn.Module): + def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0., linear=False): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.dwconv = DWConv(hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + self.linear = linear + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + elif isinstance(m, nn.Conv2d): + fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + fan_out //= m.groups + m.weight.data.normal_(0, math.sqrt(2.0 / fan_out)) + if m.bias is not None: + m.bias.data.zero_() + + def forward(self, x, H, W): + x = self.fc1(x) + B, N, C = x.shape + x = x.transpose(1, 2).view(B, C, H, W) + x = self.dwconv(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + +class Mlp(nn.Module): + """ Multilayer perceptron.""" + + def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0., sr_ratio=1, alpha=0.5): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + +class PatchEmbed(nn.Module): + """ Image to Patch Embedding + + Args: + patch_size (int): Patch token size. Default: 4. + in_chans (int): Number of input image channels. Default: 3. + embed_dim (int): Number of linear projection output channels. Default: 96. + norm_layer (nn.Module, optional): Normalization layer. Default: None + """ + + def __init__(self, patch_size=4, in_chans=3, embed_dim=96, norm_layer=None): + super().__init__() + patch_size = to_2tuple(patch_size) + self.patch_size = patch_size + + self.in_chans = in_chans + self.embed_dim = embed_dim + + self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size) + if norm_layer is not None: + self.norm = norm_layer(embed_dim) + else: + self.norm = None + + def forward(self, x): + """Forward function.""" + # padding + _, _, H, W = x.size() + if W % self.patch_size[1] != 0: + x = F.pad(x, (0, self.patch_size[1] - W % self.patch_size[1])) + if H % self.patch_size[0] != 0: + x = F.pad(x, (0, 0, 0, self.patch_size[0] - H % self.patch_size[0])) + + x = self.proj(x) # B C Wh Ww + if self.norm is not None: + Wh, Ww = x.size(2), x.size(3) + x = x.flatten(2).transpose(1, 2) + x = self.norm(x) + x = x.transpose(1, 2).view(-1, self.embed_dim, Wh, Ww) + + return x + + +class HiLo(nn.Module): + """ + HiLo Attention + + Link: https://arxiv.org/abs/2205.13213 + """ + def __init__(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0., window_size=2, alpha=0.5): + super().__init__() + assert dim % num_heads == 0, f"dim {dim} should be divided by num_heads {num_heads}." + head_dim = int(dim/num_heads) + + self.dim = dim + # self-attention heads in Lo-Fi + self.l_heads = int(num_heads * alpha) + # token dimension in Lo-Fi + self.l_dim = self.l_heads * head_dim + + # self-attention heads in Hi-Fi + self.h_heads = num_heads - self.l_heads + # token dimension in Hi-Fi + self.h_dim = self.h_heads * head_dim + + # local window size. The `s` in our paper. + self.ws = window_size + + if self.ws == 1: + # ws == 1 is equal to a standard multi-head self-attention + self.h_heads = 0 + self.h_dim = 0 + self.l_heads = num_heads + self.l_dim = dim + + self.scale = qk_scale or head_dim ** -0.5 + + # Low frequence attention (Lo-Fi) + if self.l_heads > 0: + if self.ws != 1: + self.sr = nn.AvgPool2d(kernel_size=window_size, stride=window_size) + self.l_q = nn.Linear(self.dim, self.l_dim, bias=qkv_bias) + self.l_kv = nn.Linear(self.dim, self.l_dim * 2, bias=qkv_bias) + self.l_proj = nn.Linear(self.l_dim, self.l_dim) + + # High frequence attention (Hi-Fi) + if self.h_heads > 0: + self.h_qkv = nn.Linear(self.dim, self.h_dim * 3, bias=qkv_bias) + self.h_proj = nn.Linear(self.h_dim, self.h_dim) + + + def hifi(self, x): + B, H, W, C = x.shape + h_group, w_group = H // self.ws, W // self.ws + total_groups = h_group * w_group + + x = x.reshape(B, h_group, self.ws, w_group, self.ws, C).transpose(2, 3) + + qkv = self.h_qkv(x).reshape(B, total_groups, -1, 3, self.h_heads, self.h_dim // self.h_heads).permute(3, 0, 1, 4, 2, 5) + q, k, v = qkv[0], qkv[1], qkv[2] # B, hw, n_head, ws*ws, head_dim + attn = (q @ k.transpose(-2, -1)) * self.scale # B, hw, n_head, ws*ws, ws*ws + attn = attn.softmax(dim=-1) + attn = (attn @ v).transpose(2, 3).reshape(B, h_group, w_group, self.ws, self.ws, self.h_dim) + x = attn.transpose(2, 3).reshape(B, h_group * self.ws, w_group * self.ws, self.h_dim) + x = self.h_proj(x) + return x + + def lofi(self, x): + B, H, W, C = x.shape + + q = self.l_q(x).reshape(B, H * W, self.l_heads, self.l_dim // self.l_heads).permute(0, 2, 1, 3) + + if self.ws > 1: + x_ = x.permute(0, 3, 1, 2) + x_ = self.sr(x_).reshape(B, C, -1).permute(0, 2, 1) + kv = self.l_kv(x_).reshape(B, -1, 2, self.l_heads, self.l_dim // self.l_heads).permute(2, 0, 3, 1, 4) + else: + kv = self.l_kv(x).reshape(B, -1, 2, self.l_heads, self.l_dim // self.l_heads).permute(2, 0, 3, 1, 4) + k, v = kv[0], kv[1] + + attn = (q @ k.transpose(-2, -1)) * self.scale + attn = attn.softmax(dim=-1) + + x = (attn @ v).transpose(1, 2).reshape(B, H, W, self.l_dim) + x = self.l_proj(x) + return x + + def forward(self, x, H, W): + B, N, C = x.shape + + x = x.reshape(B, H, W, C) + + # pad feature maps to multiples of window size + pad_l = pad_t = 0 + pad_r = (self.ws - W % self.ws) % self.ws + pad_b = (self.ws - H % self.ws) % self.ws + x = F.pad(x, (0, 0, pad_l, pad_r, pad_t, pad_b)) + + if self.h_heads == 0: + x = self.lofi(x) + if pad_r > 0 or pad_b > 0: + x = x[:, :H, :W, :] + return x.reshape(B, N, C) + + if self.l_heads == 0: + x = self.hifi(x) + if pad_r > 0 or pad_b > 0: + x = x[:, :H, :W, :] + return x.reshape(B, N, C) + + hifi_out = self.hifi(x) + lofi_out = self.lofi(x) + if pad_r > 0 or pad_b > 0: + x = torch.cat((hifi_out[:, :H, :W, :], lofi_out[:, :H, :W, :]), dim=-1) + else: + x = torch.cat((hifi_out, lofi_out), dim=-1) + + x = x.reshape(B, N, C) + return x + + def flops(self, N): + H = int(N ** 0.5) + Hp = Wp = self.ws * math.ceil(H / self.ws) + + Np = Hp * Wp + + # For Hi-Fi + # qkv + hifi_flops = Np * self.dim * self.h_dim * 3 + nW = Np / self.ws / self.ws + window_len = self.ws * self.ws + # q @ k and attn @ v + window_flops = window_len * window_len * self.h_dim * 2 + hifi_flops += nW * window_flops + # projection + hifi_flops += Np * self.h_dim * self.h_dim + + # for Lo-Fi + # q + lofi_flops = Np * self.dim * self.l_dim + # H = int(Np ** 0.5) + kv_len = (Hp // self.ws) ** 2 + # k, v + lofi_flops += kv_len * self.dim * self.l_dim * 2 + # q @ k and attn @ v + lofi_flops += Np * self.l_dim * kv_len * 2 + # projection + lofi_flops += Np * self.l_dim * self.l_dim + + return hifi_flops + lofi_flops + + +class Block(nn.Module): + """ Transformer Block. + + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads. + window_size (int): Window size. + shift_size (int): Shift size for SW-MSA. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float, optional): Stochastic depth rate. Default: 0.0 + act_layer (nn.Module, optional): Activation layer. Default: nn.GELU + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__(self, dim, num_heads, input_resolution, window_size=7, shift_size=0, + mlp_ratio=4., qkv_bias=True, qk_scale=None, drop=0., attn_drop=0., drop_path=0., + act_layer=nn.GELU, norm_layer=nn.LayerNorm, local_ws=1, alpha=0.5): + super().__init__() + self.dim = dim + self.num_heads = num_heads + self.input_resolution = input_resolution + self.shift_size = shift_size + self.mlp_ratio = mlp_ratio + + self.norm1 = norm_layer(dim) + self.attn = HiLo(dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, + attn_drop=attn_drop, proj_drop=drop, window_size=local_ws, alpha=alpha) + + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = DWMlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) + + def forward(self, x): + x = x + self.drop_path(self.attn(self.norm1(x), self.H, self.W)) + x = x + self.drop_path(self.mlp(self.norm2(x), self.H, self.W)) + return x + + +class ConvFFNBlock(nn.Module): + """ Convolutional FFN Block. + + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads. + window_size (int): Window size. + shift_size (int): Shift size for SW-MSA. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float, optional): Stochastic depth rate. Default: 0.0 + act_layer (nn.Module, optional): Activation layer. Default: nn.GELU + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__(self, dim, num_heads, input_resolution, window_size=7, shift_size=0, + mlp_ratio=4., qkv_bias=True, qk_scale=None, drop=0., attn_drop=0., drop_path=0., + act_layer=nn.GELU, norm_layer=nn.LayerNorm, local_ws=1, alpha=0.5): + super().__init__() + self.dim = dim + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + + self.local_ws = local_ws + self.mlp = DWMlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) + + + def forward(self, x): + x = x + self.drop_path(self.mlp(self.norm2(x), self.H, self.W)) + return x + + +class DTM(nn.Module): + r""" Deformable Token Merging. + + Link: https://arxiv.org/abs/2105.14217 + + Args: + input_resolution (tuple[int]): Resolution of input feature. + dim (int): Number of input channels. + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__(self, input_resolution, dim, norm_layer=nn.LayerNorm): + super().__init__() + self.input_resolution = input_resolution + self.dim = dim + self.kernel_size = 2 + self.stride = 2 + self.padding = 0 + self.c_in = dim + self.c_out = dim*2 + self.dconv = DeformConv2dPack(dim, dim*2, kernel_size=2, stride=2, padding=0) + self.norm_layer = nn.BatchNorm2d(dim*2) + self.act_layer = nn.GELU() + + def forward(self, x, H, W, return_offset=False): + """ + x: B, H*W, C + """ + B, L, C = x.shape + x = x.reshape(B, H, W, C).permute(0, 3, 1, 2).contiguous() + x, offset = self.dconv(x, return_offset=False) + _, _, new_H, new_W = x.shape + x = self.act_layer(self.norm_layer(x)).flatten(2).transpose(1, 2) + if return_offset: + return x, new_H, new_W, offset + else: + return x, new_H, new_W + + def extra_repr(self) -> str: + return f"input_resolution={self.input_resolution}, dim={self.dim}" + + +class LITLayer(nn.Module): + """ A basic Swin Transformer layer for one stage. + + Args: + dim (int): Number of feature channels + depth (int): Depths of this stage. + num_heads (int): Number of attention head. + window_size (int): Local window size. Default: 7. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None + use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. + """ + + def __init__(self, + dim, + depth, + num_heads, + input_resolution, + window_size=7, + mlp_ratio=4., + qkv_bias=True, + qk_scale=None, + drop=0., + attn_drop=0., + drop_path=0., + norm_layer=nn.LayerNorm, + downsample=None, + use_checkpoint=False, + has_msa=True, + local_ws=1, + alpha=0.5 + ): + super().__init__() + self.window_size = window_size + self.shift_size = window_size // 2 + self.depth = depth + self.use_checkpoint = use_checkpoint + self.input_resolution = input_resolution + # build blocks + self.has_msa = has_msa + block = Block if has_msa else ConvFFNBlock + self.blocks = nn.ModuleList([ + block( + dim=dim, + num_heads=num_heads, + input_resolution=input_resolution, + window_size=window_size, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop, + attn_drop=attn_drop, + drop_path=drop_path[i] if isinstance(drop_path, list) else drop_path, + norm_layer=norm_layer, + local_ws=local_ws, + alpha=alpha + ) + for i in range(depth)]) + + # patch merging layer + if downsample is not None: + self.downsample = downsample(input_resolution, dim=dim, norm_layer=norm_layer) + else: + self.downsample = None + + def forward(self, x, H, W): + """ Forward function. + + Args: + x: Input feature, tensor size (B, H*W, C). + H, W: Spatial resolution of the input feature. + """ + for i, blk in enumerate(self.blocks): + blk.H = H + blk.W = W + if self.use_checkpoint: + x = checkpoint.checkpoint(blk, x) + else: + x = blk(x) + if self.downsample is not None: + x_down, Wh, Ww = self.downsample(x, H, W) + return x, H, W, x_down, Wh, Ww + else: + return x, H, W, x, H, W + +@BACKBONES.register_module() +class LITv2(nn.Module): + def __init__(self, + pretrain_img_size=224, + patch_size=4, + in_chans=3, + embed_dim=96, + depths=[2, 2, 6, 2], + num_heads=[3, 6, 12, 24], + window_size=7, + mlp_ratio=4., + qkv_bias=True, + qk_scale=None, + drop_rate=0., + attn_drop_rate=0., + drop_path_rate=0.2, + norm_layer=nn.LayerNorm, + ape=False, + patch_norm=True, + out_indices=(0, 1, 2, 3), + frozen_stages=-1, + use_checkpoint=False, + has_msa=[0, 0, 1, 1], + alpha=0.5, + local_ws=[0, 0, 2, 1]): + super().__init__() + + # new from v2 + self.local_ws = local_ws + self.alpha = alpha + self.num_heads = num_heads + + # self.fp16_enabled = True + self.pretrain_img_size = pretrain_img_size + self.num_layers = len(depths) + self.embed_dim = embed_dim + self.ape = ape + self.patch_norm = patch_norm + self.out_indices = out_indices + self.frozen_stages = frozen_stages + self.has_msa = has_msa + # split image into non-overlapping patches + self.patch_embed = PatchEmbed( + patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim, + norm_layer=norm_layer if self.patch_norm else None) + + # absolute position embedding + input_resolution = [800 // patch_size, 800 // patch_size] + + # stochastic depth + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))] # stochastic depth decay rule + + # build layers + self.layers = nn.ModuleList() + for i_layer in range(self.num_layers): + layer = LITLayer( + dim=int(embed_dim * 2 ** i_layer), + depth=depths[i_layer], + num_heads=num_heads[i_layer], + input_resolution=(input_resolution[0] // (2 ** i_layer), + input_resolution[1] // (2 ** i_layer)), + window_size=window_size, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop_rate, + attn_drop=attn_drop_rate, + drop_path=dpr[sum(depths[:i_layer]):sum(depths[:i_layer + 1])], + norm_layer=norm_layer, + downsample=DTM if (i_layer < self.num_layers - 1) else None, + use_checkpoint=use_checkpoint, + has_msa=self.has_msa[i_layer] == 1, + local_ws=self.local_ws[i_layer], + alpha=alpha + ) + self.layers.append(layer) + + num_features = [int(embed_dim * 2 ** i) for i in range(self.num_layers)] + self.num_features = num_features + + # add a norm layer for each output + for i_layer in out_indices: + if i_layer == 0: + continue + layer = norm_layer(num_features[i_layer]) + layer_name = f'norm{i_layer}' + self.add_module(layer_name, layer) + + self._freeze_stages() + + + def _freeze_stages(self): + if self.frozen_stages >= 0: + self.patch_embed.eval() + for param in self.patch_embed.parameters(): + param.requires_grad = False + + if self.frozen_stages >= 1 and self.ape: + self.absolute_pos_embed.requires_grad = False + + if self.frozen_stages >= 2: + self.pos_drop.eval() + for i in range(0, self.frozen_stages - 1): + m = self.layers[i] + m.eval() + for param in m.parameters(): + param.requires_grad = False + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + + def _init_weights(m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + if isinstance(pretrained, str): + self.apply(_init_weights) + logger = get_root_logger() + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + self.apply(_init_weights) + else: + raise TypeError('pretrained must be a str or None') + + def forward(self, x): + """Forward function.""" + + x = self.patch_embed(x) + + Wh, Ww = x.size(2), x.size(3) + x = x.flatten(2).transpose(1, 2) + + outs = [] + for i in range(self.num_layers): + layer = self.layers[i] + x_out, H, W, x, Wh, Ww = layer(x, Wh, Ww) + if i in self.out_indices: + if i != 0: + norm_layer = getattr(self, f'norm{i}') + x_out = norm_layer(x_out) + out = x_out.view(-1, H, W, self.num_features[i]).permute(0, 3, 1, 2).contiguous() + outs.append(out) + + return tuple(outs) + + def train(self, mode=True): + """Convert the model into training mode while keep layers freezed.""" + super(LITv2, self).train(mode) + self._freeze_stages() \ No newline at end of file diff --git a/detection/mmdet/models/backbones/regnet.py b/detection/mmdet/models/backbones/regnet.py new file mode 100644 index 0000000..91a602a --- /dev/null +++ b/detection/mmdet/models/backbones/regnet.py @@ -0,0 +1,325 @@ +import numpy as np +import torch.nn as nn +from mmcv.cnn import build_conv_layer, build_norm_layer + +from ..builder import BACKBONES +from .resnet import ResNet +from .resnext import Bottleneck + + +@BACKBONES.register_module() +class RegNet(ResNet): + """RegNet backbone. + + More details can be found in `paper `_ . + + Args: + arch (dict): The parameter of RegNets. + + - w0 (int): initial width + - wa (float): slope of width + - wm (float): quantization parameter to quantize the width + - depth (int): depth of the backbone + - group_w (int): width of group + - bot_mul (float): bottleneck ratio, i.e. expansion of bottleneck. + strides (Sequence[int]): Strides of the first block of each stage. + base_channels (int): Base channels after stem layer. + in_channels (int): Number of input image channels. Default: 3. + dilations (Sequence[int]): Dilation of each stage. + out_indices (Sequence[int]): Output from which stages. + style (str): `pytorch` or `caffe`. If set to "pytorch", the stride-two + layer is the 3x3 conv layer, otherwise the stride-two layer is + the first 1x1 conv layer. + frozen_stages (int): Stages to be frozen (all param fixed). -1 means + not freezing any parameters. + norm_cfg (dict): dictionary to construct and config norm layer. + norm_eval (bool): Whether to set norm layers to eval mode, namely, + freeze running stats (mean and var). Note: Effect on Batch Norm + and its variants only. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. + zero_init_residual (bool): whether to use zero init for last norm layer + in resblocks to let them behave as identity. + + Example: + >>> from mmdet.models import RegNet + >>> import torch + >>> self = RegNet( + arch=dict( + w0=88, + wa=26.31, + wm=2.25, + group_w=48, + depth=25, + bot_mul=1.0)) + >>> self.eval() + >>> inputs = torch.rand(1, 3, 32, 32) + >>> level_outputs = self.forward(inputs) + >>> for level_out in level_outputs: + ... print(tuple(level_out.shape)) + (1, 96, 8, 8) + (1, 192, 4, 4) + (1, 432, 2, 2) + (1, 1008, 1, 1) + """ + arch_settings = { + 'regnetx_400mf': + dict(w0=24, wa=24.48, wm=2.54, group_w=16, depth=22, bot_mul=1.0), + 'regnetx_800mf': + dict(w0=56, wa=35.73, wm=2.28, group_w=16, depth=16, bot_mul=1.0), + 'regnetx_1.6gf': + dict(w0=80, wa=34.01, wm=2.25, group_w=24, depth=18, bot_mul=1.0), + 'regnetx_3.2gf': + dict(w0=88, wa=26.31, wm=2.25, group_w=48, depth=25, bot_mul=1.0), + 'regnetx_4.0gf': + dict(w0=96, wa=38.65, wm=2.43, group_w=40, depth=23, bot_mul=1.0), + 'regnetx_6.4gf': + dict(w0=184, wa=60.83, wm=2.07, group_w=56, depth=17, bot_mul=1.0), + 'regnetx_8.0gf': + dict(w0=80, wa=49.56, wm=2.88, group_w=120, depth=23, bot_mul=1.0), + 'regnetx_12gf': + dict(w0=168, wa=73.36, wm=2.37, group_w=112, depth=19, bot_mul=1.0), + } + + def __init__(self, + arch, + in_channels=3, + stem_channels=32, + base_channels=32, + strides=(2, 2, 2, 2), + dilations=(1, 1, 1, 1), + out_indices=(0, 1, 2, 3), + style='pytorch', + deep_stem=False, + avg_down=False, + frozen_stages=-1, + conv_cfg=None, + norm_cfg=dict(type='BN', requires_grad=True), + norm_eval=True, + dcn=None, + stage_with_dcn=(False, False, False, False), + plugins=None, + with_cp=False, + zero_init_residual=True): + super(ResNet, self).__init__() + + # Generate RegNet parameters first + if isinstance(arch, str): + assert arch in self.arch_settings, \ + f'"arch": "{arch}" is not one of the' \ + ' arch_settings' + arch = self.arch_settings[arch] + elif not isinstance(arch, dict): + raise ValueError('Expect "arch" to be either a string ' + f'or a dict, got {type(arch)}') + + widths, num_stages = self.generate_regnet( + arch['w0'], + arch['wa'], + arch['wm'], + arch['depth'], + ) + # Convert to per stage format + stage_widths, stage_blocks = self.get_stages_from_blocks(widths) + # Generate group widths and bot muls + group_widths = [arch['group_w'] for _ in range(num_stages)] + self.bottleneck_ratio = [arch['bot_mul'] for _ in range(num_stages)] + # Adjust the compatibility of stage_widths and group_widths + stage_widths, group_widths = self.adjust_width_group( + stage_widths, self.bottleneck_ratio, group_widths) + + # Group params by stage + self.stage_widths = stage_widths + self.group_widths = group_widths + self.depth = sum(stage_blocks) + self.stem_channels = stem_channels + self.base_channels = base_channels + self.num_stages = num_stages + assert num_stages >= 1 and num_stages <= 4 + self.strides = strides + self.dilations = dilations + assert len(strides) == len(dilations) == num_stages + self.out_indices = out_indices + assert max(out_indices) < num_stages + self.style = style + self.deep_stem = deep_stem + self.avg_down = avg_down + self.frozen_stages = frozen_stages + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.with_cp = with_cp + self.norm_eval = norm_eval + self.dcn = dcn + self.stage_with_dcn = stage_with_dcn + if dcn is not None: + assert len(stage_with_dcn) == num_stages + self.plugins = plugins + self.zero_init_residual = zero_init_residual + self.block = Bottleneck + expansion_bak = self.block.expansion + self.block.expansion = 1 + self.stage_blocks = stage_blocks[:num_stages] + + self._make_stem_layer(in_channels, stem_channels) + + self.inplanes = stem_channels + self.res_layers = [] + for i, num_blocks in enumerate(self.stage_blocks): + stride = self.strides[i] + dilation = self.dilations[i] + group_width = self.group_widths[i] + width = int(round(self.stage_widths[i] * self.bottleneck_ratio[i])) + stage_groups = width // group_width + + dcn = self.dcn if self.stage_with_dcn[i] else None + if self.plugins is not None: + stage_plugins = self.make_stage_plugins(self.plugins, i) + else: + stage_plugins = None + + res_layer = self.make_res_layer( + block=self.block, + inplanes=self.inplanes, + planes=self.stage_widths[i], + num_blocks=num_blocks, + stride=stride, + dilation=dilation, + style=self.style, + avg_down=self.avg_down, + with_cp=self.with_cp, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + dcn=dcn, + plugins=stage_plugins, + groups=stage_groups, + base_width=group_width, + base_channels=self.stage_widths[i]) + self.inplanes = self.stage_widths[i] + layer_name = f'layer{i + 1}' + self.add_module(layer_name, res_layer) + self.res_layers.append(layer_name) + + self._freeze_stages() + + self.feat_dim = stage_widths[-1] + self.block.expansion = expansion_bak + + def _make_stem_layer(self, in_channels, base_channels): + self.conv1 = build_conv_layer( + self.conv_cfg, + in_channels, + base_channels, + kernel_size=3, + stride=2, + padding=1, + bias=False) + self.norm1_name, norm1 = build_norm_layer( + self.norm_cfg, base_channels, postfix=1) + self.add_module(self.norm1_name, norm1) + self.relu = nn.ReLU(inplace=True) + + def generate_regnet(self, + initial_width, + width_slope, + width_parameter, + depth, + divisor=8): + """Generates per block width from RegNet parameters. + + Args: + initial_width ([int]): Initial width of the backbone + width_slope ([float]): Slope of the quantized linear function + width_parameter ([int]): Parameter used to quantize the width. + depth ([int]): Depth of the backbone. + divisor (int, optional): The divisor of channels. Defaults to 8. + + Returns: + list, int: return a list of widths of each stage and the number \ + of stages + """ + assert width_slope >= 0 + assert initial_width > 0 + assert width_parameter > 1 + assert initial_width % divisor == 0 + widths_cont = np.arange(depth) * width_slope + initial_width + ks = np.round( + np.log(widths_cont / initial_width) / np.log(width_parameter)) + widths = initial_width * np.power(width_parameter, ks) + widths = np.round(np.divide(widths, divisor)) * divisor + num_stages = len(np.unique(widths)) + widths, widths_cont = widths.astype(int).tolist(), widths_cont.tolist() + return widths, num_stages + + @staticmethod + def quantize_float(number, divisor): + """Converts a float to closest non-zero int divisible by divisor. + + Args: + number (int): Original number to be quantized. + divisor (int): Divisor used to quantize the number. + + Returns: + int: quantized number that is divisible by devisor. + """ + return int(round(number / divisor) * divisor) + + def adjust_width_group(self, widths, bottleneck_ratio, groups): + """Adjusts the compatibility of widths and groups. + + Args: + widths (list[int]): Width of each stage. + bottleneck_ratio (float): Bottleneck ratio. + groups (int): number of groups in each stage + + Returns: + tuple(list): The adjusted widths and groups of each stage. + """ + bottleneck_width = [ + int(w * b) for w, b in zip(widths, bottleneck_ratio) + ] + groups = [min(g, w_bot) for g, w_bot in zip(groups, bottleneck_width)] + bottleneck_width = [ + self.quantize_float(w_bot, g) + for w_bot, g in zip(bottleneck_width, groups) + ] + widths = [ + int(w_bot / b) + for w_bot, b in zip(bottleneck_width, bottleneck_ratio) + ] + return widths, groups + + def get_stages_from_blocks(self, widths): + """Gets widths/stage_blocks of network at each stage. + + Args: + widths (list[int]): Width in each stage. + + Returns: + tuple(list): width and depth of each stage + """ + width_diff = [ + width != width_prev + for width, width_prev in zip(widths + [0], [0] + widths) + ] + stage_widths = [ + width for width, diff in zip(widths, width_diff[:-1]) if diff + ] + stage_blocks = np.diff([ + depth for depth, diff in zip(range(len(width_diff)), width_diff) + if diff + ]).tolist() + return stage_widths, stage_blocks + + def forward(self, x): + """Forward function.""" + x = self.conv1(x) + x = self.norm1(x) + x = self.relu(x) + + outs = [] + for i, layer_name in enumerate(self.res_layers): + res_layer = getattr(self, layer_name) + x = res_layer(x) + if i in self.out_indices: + outs.append(x) + return tuple(outs) diff --git a/detection/mmdet/models/backbones/res2net.py b/detection/mmdet/models/backbones/res2net.py new file mode 100644 index 0000000..7901b7f --- /dev/null +++ b/detection/mmdet/models/backbones/res2net.py @@ -0,0 +1,351 @@ +import math + +import torch +import torch.nn as nn +import torch.utils.checkpoint as cp +from mmcv.cnn import (build_conv_layer, build_norm_layer, constant_init, + kaiming_init) +from mmcv.runner import load_checkpoint +from torch.nn.modules.batchnorm import _BatchNorm + +from mmdet.utils import get_root_logger +from ..builder import BACKBONES +from .resnet import Bottleneck as _Bottleneck +from .resnet import ResNet + + +class Bottle2neck(_Bottleneck): + expansion = 4 + + def __init__(self, + inplanes, + planes, + scales=4, + base_width=26, + base_channels=64, + stage_type='normal', + **kwargs): + """Bottle2neck block for Res2Net. + + If style is "pytorch", the stride-two layer is the 3x3 conv layer, if + it is "caffe", the stride-two layer is the first 1x1 conv layer. + """ + super(Bottle2neck, self).__init__(inplanes, planes, **kwargs) + assert scales > 1, 'Res2Net degenerates to ResNet when scales = 1.' + width = int(math.floor(self.planes * (base_width / base_channels))) + + self.norm1_name, norm1 = build_norm_layer( + self.norm_cfg, width * scales, postfix=1) + self.norm3_name, norm3 = build_norm_layer( + self.norm_cfg, self.planes * self.expansion, postfix=3) + + self.conv1 = build_conv_layer( + self.conv_cfg, + self.inplanes, + width * scales, + kernel_size=1, + stride=self.conv1_stride, + bias=False) + self.add_module(self.norm1_name, norm1) + + if stage_type == 'stage' and self.conv2_stride != 1: + self.pool = nn.AvgPool2d( + kernel_size=3, stride=self.conv2_stride, padding=1) + convs = [] + bns = [] + + fallback_on_stride = False + if self.with_dcn: + fallback_on_stride = self.dcn.pop('fallback_on_stride', False) + if not self.with_dcn or fallback_on_stride: + for i in range(scales - 1): + convs.append( + build_conv_layer( + self.conv_cfg, + width, + width, + kernel_size=3, + stride=self.conv2_stride, + padding=self.dilation, + dilation=self.dilation, + bias=False)) + bns.append( + build_norm_layer(self.norm_cfg, width, postfix=i + 1)[1]) + self.convs = nn.ModuleList(convs) + self.bns = nn.ModuleList(bns) + else: + assert self.conv_cfg is None, 'conv_cfg must be None for DCN' + for i in range(scales - 1): + convs.append( + build_conv_layer( + self.dcn, + width, + width, + kernel_size=3, + stride=self.conv2_stride, + padding=self.dilation, + dilation=self.dilation, + bias=False)) + bns.append( + build_norm_layer(self.norm_cfg, width, postfix=i + 1)[1]) + self.convs = nn.ModuleList(convs) + self.bns = nn.ModuleList(bns) + + self.conv3 = build_conv_layer( + self.conv_cfg, + width * scales, + self.planes * self.expansion, + kernel_size=1, + bias=False) + self.add_module(self.norm3_name, norm3) + + self.stage_type = stage_type + self.scales = scales + self.width = width + delattr(self, 'conv2') + delattr(self, self.norm2_name) + + def forward(self, x): + """Forward function.""" + + def _inner_forward(x): + identity = x + + out = self.conv1(x) + out = self.norm1(out) + out = self.relu(out) + + if self.with_plugins: + out = self.forward_plugin(out, self.after_conv1_plugin_names) + + spx = torch.split(out, self.width, 1) + sp = self.convs[0](spx[0].contiguous()) + sp = self.relu(self.bns[0](sp)) + out = sp + for i in range(1, self.scales - 1): + if self.stage_type == 'stage': + sp = spx[i] + else: + sp = sp + spx[i] + sp = self.convs[i](sp.contiguous()) + sp = self.relu(self.bns[i](sp)) + out = torch.cat((out, sp), 1) + + if self.stage_type == 'normal' or self.conv2_stride == 1: + out = torch.cat((out, spx[self.scales - 1]), 1) + elif self.stage_type == 'stage': + out = torch.cat((out, self.pool(spx[self.scales - 1])), 1) + + if self.with_plugins: + out = self.forward_plugin(out, self.after_conv2_plugin_names) + + out = self.conv3(out) + out = self.norm3(out) + + if self.with_plugins: + out = self.forward_plugin(out, self.after_conv3_plugin_names) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + + return out + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x) + else: + out = _inner_forward(x) + + out = self.relu(out) + + return out + + +class Res2Layer(nn.Sequential): + """Res2Layer to build Res2Net style backbone. + + Args: + block (nn.Module): block used to build ResLayer. + inplanes (int): inplanes of block. + planes (int): planes of block. + num_blocks (int): number of blocks. + stride (int): stride of the first block. Default: 1 + avg_down (bool): Use AvgPool instead of stride conv when + downsampling in the bottle2neck. Default: False + conv_cfg (dict): dictionary to construct and config conv layer. + Default: None + norm_cfg (dict): dictionary to construct and config norm layer. + Default: dict(type='BN') + scales (int): Scales used in Res2Net. Default: 4 + base_width (int): Basic width of each scale. Default: 26 + """ + + def __init__(self, + block, + inplanes, + planes, + num_blocks, + stride=1, + avg_down=True, + conv_cfg=None, + norm_cfg=dict(type='BN'), + scales=4, + base_width=26, + **kwargs): + self.block = block + + downsample = None + if stride != 1 or inplanes != planes * block.expansion: + downsample = nn.Sequential( + nn.AvgPool2d( + kernel_size=stride, + stride=stride, + ceil_mode=True, + count_include_pad=False), + build_conv_layer( + conv_cfg, + inplanes, + planes * block.expansion, + kernel_size=1, + stride=1, + bias=False), + build_norm_layer(norm_cfg, planes * block.expansion)[1], + ) + + layers = [] + layers.append( + block( + inplanes=inplanes, + planes=planes, + stride=stride, + downsample=downsample, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + scales=scales, + base_width=base_width, + stage_type='stage', + **kwargs)) + inplanes = planes * block.expansion + for i in range(1, num_blocks): + layers.append( + block( + inplanes=inplanes, + planes=planes, + stride=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + scales=scales, + base_width=base_width, + **kwargs)) + super(Res2Layer, self).__init__(*layers) + + +@BACKBONES.register_module() +class Res2Net(ResNet): + """Res2Net backbone. + + Args: + scales (int): Scales used in Res2Net. Default: 4 + base_width (int): Basic width of each scale. Default: 26 + depth (int): Depth of res2net, from {50, 101, 152}. + in_channels (int): Number of input image channels. Default: 3. + num_stages (int): Res2net stages. Default: 4. + strides (Sequence[int]): Strides of the first block of each stage. + dilations (Sequence[int]): Dilation of each stage. + out_indices (Sequence[int]): Output from which stages. + style (str): `pytorch` or `caffe`. If set to "pytorch", the stride-two + layer is the 3x3 conv layer, otherwise the stride-two layer is + the first 1x1 conv layer. + deep_stem (bool): Replace 7x7 conv in input stem with 3 3x3 conv + avg_down (bool): Use AvgPool instead of stride conv when + downsampling in the bottle2neck. + frozen_stages (int): Stages to be frozen (stop grad and set eval mode). + -1 means not freezing any parameters. + norm_cfg (dict): Dictionary to construct and config norm layer. + norm_eval (bool): Whether to set norm layers to eval mode, namely, + freeze running stats (mean and var). Note: Effect on Batch Norm + and its variants only. + plugins (list[dict]): List of plugins for stages, each dict contains: + + - cfg (dict, required): Cfg dict to build plugin. + - position (str, required): Position inside block to insert + plugin, options are 'after_conv1', 'after_conv2', 'after_conv3'. + - stages (tuple[bool], optional): Stages to apply plugin, length + should be same as 'num_stages'. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. + zero_init_residual (bool): Whether to use zero init for last norm layer + in resblocks to let them behave as identity. + + Example: + >>> from mmdet.models import Res2Net + >>> import torch + >>> self = Res2Net(depth=50, scales=4, base_width=26) + >>> self.eval() + >>> inputs = torch.rand(1, 3, 32, 32) + >>> level_outputs = self.forward(inputs) + >>> for level_out in level_outputs: + ... print(tuple(level_out.shape)) + (1, 256, 8, 8) + (1, 512, 4, 4) + (1, 1024, 2, 2) + (1, 2048, 1, 1) + """ + + arch_settings = { + 50: (Bottle2neck, (3, 4, 6, 3)), + 101: (Bottle2neck, (3, 4, 23, 3)), + 152: (Bottle2neck, (3, 8, 36, 3)) + } + + def __init__(self, + scales=4, + base_width=26, + style='pytorch', + deep_stem=True, + avg_down=True, + **kwargs): + self.scales = scales + self.base_width = base_width + super(Res2Net, self).__init__( + style='pytorch', deep_stem=True, avg_down=True, **kwargs) + + def make_res_layer(self, **kwargs): + return Res2Layer( + scales=self.scales, + base_width=self.base_width, + base_channels=self.base_channels, + **kwargs) + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + if isinstance(pretrained, str): + logger = get_root_logger() + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + for m in self.modules(): + if isinstance(m, nn.Conv2d): + kaiming_init(m) + elif isinstance(m, (_BatchNorm, nn.GroupNorm)): + constant_init(m, 1) + + if self.dcn is not None: + for m in self.modules(): + if isinstance(m, Bottle2neck): + # dcn in Res2Net bottle2neck is in ModuleList + for n in m.convs: + if hasattr(n, 'conv_offset'): + constant_init(n.conv_offset, 0) + + if self.zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottle2neck): + constant_init(m.norm3, 0) + else: + raise TypeError('pretrained must be a str or None') diff --git a/detection/mmdet/models/backbones/resnest.py b/detection/mmdet/models/backbones/resnest.py new file mode 100644 index 0000000..48e1d8b --- /dev/null +++ b/detection/mmdet/models/backbones/resnest.py @@ -0,0 +1,317 @@ +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as cp +from mmcv.cnn import build_conv_layer, build_norm_layer + +from ..builder import BACKBONES +from ..utils import ResLayer +from .resnet import Bottleneck as _Bottleneck +from .resnet import ResNetV1d + + +class RSoftmax(nn.Module): + """Radix Softmax module in ``SplitAttentionConv2d``. + + Args: + radix (int): Radix of input. + groups (int): Groups of input. + """ + + def __init__(self, radix, groups): + super().__init__() + self.radix = radix + self.groups = groups + + def forward(self, x): + batch = x.size(0) + if self.radix > 1: + x = x.view(batch, self.groups, self.radix, -1).transpose(1, 2) + x = F.softmax(x, dim=1) + x = x.reshape(batch, -1) + else: + x = torch.sigmoid(x) + return x + + +class SplitAttentionConv2d(nn.Module): + """Split-Attention Conv2d in ResNeSt. + + Args: + in_channels (int): Number of channels in the input feature map. + channels (int): Number of intermediate channels. + kernel_size (int | tuple[int]): Size of the convolution kernel. + stride (int | tuple[int]): Stride of the convolution. + padding (int | tuple[int]): Zero-padding added to both sides of + dilation (int | tuple[int]): Spacing between kernel elements. + groups (int): Number of blocked connections from input channels to + output channels. + groups (int): Same as nn.Conv2d. + radix (int): Radix of SpltAtConv2d. Default: 2 + reduction_factor (int): Reduction factor of inter_channels. Default: 4. + conv_cfg (dict): Config dict for convolution layer. Default: None, + which means using conv2d. + norm_cfg (dict): Config dict for normalization layer. Default: None. + dcn (dict): Config dict for DCN. Default: None. + """ + + def __init__(self, + in_channels, + channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + radix=2, + reduction_factor=4, + conv_cfg=None, + norm_cfg=dict(type='BN'), + dcn=None): + super(SplitAttentionConv2d, self).__init__() + inter_channels = max(in_channels * radix // reduction_factor, 32) + self.radix = radix + self.groups = groups + self.channels = channels + self.with_dcn = dcn is not None + self.dcn = dcn + fallback_on_stride = False + if self.with_dcn: + fallback_on_stride = self.dcn.pop('fallback_on_stride', False) + if self.with_dcn and not fallback_on_stride: + assert conv_cfg is None, 'conv_cfg must be None for DCN' + conv_cfg = dcn + self.conv = build_conv_layer( + conv_cfg, + in_channels, + channels * radix, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups * radix, + bias=False) + # To be consistent with original implementation, starting from 0 + self.norm0_name, norm0 = build_norm_layer( + norm_cfg, channels * radix, postfix=0) + self.add_module(self.norm0_name, norm0) + self.relu = nn.ReLU(inplace=True) + self.fc1 = build_conv_layer( + None, channels, inter_channels, 1, groups=self.groups) + self.norm1_name, norm1 = build_norm_layer( + norm_cfg, inter_channels, postfix=1) + self.add_module(self.norm1_name, norm1) + self.fc2 = build_conv_layer( + None, inter_channels, channels * radix, 1, groups=self.groups) + self.rsoftmax = RSoftmax(radix, groups) + + @property + def norm0(self): + """nn.Module: the normalization layer named "norm0" """ + return getattr(self, self.norm0_name) + + @property + def norm1(self): + """nn.Module: the normalization layer named "norm1" """ + return getattr(self, self.norm1_name) + + def forward(self, x): + x = self.conv(x) + x = self.norm0(x) + x = self.relu(x) + + batch, rchannel = x.shape[:2] + batch = x.size(0) + if self.radix > 1: + splits = x.view(batch, self.radix, -1, *x.shape[2:]) + gap = splits.sum(dim=1) + else: + gap = x + gap = F.adaptive_avg_pool2d(gap, 1) + gap = self.fc1(gap) + + gap = self.norm1(gap) + gap = self.relu(gap) + + atten = self.fc2(gap) + atten = self.rsoftmax(atten).view(batch, -1, 1, 1) + + if self.radix > 1: + attens = atten.view(batch, self.radix, -1, *atten.shape[2:]) + out = torch.sum(attens * splits, dim=1) + else: + out = atten * x + return out.contiguous() + + +class Bottleneck(_Bottleneck): + """Bottleneck block for ResNeSt. + + Args: + inplane (int): Input planes of this block. + planes (int): Middle planes of this block. + groups (int): Groups of conv2. + base_width (int): Base of width in terms of base channels. Default: 4. + base_channels (int): Base of channels for calculating width. + Default: 64. + radix (int): Radix of SpltAtConv2d. Default: 2 + reduction_factor (int): Reduction factor of inter_channels in + SplitAttentionConv2d. Default: 4. + avg_down_stride (bool): Whether to use average pool for stride in + Bottleneck. Default: True. + kwargs (dict): Key word arguments for base class. + """ + expansion = 4 + + def __init__(self, + inplanes, + planes, + groups=1, + base_width=4, + base_channels=64, + radix=2, + reduction_factor=4, + avg_down_stride=True, + **kwargs): + """Bottleneck block for ResNeSt.""" + super(Bottleneck, self).__init__(inplanes, planes, **kwargs) + + if groups == 1: + width = self.planes + else: + width = math.floor(self.planes * + (base_width / base_channels)) * groups + + self.avg_down_stride = avg_down_stride and self.conv2_stride > 1 + + self.norm1_name, norm1 = build_norm_layer( + self.norm_cfg, width, postfix=1) + self.norm3_name, norm3 = build_norm_layer( + self.norm_cfg, self.planes * self.expansion, postfix=3) + + self.conv1 = build_conv_layer( + self.conv_cfg, + self.inplanes, + width, + kernel_size=1, + stride=self.conv1_stride, + bias=False) + self.add_module(self.norm1_name, norm1) + self.with_modulated_dcn = False + self.conv2 = SplitAttentionConv2d( + width, + width, + kernel_size=3, + stride=1 if self.avg_down_stride else self.conv2_stride, + padding=self.dilation, + dilation=self.dilation, + groups=groups, + radix=radix, + reduction_factor=reduction_factor, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + dcn=self.dcn) + delattr(self, self.norm2_name) + + if self.avg_down_stride: + self.avd_layer = nn.AvgPool2d(3, self.conv2_stride, padding=1) + + self.conv3 = build_conv_layer( + self.conv_cfg, + width, + self.planes * self.expansion, + kernel_size=1, + bias=False) + self.add_module(self.norm3_name, norm3) + + def forward(self, x): + + def _inner_forward(x): + identity = x + + out = self.conv1(x) + out = self.norm1(out) + out = self.relu(out) + + if self.with_plugins: + out = self.forward_plugin(out, self.after_conv1_plugin_names) + + out = self.conv2(out) + + if self.avg_down_stride: + out = self.avd_layer(out) + + if self.with_plugins: + out = self.forward_plugin(out, self.after_conv2_plugin_names) + + out = self.conv3(out) + out = self.norm3(out) + + if self.with_plugins: + out = self.forward_plugin(out, self.after_conv3_plugin_names) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + + return out + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x) + else: + out = _inner_forward(x) + + out = self.relu(out) + + return out + + +@BACKBONES.register_module() +class ResNeSt(ResNetV1d): + """ResNeSt backbone. + + Args: + groups (int): Number of groups of Bottleneck. Default: 1 + base_width (int): Base width of Bottleneck. Default: 4 + radix (int): Radix of SplitAttentionConv2d. Default: 2 + reduction_factor (int): Reduction factor of inter_channels in + SplitAttentionConv2d. Default: 4. + avg_down_stride (bool): Whether to use average pool for stride in + Bottleneck. Default: True. + kwargs (dict): Keyword arguments for ResNet. + """ + + arch_settings = { + 50: (Bottleneck, (3, 4, 6, 3)), + 101: (Bottleneck, (3, 4, 23, 3)), + 152: (Bottleneck, (3, 8, 36, 3)), + 200: (Bottleneck, (3, 24, 36, 3)) + } + + def __init__(self, + groups=1, + base_width=4, + radix=2, + reduction_factor=4, + avg_down_stride=True, + **kwargs): + self.groups = groups + self.base_width = base_width + self.radix = radix + self.reduction_factor = reduction_factor + self.avg_down_stride = avg_down_stride + super(ResNeSt, self).__init__(**kwargs) + + def make_res_layer(self, **kwargs): + """Pack all blocks in a stage into a ``ResLayer``.""" + return ResLayer( + groups=self.groups, + base_width=self.base_width, + base_channels=self.base_channels, + radix=self.radix, + reduction_factor=self.reduction_factor, + avg_down_stride=self.avg_down_stride, + **kwargs) diff --git a/detection/mmdet/models/backbones/resnet.py b/detection/mmdet/models/backbones/resnet.py new file mode 100644 index 0000000..3826815 --- /dev/null +++ b/detection/mmdet/models/backbones/resnet.py @@ -0,0 +1,663 @@ +import torch.nn as nn +import torch.utils.checkpoint as cp +from mmcv.cnn import (build_conv_layer, build_norm_layer, build_plugin_layer, + constant_init, kaiming_init) +from mmcv.runner import load_checkpoint +from torch.nn.modules.batchnorm import _BatchNorm + +from mmdet.utils import get_root_logger +from ..builder import BACKBONES +from ..utils import ResLayer + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, + inplanes, + planes, + stride=1, + dilation=1, + downsample=None, + style='pytorch', + with_cp=False, + conv_cfg=None, + norm_cfg=dict(type='BN'), + dcn=None, + plugins=None): + super(BasicBlock, self).__init__() + assert dcn is None, 'Not implemented yet.' + assert plugins is None, 'Not implemented yet.' + + self.norm1_name, norm1 = build_norm_layer(norm_cfg, planes, postfix=1) + self.norm2_name, norm2 = build_norm_layer(norm_cfg, planes, postfix=2) + + self.conv1 = build_conv_layer( + conv_cfg, + inplanes, + planes, + 3, + stride=stride, + padding=dilation, + dilation=dilation, + bias=False) + self.add_module(self.norm1_name, norm1) + self.conv2 = build_conv_layer( + conv_cfg, planes, planes, 3, padding=1, bias=False) + self.add_module(self.norm2_name, norm2) + + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + self.dilation = dilation + self.with_cp = with_cp + + @property + def norm1(self): + """nn.Module: normalization layer after the first convolution layer""" + return getattr(self, self.norm1_name) + + @property + def norm2(self): + """nn.Module: normalization layer after the second convolution layer""" + return getattr(self, self.norm2_name) + + def forward(self, x): + """Forward function.""" + + def _inner_forward(x): + identity = x + + out = self.conv1(x) + out = self.norm1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.norm2(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + + return out + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x) + else: + out = _inner_forward(x) + + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, + inplanes, + planes, + stride=1, + dilation=1, + downsample=None, + style='pytorch', + with_cp=False, + conv_cfg=None, + norm_cfg=dict(type='BN'), + dcn=None, + plugins=None): + """Bottleneck block for ResNet. + + If style is "pytorch", the stride-two layer is the 3x3 conv layer, if + it is "caffe", the stride-two layer is the first 1x1 conv layer. + """ + super(Bottleneck, self).__init__() + assert style in ['pytorch', 'caffe'] + assert dcn is None or isinstance(dcn, dict) + assert plugins is None or isinstance(plugins, list) + if plugins is not None: + allowed_position = ['after_conv1', 'after_conv2', 'after_conv3'] + assert all(p['position'] in allowed_position for p in plugins) + + self.inplanes = inplanes + self.planes = planes + self.stride = stride + self.dilation = dilation + self.style = style + self.with_cp = with_cp + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.dcn = dcn + self.with_dcn = dcn is not None + self.plugins = plugins + self.with_plugins = plugins is not None + + if self.with_plugins: + # collect plugins for conv1/conv2/conv3 + self.after_conv1_plugins = [ + plugin['cfg'] for plugin in plugins + if plugin['position'] == 'after_conv1' + ] + self.after_conv2_plugins = [ + plugin['cfg'] for plugin in plugins + if plugin['position'] == 'after_conv2' + ] + self.after_conv3_plugins = [ + plugin['cfg'] for plugin in plugins + if plugin['position'] == 'after_conv3' + ] + + if self.style == 'pytorch': + self.conv1_stride = 1 + self.conv2_stride = stride + else: + self.conv1_stride = stride + self.conv2_stride = 1 + + self.norm1_name, norm1 = build_norm_layer(norm_cfg, planes, postfix=1) + self.norm2_name, norm2 = build_norm_layer(norm_cfg, planes, postfix=2) + self.norm3_name, norm3 = build_norm_layer( + norm_cfg, planes * self.expansion, postfix=3) + + self.conv1 = build_conv_layer( + conv_cfg, + inplanes, + planes, + kernel_size=1, + stride=self.conv1_stride, + bias=False) + self.add_module(self.norm1_name, norm1) + fallback_on_stride = False + if self.with_dcn: + fallback_on_stride = dcn.pop('fallback_on_stride', False) + if not self.with_dcn or fallback_on_stride: + self.conv2 = build_conv_layer( + conv_cfg, + planes, + planes, + kernel_size=3, + stride=self.conv2_stride, + padding=dilation, + dilation=dilation, + bias=False) + else: + assert self.conv_cfg is None, 'conv_cfg must be None for DCN' + self.conv2 = build_conv_layer( + dcn, + planes, + planes, + kernel_size=3, + stride=self.conv2_stride, + padding=dilation, + dilation=dilation, + bias=False) + + self.add_module(self.norm2_name, norm2) + self.conv3 = build_conv_layer( + conv_cfg, + planes, + planes * self.expansion, + kernel_size=1, + bias=False) + self.add_module(self.norm3_name, norm3) + + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + + if self.with_plugins: + self.after_conv1_plugin_names = self.make_block_plugins( + planes, self.after_conv1_plugins) + self.after_conv2_plugin_names = self.make_block_plugins( + planes, self.after_conv2_plugins) + self.after_conv3_plugin_names = self.make_block_plugins( + planes * self.expansion, self.after_conv3_plugins) + + def make_block_plugins(self, in_channels, plugins): + """make plugins for block. + + Args: + in_channels (int): Input channels of plugin. + plugins (list[dict]): List of plugins cfg to build. + + Returns: + list[str]: List of the names of plugin. + """ + assert isinstance(plugins, list) + plugin_names = [] + for plugin in plugins: + plugin = plugin.copy() + name, layer = build_plugin_layer( + plugin, + in_channels=in_channels, + postfix=plugin.pop('postfix', '')) + assert not hasattr(self, name), f'duplicate plugin {name}' + self.add_module(name, layer) + plugin_names.append(name) + return plugin_names + + def forward_plugin(self, x, plugin_names): + out = x + for name in plugin_names: + out = getattr(self, name)(x) + return out + + @property + def norm1(self): + """nn.Module: normalization layer after the first convolution layer""" + return getattr(self, self.norm1_name) + + @property + def norm2(self): + """nn.Module: normalization layer after the second convolution layer""" + return getattr(self, self.norm2_name) + + @property + def norm3(self): + """nn.Module: normalization layer after the third convolution layer""" + return getattr(self, self.norm3_name) + + def forward(self, x): + """Forward function.""" + + def _inner_forward(x): + identity = x + out = self.conv1(x) + out = self.norm1(out) + out = self.relu(out) + + if self.with_plugins: + out = self.forward_plugin(out, self.after_conv1_plugin_names) + + out = self.conv2(out) + out = self.norm2(out) + out = self.relu(out) + + if self.with_plugins: + out = self.forward_plugin(out, self.after_conv2_plugin_names) + + out = self.conv3(out) + out = self.norm3(out) + + if self.with_plugins: + out = self.forward_plugin(out, self.after_conv3_plugin_names) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + + return out + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x) + else: + out = _inner_forward(x) + + out = self.relu(out) + + return out + + +@BACKBONES.register_module() +class ResNet(nn.Module): + """ResNet backbone. + + Args: + depth (int): Depth of resnet, from {18, 34, 50, 101, 152}. + stem_channels (int | None): Number of stem channels. If not specified, + it will be the same as `base_channels`. Default: None. + base_channels (int): Number of base channels of res layer. Default: 64. + in_channels (int): Number of input image channels. Default: 3. + num_stages (int): Resnet stages. Default: 4. + strides (Sequence[int]): Strides of the first block of each stage. + dilations (Sequence[int]): Dilation of each stage. + out_indices (Sequence[int]): Output from which stages. + style (str): `pytorch` or `caffe`. If set to "pytorch", the stride-two + layer is the 3x3 conv layer, otherwise the stride-two layer is + the first 1x1 conv layer. + deep_stem (bool): Replace 7x7 conv in input stem with 3 3x3 conv + avg_down (bool): Use AvgPool instead of stride conv when + downsampling in the bottleneck. + frozen_stages (int): Stages to be frozen (stop grad and set eval mode). + -1 means not freezing any parameters. + norm_cfg (dict): Dictionary to construct and config norm layer. + norm_eval (bool): Whether to set norm layers to eval mode, namely, + freeze running stats (mean and var). Note: Effect on Batch Norm + and its variants only. + plugins (list[dict]): List of plugins for stages, each dict contains: + + - cfg (dict, required): Cfg dict to build plugin. + - position (str, required): Position inside block to insert + plugin, options are 'after_conv1', 'after_conv2', 'after_conv3'. + - stages (tuple[bool], optional): Stages to apply plugin, length + should be same as 'num_stages'. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. + zero_init_residual (bool): Whether to use zero init for last norm layer + in resblocks to let them behave as identity. + + Example: + >>> from mmdet.models import ResNet + >>> import torch + >>> self = ResNet(depth=18) + >>> self.eval() + >>> inputs = torch.rand(1, 3, 32, 32) + >>> level_outputs = self.forward(inputs) + >>> for level_out in level_outputs: + ... print(tuple(level_out.shape)) + (1, 64, 8, 8) + (1, 128, 4, 4) + (1, 256, 2, 2) + (1, 512, 1, 1) + """ + + arch_settings = { + 18: (BasicBlock, (2, 2, 2, 2)), + 34: (BasicBlock, (3, 4, 6, 3)), + 50: (Bottleneck, (3, 4, 6, 3)), + 101: (Bottleneck, (3, 4, 23, 3)), + 152: (Bottleneck, (3, 8, 36, 3)) + } + + def __init__(self, + depth, + in_channels=3, + stem_channels=None, + base_channels=64, + num_stages=4, + strides=(1, 2, 2, 2), + dilations=(1, 1, 1, 1), + out_indices=(0, 1, 2, 3), + style='pytorch', + deep_stem=False, + avg_down=False, + frozen_stages=-1, + conv_cfg=None, + norm_cfg=dict(type='BN', requires_grad=True), + norm_eval=True, + dcn=None, + stage_with_dcn=(False, False, False, False), + plugins=None, + with_cp=False, + zero_init_residual=True): + super(ResNet, self).__init__() + if depth not in self.arch_settings: + raise KeyError(f'invalid depth {depth} for resnet') + self.depth = depth + if stem_channels is None: + stem_channels = base_channels + self.stem_channels = stem_channels + self.base_channels = base_channels + self.num_stages = num_stages + assert num_stages >= 1 and num_stages <= 4 + self.strides = strides + self.dilations = dilations + assert len(strides) == len(dilations) == num_stages + self.out_indices = out_indices + assert max(out_indices) < num_stages + self.style = style + self.deep_stem = deep_stem + self.avg_down = avg_down + self.frozen_stages = frozen_stages + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.with_cp = with_cp + self.norm_eval = norm_eval + self.dcn = dcn + self.stage_with_dcn = stage_with_dcn + if dcn is not None: + assert len(stage_with_dcn) == num_stages + self.plugins = plugins + self.zero_init_residual = zero_init_residual + self.block, stage_blocks = self.arch_settings[depth] + self.stage_blocks = stage_blocks[:num_stages] + self.inplanes = stem_channels + + self._make_stem_layer(in_channels, stem_channels) + + self.res_layers = [] + for i, num_blocks in enumerate(self.stage_blocks): + stride = strides[i] + dilation = dilations[i] + dcn = self.dcn if self.stage_with_dcn[i] else None + if plugins is not None: + stage_plugins = self.make_stage_plugins(plugins, i) + else: + stage_plugins = None + planes = base_channels * 2**i + res_layer = self.make_res_layer( + block=self.block, + inplanes=self.inplanes, + planes=planes, + num_blocks=num_blocks, + stride=stride, + dilation=dilation, + style=self.style, + avg_down=self.avg_down, + with_cp=with_cp, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + dcn=dcn, + plugins=stage_plugins) + self.inplanes = planes * self.block.expansion + layer_name = f'layer{i + 1}' + self.add_module(layer_name, res_layer) + self.res_layers.append(layer_name) + + self._freeze_stages() + + self.feat_dim = self.block.expansion * base_channels * 2**( + len(self.stage_blocks) - 1) + + def make_stage_plugins(self, plugins, stage_idx): + """Make plugins for ResNet ``stage_idx`` th stage. + + Currently we support to insert ``context_block``, + ``empirical_attention_block``, ``nonlocal_block`` into the backbone + like ResNet/ResNeXt. They could be inserted after conv1/conv2/conv3 of + Bottleneck. + + An example of plugins format could be: + + Examples: + >>> plugins=[ + ... dict(cfg=dict(type='xxx', arg1='xxx'), + ... stages=(False, True, True, True), + ... position='after_conv2'), + ... dict(cfg=dict(type='yyy'), + ... stages=(True, True, True, True), + ... position='after_conv3'), + ... dict(cfg=dict(type='zzz', postfix='1'), + ... stages=(True, True, True, True), + ... position='after_conv3'), + ... dict(cfg=dict(type='zzz', postfix='2'), + ... stages=(True, True, True, True), + ... position='after_conv3') + ... ] + >>> self = ResNet(depth=18) + >>> stage_plugins = self.make_stage_plugins(plugins, 0) + >>> assert len(stage_plugins) == 3 + + Suppose ``stage_idx=0``, the structure of blocks in the stage would be: + + .. code-block:: none + + conv1-> conv2->conv3->yyy->zzz1->zzz2 + + Suppose 'stage_idx=1', the structure of blocks in the stage would be: + + .. code-block:: none + + conv1-> conv2->xxx->conv3->yyy->zzz1->zzz2 + + If stages is missing, the plugin would be applied to all stages. + + Args: + plugins (list[dict]): List of plugins cfg to build. The postfix is + required if multiple same type plugins are inserted. + stage_idx (int): Index of stage to build + + Returns: + list[dict]: Plugins for current stage + """ + stage_plugins = [] + for plugin in plugins: + plugin = plugin.copy() + stages = plugin.pop('stages', None) + assert stages is None or len(stages) == self.num_stages + # whether to insert plugin into current stage + if stages is None or stages[stage_idx]: + stage_plugins.append(plugin) + + return stage_plugins + + def make_res_layer(self, **kwargs): + """Pack all blocks in a stage into a ``ResLayer``.""" + return ResLayer(**kwargs) + + @property + def norm1(self): + """nn.Module: the normalization layer named "norm1" """ + return getattr(self, self.norm1_name) + + def _make_stem_layer(self, in_channels, stem_channels): + if self.deep_stem: + self.stem = nn.Sequential( + build_conv_layer( + self.conv_cfg, + in_channels, + stem_channels // 2, + kernel_size=3, + stride=2, + padding=1, + bias=False), + build_norm_layer(self.norm_cfg, stem_channels // 2)[1], + nn.ReLU(inplace=True), + build_conv_layer( + self.conv_cfg, + stem_channels // 2, + stem_channels // 2, + kernel_size=3, + stride=1, + padding=1, + bias=False), + build_norm_layer(self.norm_cfg, stem_channels // 2)[1], + nn.ReLU(inplace=True), + build_conv_layer( + self.conv_cfg, + stem_channels // 2, + stem_channels, + kernel_size=3, + stride=1, + padding=1, + bias=False), + build_norm_layer(self.norm_cfg, stem_channels)[1], + nn.ReLU(inplace=True)) + else: + self.conv1 = build_conv_layer( + self.conv_cfg, + in_channels, + stem_channels, + kernel_size=7, + stride=2, + padding=3, + bias=False) + self.norm1_name, norm1 = build_norm_layer( + self.norm_cfg, stem_channels, postfix=1) + self.add_module(self.norm1_name, norm1) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + + def _freeze_stages(self): + if self.frozen_stages >= 0: + if self.deep_stem: + self.stem.eval() + for param in self.stem.parameters(): + param.requires_grad = False + else: + self.norm1.eval() + for m in [self.conv1, self.norm1]: + for param in m.parameters(): + param.requires_grad = False + + for i in range(1, self.frozen_stages + 1): + m = getattr(self, f'layer{i}') + m.eval() + for param in m.parameters(): + param.requires_grad = False + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + if isinstance(pretrained, str): + logger = get_root_logger() + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + for m in self.modules(): + if isinstance(m, nn.Conv2d): + kaiming_init(m) + elif isinstance(m, (_BatchNorm, nn.GroupNorm)): + constant_init(m, 1) + + if self.dcn is not None: + for m in self.modules(): + if isinstance(m, Bottleneck) and hasattr( + m.conv2, 'conv_offset'): + constant_init(m.conv2.conv_offset, 0) + + if self.zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + constant_init(m.norm3, 0) + elif isinstance(m, BasicBlock): + constant_init(m.norm2, 0) + else: + raise TypeError('pretrained must be a str or None') + + def forward(self, x): + """Forward function.""" + if self.deep_stem: + x = self.stem(x) + else: + x = self.conv1(x) + x = self.norm1(x) + x = self.relu(x) + x = self.maxpool(x) + outs = [] + for i, layer_name in enumerate(self.res_layers): + res_layer = getattr(self, layer_name) + x = res_layer(x) + if i in self.out_indices: + outs.append(x) + return tuple(outs) + + def train(self, mode=True): + """Convert the model into training mode while keep normalization layer + freezed.""" + super(ResNet, self).train(mode) + self._freeze_stages() + if mode and self.norm_eval: + for m in self.modules(): + # trick: eval have effect on BatchNorm only + if isinstance(m, _BatchNorm): + m.eval() + + +@BACKBONES.register_module() +class ResNetV1d(ResNet): + r"""ResNetV1d variant described in `Bag of Tricks + `_. + + Compared with default ResNet(ResNetV1b), ResNetV1d replaces the 7x7 conv in + the input stem with three 3x3 convs. And in the downsampling block, a 2x2 + avg_pool with stride 2 is added before conv, whose stride is changed to 1. + """ + + def __init__(self, **kwargs): + super(ResNetV1d, self).__init__( + deep_stem=True, avg_down=True, **kwargs) diff --git a/detection/mmdet/models/backbones/resnext.py b/detection/mmdet/models/backbones/resnext.py new file mode 100644 index 0000000..6dbcbd5 --- /dev/null +++ b/detection/mmdet/models/backbones/resnext.py @@ -0,0 +1,153 @@ +import math + +from mmcv.cnn import build_conv_layer, build_norm_layer + +from ..builder import BACKBONES +from ..utils import ResLayer +from .resnet import Bottleneck as _Bottleneck +from .resnet import ResNet + + +class Bottleneck(_Bottleneck): + expansion = 4 + + def __init__(self, + inplanes, + planes, + groups=1, + base_width=4, + base_channels=64, + **kwargs): + """Bottleneck block for ResNeXt. + + If style is "pytorch", the stride-two layer is the 3x3 conv layer, if + it is "caffe", the stride-two layer is the first 1x1 conv layer. + """ + super(Bottleneck, self).__init__(inplanes, planes, **kwargs) + + if groups == 1: + width = self.planes + else: + width = math.floor(self.planes * + (base_width / base_channels)) * groups + + self.norm1_name, norm1 = build_norm_layer( + self.norm_cfg, width, postfix=1) + self.norm2_name, norm2 = build_norm_layer( + self.norm_cfg, width, postfix=2) + self.norm3_name, norm3 = build_norm_layer( + self.norm_cfg, self.planes * self.expansion, postfix=3) + + self.conv1 = build_conv_layer( + self.conv_cfg, + self.inplanes, + width, + kernel_size=1, + stride=self.conv1_stride, + bias=False) + self.add_module(self.norm1_name, norm1) + fallback_on_stride = False + self.with_modulated_dcn = False + if self.with_dcn: + fallback_on_stride = self.dcn.pop('fallback_on_stride', False) + if not self.with_dcn or fallback_on_stride: + self.conv2 = build_conv_layer( + self.conv_cfg, + width, + width, + kernel_size=3, + stride=self.conv2_stride, + padding=self.dilation, + dilation=self.dilation, + groups=groups, + bias=False) + else: + assert self.conv_cfg is None, 'conv_cfg must be None for DCN' + self.conv2 = build_conv_layer( + self.dcn, + width, + width, + kernel_size=3, + stride=self.conv2_stride, + padding=self.dilation, + dilation=self.dilation, + groups=groups, + bias=False) + + self.add_module(self.norm2_name, norm2) + self.conv3 = build_conv_layer( + self.conv_cfg, + width, + self.planes * self.expansion, + kernel_size=1, + bias=False) + self.add_module(self.norm3_name, norm3) + + if self.with_plugins: + self._del_block_plugins(self.after_conv1_plugin_names + + self.after_conv2_plugin_names + + self.after_conv3_plugin_names) + self.after_conv1_plugin_names = self.make_block_plugins( + width, self.after_conv1_plugins) + self.after_conv2_plugin_names = self.make_block_plugins( + width, self.after_conv2_plugins) + self.after_conv3_plugin_names = self.make_block_plugins( + self.planes * self.expansion, self.after_conv3_plugins) + + def _del_block_plugins(self, plugin_names): + """delete plugins for block if exist. + + Args: + plugin_names (list[str]): List of plugins name to delete. + """ + assert isinstance(plugin_names, list) + for plugin_name in plugin_names: + del self._modules[plugin_name] + + +@BACKBONES.register_module() +class ResNeXt(ResNet): + """ResNeXt backbone. + + Args: + depth (int): Depth of resnet, from {18, 34, 50, 101, 152}. + in_channels (int): Number of input image channels. Default: 3. + num_stages (int): Resnet stages. Default: 4. + groups (int): Group of resnext. + base_width (int): Base width of resnext. + strides (Sequence[int]): Strides of the first block of each stage. + dilations (Sequence[int]): Dilation of each stage. + out_indices (Sequence[int]): Output from which stages. + style (str): `pytorch` or `caffe`. If set to "pytorch", the stride-two + layer is the 3x3 conv layer, otherwise the stride-two layer is + the first 1x1 conv layer. + frozen_stages (int): Stages to be frozen (all param fixed). -1 means + not freezing any parameters. + norm_cfg (dict): dictionary to construct and config norm layer. + norm_eval (bool): Whether to set norm layers to eval mode, namely, + freeze running stats (mean and var). Note: Effect on Batch Norm + and its variants only. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. + zero_init_residual (bool): whether to use zero init for last norm layer + in resblocks to let them behave as identity. + """ + + arch_settings = { + 50: (Bottleneck, (3, 4, 6, 3)), + 101: (Bottleneck, (3, 4, 23, 3)), + 152: (Bottleneck, (3, 8, 36, 3)) + } + + def __init__(self, groups=1, base_width=4, **kwargs): + self.groups = groups + self.base_width = base_width + super(ResNeXt, self).__init__(**kwargs) + + def make_res_layer(self, **kwargs): + """Pack all blocks in a stage into a ``ResLayer``""" + return ResLayer( + groups=self.groups, + base_width=self.base_width, + base_channels=self.base_channels, + **kwargs) diff --git a/detection/mmdet/models/backbones/ssd_vgg.py b/detection/mmdet/models/backbones/ssd_vgg.py new file mode 100644 index 0000000..cbc4fbb --- /dev/null +++ b/detection/mmdet/models/backbones/ssd_vgg.py @@ -0,0 +1,169 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import VGG, constant_init, kaiming_init, normal_init, xavier_init +from mmcv.runner import load_checkpoint + +from mmdet.utils import get_root_logger +from ..builder import BACKBONES + + +@BACKBONES.register_module() +class SSDVGG(VGG): + """VGG Backbone network for single-shot-detection. + + Args: + input_size (int): width and height of input, from {300, 512}. + depth (int): Depth of vgg, from {11, 13, 16, 19}. + out_indices (Sequence[int]): Output from which stages. + + Example: + >>> self = SSDVGG(input_size=300, depth=11) + >>> self.eval() + >>> inputs = torch.rand(1, 3, 300, 300) + >>> level_outputs = self.forward(inputs) + >>> for level_out in level_outputs: + ... print(tuple(level_out.shape)) + (1, 1024, 19, 19) + (1, 512, 10, 10) + (1, 256, 5, 5) + (1, 256, 3, 3) + (1, 256, 1, 1) + """ + extra_setting = { + 300: (256, 'S', 512, 128, 'S', 256, 128, 256, 128, 256), + 512: (256, 'S', 512, 128, 'S', 256, 128, 'S', 256, 128, 'S', 256, 128), + } + + def __init__(self, + input_size, + depth, + with_last_pool=False, + ceil_mode=True, + out_indices=(3, 4), + out_feature_indices=(22, 34), + l2_norm_scale=20.): + # TODO: in_channels for mmcv.VGG + super(SSDVGG, self).__init__( + depth, + with_last_pool=with_last_pool, + ceil_mode=ceil_mode, + out_indices=out_indices) + assert input_size in (300, 512) + self.input_size = input_size + + self.features.add_module( + str(len(self.features)), + nn.MaxPool2d(kernel_size=3, stride=1, padding=1)) + self.features.add_module( + str(len(self.features)), + nn.Conv2d(512, 1024, kernel_size=3, padding=6, dilation=6)) + self.features.add_module( + str(len(self.features)), nn.ReLU(inplace=True)) + self.features.add_module( + str(len(self.features)), nn.Conv2d(1024, 1024, kernel_size=1)) + self.features.add_module( + str(len(self.features)), nn.ReLU(inplace=True)) + self.out_feature_indices = out_feature_indices + + self.inplanes = 1024 + self.extra = self._make_extra_layers(self.extra_setting[input_size]) + self.l2_norm = L2Norm( + self.features[out_feature_indices[0] - 1].out_channels, + l2_norm_scale) + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + if isinstance(pretrained, str): + logger = get_root_logger() + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + for m in self.features.modules(): + if isinstance(m, nn.Conv2d): + kaiming_init(m) + elif isinstance(m, nn.BatchNorm2d): + constant_init(m, 1) + elif isinstance(m, nn.Linear): + normal_init(m, std=0.01) + else: + raise TypeError('pretrained must be a str or None') + + for m in self.extra.modules(): + if isinstance(m, nn.Conv2d): + xavier_init(m, distribution='uniform') + + constant_init(self.l2_norm, self.l2_norm.scale) + + def forward(self, x): + """Forward function.""" + outs = [] + for i, layer in enumerate(self.features): + x = layer(x) + if i in self.out_feature_indices: + outs.append(x) + for i, layer in enumerate(self.extra): + x = F.relu(layer(x), inplace=True) + if i % 2 == 1: + outs.append(x) + outs[0] = self.l2_norm(outs[0]) + if len(outs) == 1: + return outs[0] + else: + return tuple(outs) + + def _make_extra_layers(self, outplanes): + layers = [] + kernel_sizes = (1, 3) + num_layers = 0 + outplane = None + for i in range(len(outplanes)): + if self.inplanes == 'S': + self.inplanes = outplane + continue + k = kernel_sizes[num_layers % 2] + if outplanes[i] == 'S': + outplane = outplanes[i + 1] + conv = nn.Conv2d( + self.inplanes, outplane, k, stride=2, padding=1) + else: + outplane = outplanes[i] + conv = nn.Conv2d( + self.inplanes, outplane, k, stride=1, padding=0) + layers.append(conv) + self.inplanes = outplanes[i] + num_layers += 1 + if self.input_size == 512: + layers.append(nn.Conv2d(self.inplanes, 256, 4, padding=1)) + + return nn.Sequential(*layers) + + +class L2Norm(nn.Module): + + def __init__(self, n_dims, scale=20., eps=1e-10): + """L2 normalization layer. + + Args: + n_dims (int): Number of dimensions to be normalized + scale (float, optional): Defaults to 20.. + eps (float, optional): Used to avoid division by zero. + Defaults to 1e-10. + """ + super(L2Norm, self).__init__() + self.n_dims = n_dims + self.weight = nn.Parameter(torch.Tensor(self.n_dims)) + self.eps = eps + self.scale = scale + + def forward(self, x): + """Forward function.""" + # normalization layer convert to FP32 in FP16 training + x_float = x.float() + norm = x_float.pow(2).sum(1, keepdim=True).sqrt() + self.eps + return (self.weight[None, :, None, None].float().expand_as(x_float) * + x_float / norm).type_as(x) diff --git a/detection/mmdet/models/backbones/trident_resnet.py b/detection/mmdet/models/backbones/trident_resnet.py new file mode 100644 index 0000000..e610013 --- /dev/null +++ b/detection/mmdet/models/backbones/trident_resnet.py @@ -0,0 +1,292 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as cp +from mmcv.cnn import build_conv_layer, build_norm_layer, kaiming_init +from torch.nn.modules.utils import _pair + +from mmdet.models.backbones.resnet import Bottleneck, ResNet +from mmdet.models.builder import BACKBONES + + +class TridentConv(nn.Module): + """Trident Convolution Module. + + Args: + in_channels (int): Number of channels in input. + out_channels (int): Number of channels in output. + kernel_size (int): Size of convolution kernel. + stride (int, optional): Convolution stride. Default: 1. + trident_dilations (tuple[int, int, int], optional): Dilations of + different trident branch. Default: (1, 2, 3). + test_branch_idx (int, optional): In inference, all 3 branches will + be used if `test_branch_idx==-1`, otherwise only branch with + index `test_branch_idx` will be used. Default: 1. + bias (bool, optional): Whether to use bias in convolution or not. + Default: False. + """ + + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + trident_dilations=(1, 2, 3), + test_branch_idx=1, + bias=False): + super(TridentConv, self).__init__() + self.num_branch = len(trident_dilations) + self.with_bias = bias + self.test_branch_idx = test_branch_idx + self.stride = _pair(stride) + self.kernel_size = _pair(kernel_size) + self.paddings = _pair(trident_dilations) + self.dilations = trident_dilations + self.in_channels = in_channels + self.out_channels = out_channels + self.bias = bias + + self.weight = nn.Parameter( + torch.Tensor(out_channels, in_channels, *self.kernel_size)) + if bias: + self.bias = nn.Parameter(torch.Tensor(out_channels)) + else: + self.bias = None + self.init_weights() + + def init_weights(self): + kaiming_init(self, distribution='uniform', mode='fan_in') + + def extra_repr(self): + tmpstr = f'in_channels={self.in_channels}' + tmpstr += f', out_channels={self.out_channels}' + tmpstr += f', kernel_size={self.kernel_size}' + tmpstr += f', num_branch={self.num_branch}' + tmpstr += f', test_branch_idx={self.test_branch_idx}' + tmpstr += f', stride={self.stride}' + tmpstr += f', paddings={self.paddings}' + tmpstr += f', dilations={self.dilations}' + tmpstr += f', bias={self.bias}' + return tmpstr + + def forward(self, inputs): + if self.training or self.test_branch_idx == -1: + outputs = [ + F.conv2d(input, self.weight, self.bias, self.stride, padding, + dilation) for input, dilation, padding in zip( + inputs, self.dilations, self.paddings) + ] + else: + assert len(inputs) == 1 + outputs = [ + F.conv2d(inputs[0], self.weight, self.bias, self.stride, + self.paddings[self.test_branch_idx], + self.dilations[self.test_branch_idx]) + ] + + return outputs + + +# Since TridentNet is defined over ResNet50 and ResNet101, here we +# only support TridentBottleneckBlock. +class TridentBottleneck(Bottleneck): + """BottleBlock for TridentResNet. + + Args: + trident_dilations (tuple[int, int, int]): Dilations of different + trident branch. + test_branch_idx (int): In inference, all 3 branches will be used + if `test_branch_idx==-1`, otherwise only branch with index + `test_branch_idx` will be used. + concat_output (bool): Whether to concat the output list to a Tensor. + `True` only in the last Block. + """ + + def __init__(self, trident_dilations, test_branch_idx, concat_output, + **kwargs): + + super(TridentBottleneck, self).__init__(**kwargs) + self.trident_dilations = trident_dilations + self.num_branch = len(trident_dilations) + self.concat_output = concat_output + self.test_branch_idx = test_branch_idx + self.conv2 = TridentConv( + self.planes, + self.planes, + kernel_size=3, + stride=self.conv2_stride, + bias=False, + trident_dilations=self.trident_dilations, + test_branch_idx=test_branch_idx) + + def forward(self, x): + + def _inner_forward(x): + num_branch = ( + self.num_branch + if self.training or self.test_branch_idx == -1 else 1) + identity = x + if not isinstance(x, list): + x = (x, ) * num_branch + identity = x + if self.downsample is not None: + identity = [self.downsample(b) for b in x] + + out = [self.conv1(b) for b in x] + out = [self.norm1(b) for b in out] + out = [self.relu(b) for b in out] + + if self.with_plugins: + for k in range(len(out)): + out[k] = self.forward_plugin(out[k], + self.after_conv1_plugin_names) + + out = self.conv2(out) + out = [self.norm2(b) for b in out] + out = [self.relu(b) for b in out] + if self.with_plugins: + for k in range(len(out)): + out[k] = self.forward_plugin(out[k], + self.after_conv2_plugin_names) + + out = [self.conv3(b) for b in out] + out = [self.norm3(b) for b in out] + + if self.with_plugins: + for k in range(len(out)): + out[k] = self.forward_plugin(out[k], + self.after_conv3_plugin_names) + + out = [ + out_b + identity_b for out_b, identity_b in zip(out, identity) + ] + return out + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x) + else: + out = _inner_forward(x) + + out = [self.relu(b) for b in out] + if self.concat_output: + out = torch.cat(out, dim=0) + return out + + +def make_trident_res_layer(block, + inplanes, + planes, + num_blocks, + stride=1, + trident_dilations=(1, 2, 3), + style='pytorch', + with_cp=False, + conv_cfg=None, + norm_cfg=dict(type='BN'), + dcn=None, + plugins=None, + test_branch_idx=-1): + """Build Trident Res Layers.""" + + downsample = None + if stride != 1 or inplanes != planes * block.expansion: + downsample = [] + conv_stride = stride + downsample.extend([ + build_conv_layer( + conv_cfg, + inplanes, + planes * block.expansion, + kernel_size=1, + stride=conv_stride, + bias=False), + build_norm_layer(norm_cfg, planes * block.expansion)[1] + ]) + downsample = nn.Sequential(*downsample) + + layers = [] + for i in range(num_blocks): + layers.append( + block( + inplanes=inplanes, + planes=planes, + stride=stride if i == 0 else 1, + trident_dilations=trident_dilations, + downsample=downsample if i == 0 else None, + style=style, + with_cp=with_cp, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + dcn=dcn, + plugins=plugins, + test_branch_idx=test_branch_idx, + concat_output=True if i == num_blocks - 1 else False)) + inplanes = planes * block.expansion + return nn.Sequential(*layers) + + +@BACKBONES.register_module() +class TridentResNet(ResNet): + """The stem layer, stage 1 and stage 2 in Trident ResNet are identical to + ResNet, while in stage 3, Trident BottleBlock is utilized to replace the + normal BottleBlock to yield trident output. Different branch shares the + convolution weight but uses different dilations to achieve multi-scale + output. + + / stage3(b0) \ + x - stem - stage1 - stage2 - stage3(b1) - output + \ stage3(b2) / + + Args: + depth (int): Depth of resnet, from {50, 101, 152}. + num_branch (int): Number of branches in TridentNet. + test_branch_idx (int): In inference, all 3 branches will be used + if `test_branch_idx==-1`, otherwise only branch with index + `test_branch_idx` will be used. + trident_dilations (tuple[int]): Dilations of different trident branch. + len(trident_dilations) should be equal to num_branch. + """ # noqa + + def __init__(self, depth, num_branch, test_branch_idx, trident_dilations, + **kwargs): + + assert num_branch == len(trident_dilations) + assert depth in (50, 101, 152) + super(TridentResNet, self).__init__(depth, **kwargs) + assert self.num_stages == 3 + self.test_branch_idx = test_branch_idx + self.num_branch = num_branch + + last_stage_idx = self.num_stages - 1 + stride = self.strides[last_stage_idx] + dilation = trident_dilations + dcn = self.dcn if self.stage_with_dcn[last_stage_idx] else None + if self.plugins is not None: + stage_plugins = self.make_stage_plugins(self.plugins, + last_stage_idx) + else: + stage_plugins = None + planes = self.base_channels * 2**last_stage_idx + res_layer = make_trident_res_layer( + TridentBottleneck, + inplanes=(self.block.expansion * self.base_channels * + 2**(last_stage_idx - 1)), + planes=planes, + num_blocks=self.stage_blocks[last_stage_idx], + stride=stride, + trident_dilations=dilation, + style=self.style, + with_cp=self.with_cp, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + dcn=dcn, + plugins=stage_plugins, + test_branch_idx=self.test_branch_idx) + + layer_name = f'layer{last_stage_idx + 1}' + + self.__setattr__(layer_name, res_layer) + self.res_layers.pop(last_stage_idx) + self.res_layers.insert(last_stage_idx, layer_name) + + self._freeze_stages() diff --git a/detection/mmdet/models/builder.py b/detection/mmdet/models/builder.py new file mode 100644 index 0000000..81c927e --- /dev/null +++ b/detection/mmdet/models/builder.py @@ -0,0 +1,77 @@ +import warnings + +from mmcv.utils import Registry, build_from_cfg +from torch import nn + +BACKBONES = Registry('backbone') +NECKS = Registry('neck') +ROI_EXTRACTORS = Registry('roi_extractor') +SHARED_HEADS = Registry('shared_head') +HEADS = Registry('head') +LOSSES = Registry('loss') +DETECTORS = Registry('detector') + + +def build(cfg, registry, default_args=None): + """Build a module. + + Args: + cfg (dict, list[dict]): The config of modules, is is either a dict + or a list of configs. + registry (:obj:`Registry`): A registry the module belongs to. + default_args (dict, optional): Default arguments to build the module. + Defaults to None. + + Returns: + nn.Module: A built nn module. + """ + if isinstance(cfg, list): + modules = [ + build_from_cfg(cfg_, registry, default_args) for cfg_ in cfg + ] + return nn.Sequential(*modules) + else: + return build_from_cfg(cfg, registry, default_args) + + +def build_backbone(cfg): + """Build backbone.""" + return build(cfg, BACKBONES) + + +def build_neck(cfg): + """Build neck.""" + return build(cfg, NECKS) + + +def build_roi_extractor(cfg): + """Build roi extractor.""" + return build(cfg, ROI_EXTRACTORS) + + +def build_shared_head(cfg): + """Build shared head.""" + return build(cfg, SHARED_HEADS) + + +def build_head(cfg): + """Build head.""" + return build(cfg, HEADS) + + +def build_loss(cfg): + """Build loss.""" + return build(cfg, LOSSES) + + +def build_detector(cfg, train_cfg=None, test_cfg=None): + """Build detector.""" + if train_cfg is not None or test_cfg is not None: + warnings.warn( + 'train_cfg and test_cfg is deprecated, ' + 'please specify them in model', UserWarning) + assert cfg.get('train_cfg') is None or train_cfg is None, \ + 'train_cfg specified in both outer field and model field ' + assert cfg.get('test_cfg') is None or test_cfg is None, \ + 'test_cfg specified in both outer field and model field ' + return build(cfg, DETECTORS, dict(train_cfg=train_cfg, test_cfg=test_cfg)) diff --git a/detection/mmdet/models/dense_heads/__init__.py b/detection/mmdet/models/dense_heads/__init__.py new file mode 100644 index 0000000..f004dd9 --- /dev/null +++ b/detection/mmdet/models/dense_heads/__init__.py @@ -0,0 +1,41 @@ +from .anchor_free_head import AnchorFreeHead +from .anchor_head import AnchorHead +from .atss_head import ATSSHead +from .cascade_rpn_head import CascadeRPNHead, StageCascadeRPNHead +from .centripetal_head import CentripetalHead +from .corner_head import CornerHead +from .embedding_rpn_head import EmbeddingRPNHead +from .fcos_head import FCOSHead +from .fovea_head import FoveaHead +from .free_anchor_retina_head import FreeAnchorRetinaHead +from .fsaf_head import FSAFHead +from .ga_retina_head import GARetinaHead +from .ga_rpn_head import GARPNHead +from .gfl_head import GFLHead +from .guided_anchor_head import FeatureAdaption, GuidedAnchorHead +from .ld_head import LDHead +from .nasfcos_head import NASFCOSHead +from .paa_head import PAAHead +from .pisa_retinanet_head import PISARetinaHead +from .pisa_ssd_head import PISASSDHead +from .reppoints_head import RepPointsHead +from .retina_head import RetinaHead +from .retina_sepbn_head import RetinaSepBNHead +from .rpn_head import RPNHead +from .sabl_retina_head import SABLRetinaHead +from .ssd_head import SSDHead +from .transformer_head import TransformerHead +from .vfnet_head import VFNetHead +from .yolact_head import YOLACTHead, YOLACTProtonet, YOLACTSegmHead +from .yolo_head import YOLOV3Head + +__all__ = [ + 'AnchorFreeHead', 'AnchorHead', 'GuidedAnchorHead', 'FeatureAdaption', + 'RPNHead', 'GARPNHead', 'RetinaHead', 'RetinaSepBNHead', 'GARetinaHead', + 'SSDHead', 'FCOSHead', 'RepPointsHead', 'FoveaHead', + 'FreeAnchorRetinaHead', 'ATSSHead', 'FSAFHead', 'NASFCOSHead', + 'PISARetinaHead', 'PISASSDHead', 'GFLHead', 'CornerHead', 'YOLACTHead', + 'YOLACTSegmHead', 'YOLACTProtonet', 'YOLOV3Head', 'PAAHead', + 'SABLRetinaHead', 'CentripetalHead', 'VFNetHead', 'TransformerHead', + 'StageCascadeRPNHead', 'CascadeRPNHead', 'EmbeddingRPNHead', 'LDHead' +] diff --git a/detection/mmdet/models/dense_heads/anchor_free_head.py b/detection/mmdet/models/dense_heads/anchor_free_head.py new file mode 100644 index 0000000..1814a0c --- /dev/null +++ b/detection/mmdet/models/dense_heads/anchor_free_head.py @@ -0,0 +1,340 @@ +from abc import abstractmethod + +import torch +import torch.nn as nn +from mmcv.cnn import ConvModule, bias_init_with_prob, normal_init +from mmcv.runner import force_fp32 + +from mmdet.core import multi_apply +from ..builder import HEADS, build_loss +from .base_dense_head import BaseDenseHead +from .dense_test_mixins import BBoxTestMixin + + +@HEADS.register_module() +class AnchorFreeHead(BaseDenseHead, BBoxTestMixin): + """Anchor-free head (FCOS, Fovea, RepPoints, etc.). + + Args: + num_classes (int): Number of categories excluding the background + category. + in_channels (int): Number of channels in the input feature map. + feat_channels (int): Number of hidden channels. Used in child classes. + stacked_convs (int): Number of stacking convs of the head. + strides (tuple): Downsample factor of each feature map. + dcn_on_last_conv (bool): If true, use dcn in the last layer of + towers. Default: False. + conv_bias (bool | str): If specified as `auto`, it will be decided by + the norm_cfg. Bias of conv will be set as True if `norm_cfg` is + None, otherwise False. Default: "auto". + loss_cls (dict): Config of classification loss. + loss_bbox (dict): Config of localization loss. + conv_cfg (dict): Config dict for convolution layer. Default: None. + norm_cfg (dict): Config dict for normalization layer. Default: None. + train_cfg (dict): Training config of anchor head. + test_cfg (dict): Testing config of anchor head. + """ # noqa: W605 + + _version = 1 + + def __init__(self, + num_classes, + in_channels, + feat_channels=256, + stacked_convs=4, + strides=(4, 8, 16, 32, 64), + dcn_on_last_conv=False, + conv_bias='auto', + loss_cls=dict( + type='FocalLoss', + use_sigmoid=True, + gamma=2.0, + alpha=0.25, + loss_weight=1.0), + loss_bbox=dict(type='IoULoss', loss_weight=1.0), + conv_cfg=None, + norm_cfg=None, + train_cfg=None, + test_cfg=None): + super(AnchorFreeHead, self).__init__() + self.num_classes = num_classes + self.cls_out_channels = num_classes + self.in_channels = in_channels + self.feat_channels = feat_channels + self.stacked_convs = stacked_convs + self.strides = strides + self.dcn_on_last_conv = dcn_on_last_conv + assert conv_bias == 'auto' or isinstance(conv_bias, bool) + self.conv_bias = conv_bias + self.loss_cls = build_loss(loss_cls) + self.loss_bbox = build_loss(loss_bbox) + self.train_cfg = train_cfg + self.test_cfg = test_cfg + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.fp16_enabled = False + + self._init_layers() + + def _init_layers(self): + """Initialize layers of the head.""" + self._init_cls_convs() + self._init_reg_convs() + self._init_predictor() + + def _init_cls_convs(self): + """Initialize classification conv layers of the head.""" + self.cls_convs = nn.ModuleList() + for i in range(self.stacked_convs): + chn = self.in_channels if i == 0 else self.feat_channels + if self.dcn_on_last_conv and i == self.stacked_convs - 1: + conv_cfg = dict(type='DCNv2') + else: + conv_cfg = self.conv_cfg + self.cls_convs.append( + ConvModule( + chn, + self.feat_channels, + 3, + stride=1, + padding=1, + conv_cfg=conv_cfg, + norm_cfg=self.norm_cfg, + bias=self.conv_bias)) + + def _init_reg_convs(self): + """Initialize bbox regression conv layers of the head.""" + self.reg_convs = nn.ModuleList() + for i in range(self.stacked_convs): + chn = self.in_channels if i == 0 else self.feat_channels + if self.dcn_on_last_conv and i == self.stacked_convs - 1: + conv_cfg = dict(type='DCNv2') + else: + conv_cfg = self.conv_cfg + self.reg_convs.append( + ConvModule( + chn, + self.feat_channels, + 3, + stride=1, + padding=1, + conv_cfg=conv_cfg, + norm_cfg=self.norm_cfg, + bias=self.conv_bias)) + + def _init_predictor(self): + """Initialize predictor layers of the head.""" + self.conv_cls = nn.Conv2d( + self.feat_channels, self.cls_out_channels, 3, padding=1) + self.conv_reg = nn.Conv2d(self.feat_channels, 4, 3, padding=1) + + def init_weights(self): + """Initialize weights of the head.""" + for m in self.cls_convs: + if isinstance(m.conv, nn.Conv2d): + normal_init(m.conv, std=0.01) + for m in self.reg_convs: + if isinstance(m.conv, nn.Conv2d): + normal_init(m.conv, std=0.01) + bias_cls = bias_init_with_prob(0.01) + normal_init(self.conv_cls, std=0.01, bias=bias_cls) + normal_init(self.conv_reg, std=0.01) + + def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, + missing_keys, unexpected_keys, error_msgs): + """Hack some keys of the model state dict so that can load checkpoints + of previous version.""" + version = local_metadata.get('version', None) + if version is None: + # the key is different in early versions + # for example, 'fcos_cls' become 'conv_cls' now + bbox_head_keys = [ + k for k in state_dict.keys() if k.startswith(prefix) + ] + ori_predictor_keys = [] + new_predictor_keys = [] + # e.g. 'fcos_cls' or 'fcos_reg' + for key in bbox_head_keys: + ori_predictor_keys.append(key) + key = key.split('.') + conv_name = None + if key[1].endswith('cls'): + conv_name = 'conv_cls' + elif key[1].endswith('reg'): + conv_name = 'conv_reg' + elif key[1].endswith('centerness'): + conv_name = 'conv_centerness' + else: + assert NotImplementedError + if conv_name is not None: + key[1] = conv_name + new_predictor_keys.append('.'.join(key)) + else: + ori_predictor_keys.pop(-1) + for i in range(len(new_predictor_keys)): + state_dict[new_predictor_keys[i]] = state_dict.pop( + ori_predictor_keys[i]) + super()._load_from_state_dict(state_dict, prefix, local_metadata, + strict, missing_keys, unexpected_keys, + error_msgs) + + def forward(self, feats): + """Forward features from the upstream network. + + Args: + feats (tuple[Tensor]): Features from the upstream network, each is + a 4D-tensor. + + Returns: + tuple: Usually contain classification scores and bbox predictions. + cls_scores (list[Tensor]): Box scores for each scale level, + each is a 4D-tensor, the channel number is + num_points * num_classes. + bbox_preds (list[Tensor]): Box energies / deltas for each scale + level, each is a 4D-tensor, the channel number is + num_points * 4. + """ + return multi_apply(self.forward_single, feats)[:2] + + def forward_single(self, x): + """Forward features of a single scale level. + + Args: + x (Tensor): FPN feature maps of the specified stride. + + Returns: + tuple: Scores for each class, bbox predictions, features + after classification and regression conv layers, some + models needs these features like FCOS. + """ + cls_feat = x + reg_feat = x + + for cls_layer in self.cls_convs: + cls_feat = cls_layer(cls_feat) + cls_score = self.conv_cls(cls_feat) + + for reg_layer in self.reg_convs: + reg_feat = reg_layer(reg_feat) + bbox_pred = self.conv_reg(reg_feat) + return cls_score, bbox_pred, cls_feat, reg_feat + + @abstractmethod + @force_fp32(apply_to=('cls_scores', 'bbox_preds')) + def loss(self, + cls_scores, + bbox_preds, + gt_bboxes, + gt_labels, + img_metas, + gt_bboxes_ignore=None): + """Compute loss of the head. + + Args: + cls_scores (list[Tensor]): Box scores for each scale level, + each is a 4D-tensor, the channel number is + num_points * num_classes. + bbox_preds (list[Tensor]): Box energies / deltas for each scale + level, each is a 4D-tensor, the channel number is + num_points * 4. + gt_bboxes (list[Tensor]): Ground truth bboxes for each image with + shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + gt_labels (list[Tensor]): class indices corresponding to each box + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + gt_bboxes_ignore (None | list[Tensor]): specify which bounding + boxes can be ignored when computing the loss. + """ + + raise NotImplementedError + + @abstractmethod + @force_fp32(apply_to=('cls_scores', 'bbox_preds')) + def get_bboxes(self, + cls_scores, + bbox_preds, + img_metas, + cfg=None, + rescale=None): + """Transform network output for a batch into bbox predictions. + + Args: + cls_scores (list[Tensor]): Box scores for each scale level + Has shape (N, num_points * num_classes, H, W) + bbox_preds (list[Tensor]): Box energies / deltas for each scale + level with shape (N, num_points * 4, H, W) + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + cfg (mmcv.Config): Test / postprocessing configuration, + if None, test_cfg would be used + rescale (bool): If True, return boxes in original image space + """ + + raise NotImplementedError + + @abstractmethod + def get_targets(self, points, gt_bboxes_list, gt_labels_list): + """Compute regression, classification and centerness targets for points + in multiple images. + + Args: + points (list[Tensor]): Points of each fpn level, each has shape + (num_points, 2). + gt_bboxes_list (list[Tensor]): Ground truth bboxes of each image, + each has shape (num_gt, 4). + gt_labels_list (list[Tensor]): Ground truth labels of each box, + each has shape (num_gt,). + """ + raise NotImplementedError + + def _get_points_single(self, + featmap_size, + stride, + dtype, + device, + flatten=False): + """Get points of a single scale level.""" + h, w = featmap_size + x_range = torch.arange(w, dtype=dtype, device=device) + y_range = torch.arange(h, dtype=dtype, device=device) + y, x = torch.meshgrid(y_range, x_range) + if flatten: + y = y.flatten() + x = x.flatten() + return y, x + + def get_points(self, featmap_sizes, dtype, device, flatten=False): + """Get points according to feature map sizes. + + Args: + featmap_sizes (list[tuple]): Multi-level feature map sizes. + dtype (torch.dtype): Type of points. + device (torch.device): Device of points. + + Returns: + tuple: points of each image. + """ + mlvl_points = [] + for i in range(len(featmap_sizes)): + mlvl_points.append( + self._get_points_single(featmap_sizes[i], self.strides[i], + dtype, device, flatten)) + return mlvl_points + + def aug_test(self, feats, img_metas, rescale=False): + """Test function with test time augmentation. + + Args: + feats (list[Tensor]): the outer list indicates test-time + augmentations and inner Tensor should have a shape NxCxHxW, + which contains features for all images in the batch. + img_metas (list[list[dict]]): the outer list indicates test-time + augs (multiscale, flip, etc.) and the inner list indicates + images in a batch. each dict has image information. + rescale (bool, optional): Whether to rescale the results. + Defaults to False. + + Returns: + list[ndarray]: bbox results of each class + """ + return self.aug_test_bboxes(feats, img_metas, rescale=rescale) diff --git a/detection/mmdet/models/dense_heads/anchor_head.py b/detection/mmdet/models/dense_heads/anchor_head.py new file mode 100644 index 0000000..eea7352 --- /dev/null +++ b/detection/mmdet/models/dense_heads/anchor_head.py @@ -0,0 +1,751 @@ +import torch +import torch.nn as nn +from mmcv.cnn import normal_init +from mmcv.runner import force_fp32 + +from mmdet.core import (anchor_inside_flags, build_anchor_generator, + build_assigner, build_bbox_coder, build_sampler, + images_to_levels, multi_apply, multiclass_nms, unmap) +from ..builder import HEADS, build_loss +from .base_dense_head import BaseDenseHead +from .dense_test_mixins import BBoxTestMixin + + +@HEADS.register_module() +class AnchorHead(BaseDenseHead, BBoxTestMixin): + """Anchor-based head (RPN, RetinaNet, SSD, etc.). + + Args: + num_classes (int): Number of categories excluding the background + category. + in_channels (int): Number of channels in the input feature map. + feat_channels (int): Number of hidden channels. Used in child classes. + anchor_generator (dict): Config dict for anchor generator + bbox_coder (dict): Config of bounding box coder. + reg_decoded_bbox (bool): If true, the regression loss would be + applied directly on decoded bounding boxes, converting both + the predicted boxes and regression targets to absolute + coordinates format. Default False. It should be `True` when + using `IoULoss`, `GIoULoss`, or `DIoULoss` in the bbox head. + loss_cls (dict): Config of classification loss. + loss_bbox (dict): Config of localization loss. + train_cfg (dict): Training config of anchor head. + test_cfg (dict): Testing config of anchor head. + """ # noqa: W605 + + def __init__(self, + num_classes, + in_channels, + feat_channels=256, + anchor_generator=dict( + type='AnchorGenerator', + scales=[8, 16, 32], + ratios=[0.5, 1.0, 2.0], + strides=[4, 8, 16, 32, 64]), + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + clip_border=True, + target_means=(.0, .0, .0, .0), + target_stds=(1.0, 1.0, 1.0, 1.0)), + reg_decoded_bbox=False, + loss_cls=dict( + type='CrossEntropyLoss', + use_sigmoid=True, + loss_weight=1.0), + loss_bbox=dict( + type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0), + train_cfg=None, + test_cfg=None): + super(AnchorHead, self).__init__() + self.in_channels = in_channels + self.num_classes = num_classes + self.feat_channels = feat_channels + self.use_sigmoid_cls = loss_cls.get('use_sigmoid', False) + # TODO better way to determine whether sample or not + self.sampling = loss_cls['type'] not in [ + 'FocalLoss', 'GHMC', 'QualityFocalLoss' + ] + if self.use_sigmoid_cls: + self.cls_out_channels = num_classes + else: + self.cls_out_channels = num_classes + 1 + + if self.cls_out_channels <= 0: + raise ValueError(f'num_classes={num_classes} is too small') + self.reg_decoded_bbox = reg_decoded_bbox + + self.bbox_coder = build_bbox_coder(bbox_coder) + self.loss_cls = build_loss(loss_cls) + self.loss_bbox = build_loss(loss_bbox) + self.train_cfg = train_cfg + self.test_cfg = test_cfg + if self.train_cfg: + self.assigner = build_assigner(self.train_cfg.assigner) + # use PseudoSampler when sampling is False + if self.sampling and hasattr(self.train_cfg, 'sampler'): + sampler_cfg = self.train_cfg.sampler + else: + sampler_cfg = dict(type='PseudoSampler') + self.sampler = build_sampler(sampler_cfg, context=self) + self.fp16_enabled = False + + self.anchor_generator = build_anchor_generator(anchor_generator) + # usually the numbers of anchors for each level are the same + # except SSD detectors + self.num_anchors = self.anchor_generator.num_base_anchors[0] + self._init_layers() + + def _init_layers(self): + """Initialize layers of the head.""" + self.conv_cls = nn.Conv2d(self.in_channels, + self.num_anchors * self.cls_out_channels, 1) + self.conv_reg = nn.Conv2d(self.in_channels, self.num_anchors * 4, 1) + + def init_weights(self): + """Initialize weights of the head.""" + normal_init(self.conv_cls, std=0.01) + normal_init(self.conv_reg, std=0.01) + + def forward_single(self, x): + """Forward feature of a single scale level. + + Args: + x (Tensor): Features of a single scale level. + + Returns: + tuple: + cls_score (Tensor): Cls scores for a single scale level \ + the channels number is num_anchors * num_classes. + bbox_pred (Tensor): Box energies / deltas for a single scale \ + level, the channels number is num_anchors * 4. + """ + cls_score = self.conv_cls(x) + bbox_pred = self.conv_reg(x) + return cls_score, bbox_pred + + def forward(self, feats): + """Forward features from the upstream network. + + Args: + feats (tuple[Tensor]): Features from the upstream network, each is + a 4D-tensor. + + Returns: + tuple: A tuple of classification scores and bbox prediction. + + - cls_scores (list[Tensor]): Classification scores for all \ + scale levels, each is a 4D-tensor, the channels number \ + is num_anchors * num_classes. + - bbox_preds (list[Tensor]): Box energies / deltas for all \ + scale levels, each is a 4D-tensor, the channels number \ + is num_anchors * 4. + """ + return multi_apply(self.forward_single, feats) + + def get_anchors(self, featmap_sizes, img_metas, device='cuda'): + """Get anchors according to feature map sizes. + + Args: + featmap_sizes (list[tuple]): Multi-level feature map sizes. + img_metas (list[dict]): Image meta info. + device (torch.device | str): Device for returned tensors + + Returns: + tuple: + anchor_list (list[Tensor]): Anchors of each image. + valid_flag_list (list[Tensor]): Valid flags of each image. + """ + num_imgs = len(img_metas) + + # since feature map sizes of all images are the same, we only compute + # anchors for one time + multi_level_anchors = self.anchor_generator.grid_anchors( + featmap_sizes, device) + anchor_list = [multi_level_anchors for _ in range(num_imgs)] + + # for each image, we compute valid flags of multi level anchors + valid_flag_list = [] + for img_id, img_meta in enumerate(img_metas): + multi_level_flags = self.anchor_generator.valid_flags( + featmap_sizes, img_meta['pad_shape'], device) + valid_flag_list.append(multi_level_flags) + + return anchor_list, valid_flag_list + + def _get_targets_single(self, + flat_anchors, + valid_flags, + gt_bboxes, + gt_bboxes_ignore, + gt_labels, + img_meta, + label_channels=1, + unmap_outputs=True): + """Compute regression and classification targets for anchors in a + single image. + + Args: + flat_anchors (Tensor): Multi-level anchors of the image, which are + concatenated into a single tensor of shape (num_anchors ,4) + valid_flags (Tensor): Multi level valid flags of the image, + which are concatenated into a single tensor of + shape (num_anchors,). + gt_bboxes (Tensor): Ground truth bboxes of the image, + shape (num_gts, 4). + gt_bboxes_ignore (Tensor): Ground truth bboxes to be + ignored, shape (num_ignored_gts, 4). + img_meta (dict): Meta info of the image. + gt_labels (Tensor): Ground truth labels of each box, + shape (num_gts,). + label_channels (int): Channel of label. + unmap_outputs (bool): Whether to map outputs back to the original + set of anchors. + + Returns: + tuple: + labels_list (list[Tensor]): Labels of each level + label_weights_list (list[Tensor]): Label weights of each level + bbox_targets_list (list[Tensor]): BBox targets of each level + bbox_weights_list (list[Tensor]): BBox weights of each level + num_total_pos (int): Number of positive samples in all images + num_total_neg (int): Number of negative samples in all images + """ + inside_flags = anchor_inside_flags(flat_anchors, valid_flags, + img_meta['img_shape'][:2], + self.train_cfg.allowed_border) + if not inside_flags.any(): + return (None, ) * 7 + # assign gt and sample anchors + anchors = flat_anchors[inside_flags, :] + + assign_result = self.assigner.assign( + anchors, gt_bboxes, gt_bboxes_ignore, + None if self.sampling else gt_labels) + sampling_result = self.sampler.sample(assign_result, anchors, + gt_bboxes) + + num_valid_anchors = anchors.shape[0] + bbox_targets = torch.zeros_like(anchors) + bbox_weights = torch.zeros_like(anchors) + labels = anchors.new_full((num_valid_anchors, ), + self.num_classes, + dtype=torch.long) + label_weights = anchors.new_zeros(num_valid_anchors, dtype=torch.float) + + pos_inds = sampling_result.pos_inds + neg_inds = sampling_result.neg_inds + if len(pos_inds) > 0: + if not self.reg_decoded_bbox: + pos_bbox_targets = self.bbox_coder.encode( + sampling_result.pos_bboxes, sampling_result.pos_gt_bboxes) + else: + pos_bbox_targets = sampling_result.pos_gt_bboxes + bbox_targets[pos_inds, :] = pos_bbox_targets + bbox_weights[pos_inds, :] = 1.0 + if gt_labels is None: + # Only rpn gives gt_labels as None + # Foreground is the first class since v2.5.0 + labels[pos_inds] = 0 + else: + labels[pos_inds] = gt_labels[ + sampling_result.pos_assigned_gt_inds] + if self.train_cfg.pos_weight <= 0: + label_weights[pos_inds] = 1.0 + else: + label_weights[pos_inds] = self.train_cfg.pos_weight + if len(neg_inds) > 0: + label_weights[neg_inds] = 1.0 + + # map up to original set of anchors + if unmap_outputs: + num_total_anchors = flat_anchors.size(0) + labels = unmap( + labels, num_total_anchors, inside_flags, + fill=self.num_classes) # fill bg label + label_weights = unmap(label_weights, num_total_anchors, + inside_flags) + bbox_targets = unmap(bbox_targets, num_total_anchors, inside_flags) + bbox_weights = unmap(bbox_weights, num_total_anchors, inside_flags) + + return (labels, label_weights, bbox_targets, bbox_weights, pos_inds, + neg_inds, sampling_result) + + def get_targets(self, + anchor_list, + valid_flag_list, + gt_bboxes_list, + img_metas, + gt_bboxes_ignore_list=None, + gt_labels_list=None, + label_channels=1, + unmap_outputs=True, + return_sampling_results=False): + """Compute regression and classification targets for anchors in + multiple images. + + Args: + anchor_list (list[list[Tensor]]): Multi level anchors of each + image. The outer list indicates images, and the inner list + corresponds to feature levels of the image. Each element of + the inner list is a tensor of shape (num_anchors, 4). + valid_flag_list (list[list[Tensor]]): Multi level valid flags of + each image. The outer list indicates images, and the inner list + corresponds to feature levels of the image. Each element of + the inner list is a tensor of shape (num_anchors, ) + gt_bboxes_list (list[Tensor]): Ground truth bboxes of each image. + img_metas (list[dict]): Meta info of each image. + gt_bboxes_ignore_list (list[Tensor]): Ground truth bboxes to be + ignored. + gt_labels_list (list[Tensor]): Ground truth labels of each box. + label_channels (int): Channel of label. + unmap_outputs (bool): Whether to map outputs back to the original + set of anchors. + + Returns: + tuple: Usually returns a tuple containing learning targets. + + - labels_list (list[Tensor]): Labels of each level. + - label_weights_list (list[Tensor]): Label weights of each \ + level. + - bbox_targets_list (list[Tensor]): BBox targets of each level. + - bbox_weights_list (list[Tensor]): BBox weights of each level. + - num_total_pos (int): Number of positive samples in all \ + images. + - num_total_neg (int): Number of negative samples in all \ + images. + additional_returns: This function enables user-defined returns from + `self._get_targets_single`. These returns are currently refined + to properties at each feature map (i.e. having HxW dimension). + The results will be concatenated after the end + """ + num_imgs = len(img_metas) + assert len(anchor_list) == len(valid_flag_list) == num_imgs + + # anchor number of multi levels + num_level_anchors = [anchors.size(0) for anchors in anchor_list[0]] + # concat all level anchors to a single tensor + concat_anchor_list = [] + concat_valid_flag_list = [] + for i in range(num_imgs): + assert len(anchor_list[i]) == len(valid_flag_list[i]) + concat_anchor_list.append(torch.cat(anchor_list[i])) + concat_valid_flag_list.append(torch.cat(valid_flag_list[i])) + + # compute targets for each image + if gt_bboxes_ignore_list is None: + gt_bboxes_ignore_list = [None for _ in range(num_imgs)] + if gt_labels_list is None: + gt_labels_list = [None for _ in range(num_imgs)] + results = multi_apply( + self._get_targets_single, + concat_anchor_list, + concat_valid_flag_list, + gt_bboxes_list, + gt_bboxes_ignore_list, + gt_labels_list, + img_metas, + label_channels=label_channels, + unmap_outputs=unmap_outputs) + (all_labels, all_label_weights, all_bbox_targets, all_bbox_weights, + pos_inds_list, neg_inds_list, sampling_results_list) = results[:7] + rest_results = list(results[7:]) # user-added return values + # no valid anchors + if any([labels is None for labels in all_labels]): + return None + # sampled anchors of all images + num_total_pos = sum([max(inds.numel(), 1) for inds in pos_inds_list]) + num_total_neg = sum([max(inds.numel(), 1) for inds in neg_inds_list]) + # split targets to a list w.r.t. multiple levels + labels_list = images_to_levels(all_labels, num_level_anchors) + label_weights_list = images_to_levels(all_label_weights, + num_level_anchors) + bbox_targets_list = images_to_levels(all_bbox_targets, + num_level_anchors) + bbox_weights_list = images_to_levels(all_bbox_weights, + num_level_anchors) + res = (labels_list, label_weights_list, bbox_targets_list, + bbox_weights_list, num_total_pos, num_total_neg) + if return_sampling_results: + res = res + (sampling_results_list, ) + for i, r in enumerate(rest_results): # user-added return values + rest_results[i] = images_to_levels(r, num_level_anchors) + + return res + tuple(rest_results) + + def loss_single(self, cls_score, bbox_pred, anchors, labels, label_weights, + bbox_targets, bbox_weights, num_total_samples): + """Compute loss of a single scale level. + + Args: + cls_score (Tensor): Box scores for each scale level + Has shape (N, num_anchors * num_classes, H, W). + bbox_pred (Tensor): Box energies / deltas for each scale + level with shape (N, num_anchors * 4, H, W). + anchors (Tensor): Box reference for each scale level with shape + (N, num_total_anchors, 4). + labels (Tensor): Labels of each anchors with shape + (N, num_total_anchors). + label_weights (Tensor): Label weights of each anchor with shape + (N, num_total_anchors) + bbox_targets (Tensor): BBox regression targets of each anchor wight + shape (N, num_total_anchors, 4). + bbox_weights (Tensor): BBox regression loss weights of each anchor + with shape (N, num_total_anchors, 4). + num_total_samples (int): If sampling, num total samples equal to + the number of total anchors; Otherwise, it is the number of + positive anchors. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + # classification loss + labels = labels.reshape(-1) + label_weights = label_weights.reshape(-1) + cls_score = cls_score.permute(0, 2, 3, + 1).reshape(-1, self.cls_out_channels) + loss_cls = self.loss_cls( + cls_score, labels, label_weights, avg_factor=num_total_samples) + # regression loss + bbox_targets = bbox_targets.reshape(-1, 4) + bbox_weights = bbox_weights.reshape(-1, 4) + bbox_pred = bbox_pred.permute(0, 2, 3, 1).reshape(-1, 4) + if self.reg_decoded_bbox: + # When the regression loss (e.g. `IouLoss`, `GIouLoss`) + # is applied directly on the decoded bounding boxes, it + # decodes the already encoded coordinates to absolute format. + anchors = anchors.reshape(-1, 4) + bbox_pred = self.bbox_coder.decode(anchors, bbox_pred) + loss_bbox = self.loss_bbox( + bbox_pred, + bbox_targets, + bbox_weights, + avg_factor=num_total_samples) + return loss_cls, loss_bbox + + @force_fp32(apply_to=('cls_scores', 'bbox_preds')) + def loss(self, + cls_scores, + bbox_preds, + gt_bboxes, + gt_labels, + img_metas, + gt_bboxes_ignore=None): + """Compute losses of the head. + + Args: + cls_scores (list[Tensor]): Box scores for each scale level + Has shape (N, num_anchors * num_classes, H, W) + bbox_preds (list[Tensor]): Box energies / deltas for each scale + level with shape (N, num_anchors * 4, H, W) + gt_bboxes (list[Tensor]): Ground truth bboxes for each image with + shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + gt_labels (list[Tensor]): class indices corresponding to each box + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + gt_bboxes_ignore (None | list[Tensor]): specify which bounding + boxes can be ignored when computing the loss. Default: None + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] + assert len(featmap_sizes) == self.anchor_generator.num_levels + + device = cls_scores[0].device + + anchor_list, valid_flag_list = self.get_anchors( + featmap_sizes, img_metas, device=device) + label_channels = self.cls_out_channels if self.use_sigmoid_cls else 1 + cls_reg_targets = self.get_targets( + anchor_list, + valid_flag_list, + gt_bboxes, + img_metas, + gt_bboxes_ignore_list=gt_bboxes_ignore, + gt_labels_list=gt_labels, + label_channels=label_channels) + if cls_reg_targets is None: + return None + (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, + num_total_pos, num_total_neg) = cls_reg_targets + num_total_samples = ( + num_total_pos + num_total_neg if self.sampling else num_total_pos) + + # anchor number of multi levels + num_level_anchors = [anchors.size(0) for anchors in anchor_list[0]] + # concat all level anchors and flags to a single tensor + concat_anchor_list = [] + for i in range(len(anchor_list)): + concat_anchor_list.append(torch.cat(anchor_list[i])) + all_anchor_list = images_to_levels(concat_anchor_list, + num_level_anchors) + + losses_cls, losses_bbox = multi_apply( + self.loss_single, + cls_scores, + bbox_preds, + all_anchor_list, + labels_list, + label_weights_list, + bbox_targets_list, + bbox_weights_list, + num_total_samples=num_total_samples) + return dict(loss_cls=losses_cls, loss_bbox=losses_bbox) + + @force_fp32(apply_to=('cls_scores', 'bbox_preds')) + def get_bboxes(self, + cls_scores, + bbox_preds, + img_metas, + cfg=None, + rescale=False, + with_nms=True): + """Transform network output for a batch into bbox predictions. + + Args: + cls_scores (list[Tensor]): Box scores for each level in the + feature pyramid, has shape + (N, num_anchors * num_classes, H, W). + bbox_preds (list[Tensor]): Box energies / deltas for each + level in the feature pyramid, has shape + (N, num_anchors * 4, H, W). + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + cfg (mmcv.Config | None): Test / postprocessing configuration, + if None, test_cfg would be used + rescale (bool): If True, return boxes in original image space. + Default: False. + with_nms (bool): If True, do nms before return boxes. + Default: True. + + Returns: + list[tuple[Tensor, Tensor]]: Each item in result_list is 2-tuple. + The first item is an (n, 5) tensor, where 5 represent + (tl_x, tl_y, br_x, br_y, score) and the score between 0 and 1. + The shape of the second tensor in the tuple is (n,), and + each element represents the class label of the corresponding + box. + + Example: + >>> import mmcv + >>> self = AnchorHead( + >>> num_classes=9, + >>> in_channels=1, + >>> anchor_generator=dict( + >>> type='AnchorGenerator', + >>> scales=[8], + >>> ratios=[0.5, 1.0, 2.0], + >>> strides=[4,])) + >>> img_metas = [{'img_shape': (32, 32, 3), 'scale_factor': 1}] + >>> cfg = mmcv.Config(dict( + >>> score_thr=0.00, + >>> nms=dict(type='nms', iou_thr=1.0), + >>> max_per_img=10)) + >>> feat = torch.rand(1, 1, 3, 3) + >>> cls_score, bbox_pred = self.forward_single(feat) + >>> # note the input lists are over different levels, not images + >>> cls_scores, bbox_preds = [cls_score], [bbox_pred] + >>> result_list = self.get_bboxes(cls_scores, bbox_preds, + >>> img_metas, cfg) + >>> det_bboxes, det_labels = result_list[0] + >>> assert len(result_list) == 1 + >>> assert det_bboxes.shape[1] == 5 + >>> assert len(det_bboxes) == len(det_labels) == cfg.max_per_img + """ + assert len(cls_scores) == len(bbox_preds) + num_levels = len(cls_scores) + + device = cls_scores[0].device + featmap_sizes = [cls_scores[i].shape[-2:] for i in range(num_levels)] + mlvl_anchors = self.anchor_generator.grid_anchors( + featmap_sizes, device=device) + + mlvl_cls_scores = [cls_scores[i].detach() for i in range(num_levels)] + mlvl_bbox_preds = [bbox_preds[i].detach() for i in range(num_levels)] + + if torch.onnx.is_in_onnx_export(): + assert len( + img_metas + ) == 1, 'Only support one input image while in exporting to ONNX' + img_shapes = img_metas[0]['img_shape_for_onnx'] + else: + img_shapes = [ + img_metas[i]['img_shape'] + for i in range(cls_scores[0].shape[0]) + ] + scale_factors = [ + img_metas[i]['scale_factor'] for i in range(cls_scores[0].shape[0]) + ] + + if with_nms: + # some heads don't support with_nms argument + result_list = self._get_bboxes(mlvl_cls_scores, mlvl_bbox_preds, + mlvl_anchors, img_shapes, + scale_factors, cfg, rescale) + else: + result_list = self._get_bboxes(mlvl_cls_scores, mlvl_bbox_preds, + mlvl_anchors, img_shapes, + scale_factors, cfg, rescale, + with_nms) + return result_list + + def _get_bboxes(self, + mlvl_cls_scores, + mlvl_bbox_preds, + mlvl_anchors, + img_shapes, + scale_factors, + cfg, + rescale=False, + with_nms=True): + """Transform outputs for a batch item into bbox predictions. + + Args: + mlvl_cls_scores (list[Tensor]): Each element in the list is + the scores of bboxes of single level in the feature pyramid, + has shape (N, num_anchors * num_classes, H, W). + mlvl_bbox_preds (list[Tensor]): Each element in the list is the + bboxes predictions of single level in the feature pyramid, + has shape (N, num_anchors * 4, H, W). + mlvl_anchors (list[Tensor]): Each element in the list is + the anchors of single level in feature pyramid, has shape + (num_anchors, 4). + img_shapes (list[tuple[int]]): Each tuple in the list represent + the shape(height, width, 3) of single image in the batch. + scale_factors (list[ndarray]): Scale factor of the batch + image arange as list[(w_scale, h_scale, w_scale, h_scale)]. + cfg (mmcv.Config): Test / postprocessing configuration, + if None, test_cfg would be used. + rescale (bool): If True, return boxes in original image space. + Default: False. + with_nms (bool): If True, do nms before return boxes. + Default: True. + + Returns: + list[tuple[Tensor, Tensor]]: Each item in result_list is 2-tuple. + The first item is an (n, 5) tensor, where 5 represent + (tl_x, tl_y, br_x, br_y, score) and the score between 0 and 1. + The shape of the second tensor in the tuple is (n,), and + each element represents the class label of the corresponding + box. + """ + cfg = self.test_cfg if cfg is None else cfg + assert len(mlvl_cls_scores) == len(mlvl_bbox_preds) == len( + mlvl_anchors) + batch_size = mlvl_cls_scores[0].shape[0] + # convert to tensor to keep tracing + nms_pre_tensor = torch.tensor( + cfg.get('nms_pre', -1), + device=mlvl_cls_scores[0].device, + dtype=torch.long) + + mlvl_bboxes = [] + mlvl_scores = [] + for cls_score, bbox_pred, anchors in zip(mlvl_cls_scores, + mlvl_bbox_preds, + mlvl_anchors): + assert cls_score.size()[-2:] == bbox_pred.size()[-2:] + cls_score = cls_score.permute(0, 2, 3, + 1).reshape(batch_size, -1, + self.cls_out_channels) + if self.use_sigmoid_cls: + scores = cls_score.sigmoid() + else: + scores = cls_score.softmax(-1) + bbox_pred = bbox_pred.permute(0, 2, 3, + 1).reshape(batch_size, -1, 4) + anchors = anchors.expand_as(bbox_pred) + # Always keep topk op for dynamic input in onnx + if nms_pre_tensor > 0 and (torch.onnx.is_in_onnx_export() + or scores.shape[-2] > nms_pre_tensor): + from torch import _shape_as_tensor + # keep shape as tensor and get k + num_anchor = _shape_as_tensor(scores)[-2].to( + nms_pre_tensor.device) + nms_pre = torch.where(nms_pre_tensor < num_anchor, + nms_pre_tensor, num_anchor) + + # Get maximum scores for foreground classes. + if self.use_sigmoid_cls: + max_scores, _ = scores.max(-1) + else: + # remind that we set FG labels to [0, num_class-1] + # since mmdet v2.0 + # BG cat_id: num_class + max_scores, _ = scores[..., :-1].max(-1) + + _, topk_inds = max_scores.topk(nms_pre) + batch_inds = torch.arange(batch_size).view( + -1, 1).expand_as(topk_inds) + anchors = anchors[batch_inds, topk_inds, :] + bbox_pred = bbox_pred[batch_inds, topk_inds, :] + scores = scores[batch_inds, topk_inds, :] + + bboxes = self.bbox_coder.decode( + anchors, bbox_pred, max_shape=img_shapes) + mlvl_bboxes.append(bboxes) + mlvl_scores.append(scores) + + batch_mlvl_bboxes = torch.cat(mlvl_bboxes, dim=1) + if rescale: + batch_mlvl_bboxes /= batch_mlvl_bboxes.new_tensor( + scale_factors).unsqueeze(1) + batch_mlvl_scores = torch.cat(mlvl_scores, dim=1) + + # Set max number of box to be feed into nms in deployment + deploy_nms_pre = cfg.get('deploy_nms_pre', -1) + if deploy_nms_pre > 0 and torch.onnx.is_in_onnx_export(): + # Get maximum scores for foreground classes. + if self.use_sigmoid_cls: + max_scores, _ = batch_mlvl_scores.max(-1) + else: + # remind that we set FG labels to [0, num_class-1] + # since mmdet v2.0 + # BG cat_id: num_class + max_scores, _ = batch_mlvl_scores[..., :-1].max(-1) + _, topk_inds = max_scores.topk(deploy_nms_pre) + batch_inds = torch.arange(batch_size).view(-1, + 1).expand_as(topk_inds) + batch_mlvl_scores = batch_mlvl_scores[batch_inds, topk_inds] + batch_mlvl_bboxes = batch_mlvl_bboxes[batch_inds, topk_inds] + if self.use_sigmoid_cls: + # Add a dummy background class to the backend when using sigmoid + # remind that we set FG labels to [0, num_class-1] since mmdet v2.0 + # BG cat_id: num_class + padding = batch_mlvl_scores.new_zeros(batch_size, + batch_mlvl_scores.shape[1], + 1) + batch_mlvl_scores = torch.cat([batch_mlvl_scores, padding], dim=-1) + + if with_nms: + det_results = [] + for (mlvl_bboxes, mlvl_scores) in zip(batch_mlvl_bboxes, + batch_mlvl_scores): + det_bbox, det_label = multiclass_nms(mlvl_bboxes, mlvl_scores, + cfg.score_thr, cfg.nms, + cfg.max_per_img) + det_results.append(tuple([det_bbox, det_label])) + else: + det_results = [ + tuple(mlvl_bs) + for mlvl_bs in zip(batch_mlvl_bboxes, batch_mlvl_scores) + ] + return det_results + + def aug_test(self, feats, img_metas, rescale=False): + """Test function with test time augmentation. + + Args: + feats (list[Tensor]): the outer list indicates test-time + augmentations and inner Tensor should have a shape NxCxHxW, + which contains features for all images in the batch. + img_metas (list[list[dict]]): the outer list indicates test-time + augs (multiscale, flip, etc.) and the inner list indicates + images in a batch. each dict has image information. + rescale (bool, optional): Whether to rescale the results. + Defaults to False. + + Returns: + list[ndarray]: bbox results of each class + """ + return self.aug_test_bboxes(feats, img_metas, rescale=rescale) diff --git a/detection/mmdet/models/dense_heads/atss_head.py b/detection/mmdet/models/dense_heads/atss_head.py new file mode 100644 index 0000000..ff55dfa --- /dev/null +++ b/detection/mmdet/models/dense_heads/atss_head.py @@ -0,0 +1,689 @@ +import torch +import torch.nn as nn +from mmcv.cnn import ConvModule, Scale, bias_init_with_prob, normal_init +from mmcv.runner import force_fp32 + +from mmdet.core import (anchor_inside_flags, build_assigner, build_sampler, + images_to_levels, multi_apply, multiclass_nms, + reduce_mean, unmap) +from ..builder import HEADS, build_loss +from .anchor_head import AnchorHead + +EPS = 1e-12 + + +@HEADS.register_module() +class ATSSHead(AnchorHead): + """Bridging the Gap Between Anchor-based and Anchor-free Detection via + Adaptive Training Sample Selection. + + ATSS head structure is similar with FCOS, however ATSS use anchor boxes + and assign label by Adaptive Training Sample Selection instead max-iou. + + https://arxiv.org/abs/1912.02424 + """ + + def __init__(self, + num_classes, + in_channels, + stacked_convs=4, + conv_cfg=None, + norm_cfg=dict(type='GN', num_groups=32, requires_grad=True), + loss_centerness=dict( + type='CrossEntropyLoss', + use_sigmoid=True, + loss_weight=1.0), + **kwargs): + self.stacked_convs = stacked_convs + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + super(ATSSHead, self).__init__(num_classes, in_channels, **kwargs) + + self.sampling = False + if self.train_cfg: + self.assigner = build_assigner(self.train_cfg.assigner) + # SSD sampling=False so use PseudoSampler + sampler_cfg = dict(type='PseudoSampler') + self.sampler = build_sampler(sampler_cfg, context=self) + self.loss_centerness = build_loss(loss_centerness) + + def _init_layers(self): + """Initialize layers of the head.""" + self.relu = nn.ReLU(inplace=True) + self.cls_convs = nn.ModuleList() + self.reg_convs = nn.ModuleList() + for i in range(self.stacked_convs): + chn = self.in_channels if i == 0 else self.feat_channels + self.cls_convs.append( + ConvModule( + chn, + self.feat_channels, + 3, + stride=1, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg)) + self.reg_convs.append( + ConvModule( + chn, + self.feat_channels, + 3, + stride=1, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg)) + self.atss_cls = nn.Conv2d( + self.feat_channels, + self.num_anchors * self.cls_out_channels, + 3, + padding=1) + self.atss_reg = nn.Conv2d( + self.feat_channels, self.num_anchors * 4, 3, padding=1) + self.atss_centerness = nn.Conv2d( + self.feat_channels, self.num_anchors * 1, 3, padding=1) + self.scales = nn.ModuleList( + [Scale(1.0) for _ in self.anchor_generator.strides]) + + def init_weights(self): + """Initialize weights of the head.""" + for m in self.cls_convs: + normal_init(m.conv, std=0.01) + for m in self.reg_convs: + normal_init(m.conv, std=0.01) + bias_cls = bias_init_with_prob(0.01) + normal_init(self.atss_cls, std=0.01, bias=bias_cls) + normal_init(self.atss_reg, std=0.01) + normal_init(self.atss_centerness, std=0.01) + + def forward(self, feats): + """Forward features from the upstream network. + + Args: + feats (tuple[Tensor]): Features from the upstream network, each is + a 4D-tensor. + + Returns: + tuple: Usually a tuple of classification scores and bbox prediction + cls_scores (list[Tensor]): Classification scores for all scale + levels, each is a 4D-tensor, the channels number is + num_anchors * num_classes. + bbox_preds (list[Tensor]): Box energies / deltas for all scale + levels, each is a 4D-tensor, the channels number is + num_anchors * 4. + """ + return multi_apply(self.forward_single, feats, self.scales) + + def forward_single(self, x, scale): + """Forward feature of a single scale level. + + Args: + x (Tensor): Features of a single scale level. + scale (:obj: `mmcv.cnn.Scale`): Learnable scale module to resize + the bbox prediction. + + Returns: + tuple: + cls_score (Tensor): Cls scores for a single scale level + the channels number is num_anchors * num_classes. + bbox_pred (Tensor): Box energies / deltas for a single scale + level, the channels number is num_anchors * 4. + centerness (Tensor): Centerness for a single scale level, the + channel number is (N, num_anchors * 1, H, W). + """ + cls_feat = x + reg_feat = x + for cls_conv in self.cls_convs: + cls_feat = cls_conv(cls_feat) + for reg_conv in self.reg_convs: + reg_feat = reg_conv(reg_feat) + cls_score = self.atss_cls(cls_feat) + # we just follow atss, not apply exp in bbox_pred + bbox_pred = scale(self.atss_reg(reg_feat)).float() + centerness = self.atss_centerness(reg_feat) + return cls_score, bbox_pred, centerness + + def loss_single(self, anchors, cls_score, bbox_pred, centerness, labels, + label_weights, bbox_targets, num_total_samples): + """Compute loss of a single scale level. + + Args: + cls_score (Tensor): Box scores for each scale level + Has shape (N, num_anchors * num_classes, H, W). + bbox_pred (Tensor): Box energies / deltas for each scale + level with shape (N, num_anchors * 4, H, W). + anchors (Tensor): Box reference for each scale level with shape + (N, num_total_anchors, 4). + labels (Tensor): Labels of each anchors with shape + (N, num_total_anchors). + label_weights (Tensor): Label weights of each anchor with shape + (N, num_total_anchors) + bbox_targets (Tensor): BBox regression targets of each anchor wight + shape (N, num_total_anchors, 4). + num_total_samples (int): Number os positive samples that is + reduced over all GPUs. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + + anchors = anchors.reshape(-1, 4) + cls_score = cls_score.permute(0, 2, 3, 1).reshape( + -1, self.cls_out_channels).contiguous() + bbox_pred = bbox_pred.permute(0, 2, 3, 1).reshape(-1, 4) + centerness = centerness.permute(0, 2, 3, 1).reshape(-1) + bbox_targets = bbox_targets.reshape(-1, 4) + labels = labels.reshape(-1) + label_weights = label_weights.reshape(-1) + + # classification loss + loss_cls = self.loss_cls( + cls_score, labels, label_weights, avg_factor=num_total_samples) + + # FG cat_id: [0, num_classes -1], BG cat_id: num_classes + bg_class_ind = self.num_classes + pos_inds = ((labels >= 0) + & (labels < bg_class_ind)).nonzero().squeeze(1) + + if len(pos_inds) > 0: + pos_bbox_targets = bbox_targets[pos_inds] + pos_bbox_pred = bbox_pred[pos_inds] + pos_anchors = anchors[pos_inds] + pos_centerness = centerness[pos_inds] + + centerness_targets = self.centerness_target( + pos_anchors, pos_bbox_targets) + pos_decode_bbox_pred = self.bbox_coder.decode( + pos_anchors, pos_bbox_pred) + pos_decode_bbox_targets = self.bbox_coder.decode( + pos_anchors, pos_bbox_targets) + + # regression loss + loss_bbox = self.loss_bbox( + pos_decode_bbox_pred, + pos_decode_bbox_targets, + weight=centerness_targets, + avg_factor=1.0) + + # centerness loss + loss_centerness = self.loss_centerness( + pos_centerness, + centerness_targets, + avg_factor=num_total_samples) + + else: + loss_bbox = bbox_pred.sum() * 0 + loss_centerness = centerness.sum() * 0 + centerness_targets = bbox_targets.new_tensor(0.) + + return loss_cls, loss_bbox, loss_centerness, centerness_targets.sum() + + @force_fp32(apply_to=('cls_scores', 'bbox_preds', 'centernesses')) + def loss(self, + cls_scores, + bbox_preds, + centernesses, + gt_bboxes, + gt_labels, + img_metas, + gt_bboxes_ignore=None): + """Compute losses of the head. + + Args: + cls_scores (list[Tensor]): Box scores for each scale level + Has shape (N, num_anchors * num_classes, H, W) + bbox_preds (list[Tensor]): Box energies / deltas for each scale + level with shape (N, num_anchors * 4, H, W) + centernesses (list[Tensor]): Centerness for each scale + level with shape (N, num_anchors * 1, H, W) + gt_bboxes (list[Tensor]): Ground truth bboxes for each image with + shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + gt_labels (list[Tensor]): class indices corresponding to each box + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + gt_bboxes_ignore (list[Tensor] | None): specify which bounding + boxes can be ignored when computing the loss. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] + assert len(featmap_sizes) == self.anchor_generator.num_levels + + device = cls_scores[0].device + anchor_list, valid_flag_list = self.get_anchors( + featmap_sizes, img_metas, device=device) + label_channels = self.cls_out_channels if self.use_sigmoid_cls else 1 + + cls_reg_targets = self.get_targets( + anchor_list, + valid_flag_list, + gt_bboxes, + img_metas, + gt_bboxes_ignore_list=gt_bboxes_ignore, + gt_labels_list=gt_labels, + label_channels=label_channels) + if cls_reg_targets is None: + return None + + (anchor_list, labels_list, label_weights_list, bbox_targets_list, + bbox_weights_list, num_total_pos, num_total_neg) = cls_reg_targets + + num_total_samples = reduce_mean( + torch.tensor(num_total_pos, dtype=torch.float, + device=device)).item() + num_total_samples = max(num_total_samples, 1.0) + + losses_cls, losses_bbox, loss_centerness,\ + bbox_avg_factor = multi_apply( + self.loss_single, + anchor_list, + cls_scores, + bbox_preds, + centernesses, + labels_list, + label_weights_list, + bbox_targets_list, + num_total_samples=num_total_samples) + + bbox_avg_factor = sum(bbox_avg_factor) + bbox_avg_factor = reduce_mean(bbox_avg_factor).item() + if bbox_avg_factor < EPS: + bbox_avg_factor = 1 + losses_bbox = list(map(lambda x: x / bbox_avg_factor, losses_bbox)) + return dict( + loss_cls=losses_cls, + loss_bbox=losses_bbox, + loss_centerness=loss_centerness) + + def centerness_target(self, anchors, bbox_targets): + # only calculate pos centerness targets, otherwise there may be nan + gts = self.bbox_coder.decode(anchors, bbox_targets) + anchors_cx = (anchors[:, 2] + anchors[:, 0]) / 2 + anchors_cy = (anchors[:, 3] + anchors[:, 1]) / 2 + l_ = anchors_cx - gts[:, 0] + t_ = anchors_cy - gts[:, 1] + r_ = gts[:, 2] - anchors_cx + b_ = gts[:, 3] - anchors_cy + + left_right = torch.stack([l_, r_], dim=1) + top_bottom = torch.stack([t_, b_], dim=1) + centerness = torch.sqrt( + (left_right.min(dim=-1)[0] / left_right.max(dim=-1)[0]) * + (top_bottom.min(dim=-1)[0] / top_bottom.max(dim=-1)[0])) + assert not torch.isnan(centerness).any() + return centerness + + @force_fp32(apply_to=('cls_scores', 'bbox_preds', 'centernesses')) + def get_bboxes(self, + cls_scores, + bbox_preds, + centernesses, + img_metas, + cfg=None, + rescale=False, + with_nms=True): + """Transform network output for a batch into bbox predictions. + + Args: + cls_scores (list[Tensor]): Box scores for each scale level + with shape (N, num_anchors * num_classes, H, W). + bbox_preds (list[Tensor]): Box energies / deltas for each scale + level with shape (N, num_anchors * 4, H, W). + centernesses (list[Tensor]): Centerness for each scale level with + shape (N, num_anchors * 1, H, W). + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + cfg (mmcv.Config | None): Test / postprocessing configuration, + if None, test_cfg would be used. Default: None. + rescale (bool): If True, return boxes in original image space. + Default: False. + with_nms (bool): If True, do nms before return boxes. + Default: True. + + Returns: + list[tuple[Tensor, Tensor]]: Each item in result_list is 2-tuple. + The first item is an (n, 5) tensor, where 5 represent + (tl_x, tl_y, br_x, br_y, score) and the score between 0 and 1. + The shape of the second tensor in the tuple is (n,), and + each element represents the class label of the corresponding + box. + """ + cfg = self.test_cfg if cfg is None else cfg + assert len(cls_scores) == len(bbox_preds) + num_levels = len(cls_scores) + device = cls_scores[0].device + featmap_sizes = [cls_scores[i].shape[-2:] for i in range(num_levels)] + mlvl_anchors = self.anchor_generator.grid_anchors( + featmap_sizes, device=device) + + cls_score_list = [cls_scores[i].detach() for i in range(num_levels)] + bbox_pred_list = [bbox_preds[i].detach() for i in range(num_levels)] + centerness_pred_list = [ + centernesses[i].detach() for i in range(num_levels) + ] + img_shapes = [ + img_metas[i]['img_shape'] for i in range(cls_scores[0].shape[0]) + ] + scale_factors = [ + img_metas[i]['scale_factor'] for i in range(cls_scores[0].shape[0]) + ] + result_list = self._get_bboxes(cls_score_list, bbox_pred_list, + centerness_pred_list, mlvl_anchors, + img_shapes, scale_factors, cfg, rescale, + with_nms) + return result_list + + def _get_bboxes(self, + cls_scores, + bbox_preds, + centernesses, + mlvl_anchors, + img_shapes, + scale_factors, + cfg, + rescale=False, + with_nms=True): + """Transform outputs for a single batch item into labeled boxes. + + Args: + cls_scores (list[Tensor]): Box scores for a single scale level + with shape (N, num_anchors * num_classes, H, W). + bbox_preds (list[Tensor]): Box energies / deltas for a single + scale level with shape (N, num_anchors * 4, H, W). + centernesses (list[Tensor]): Centerness for a single scale level + with shape (N, num_anchors * 1, H, W). + mlvl_anchors (list[Tensor]): Box reference for a single scale level + with shape (num_total_anchors, 4). + img_shapes (list[tuple[int]]): Shape of the input image, + list[(height, width, 3)]. + scale_factors (list[ndarray]): Scale factor of the image arrange as + (w_scale, h_scale, w_scale, h_scale). + cfg (mmcv.Config | None): Test / postprocessing configuration, + if None, test_cfg would be used. + rescale (bool): If True, return boxes in original image space. + Default: False. + with_nms (bool): If True, do nms before return boxes. + Default: True. + + Returns: + list[tuple[Tensor, Tensor]]: Each item in result_list is 2-tuple. + The first item is an (n, 5) tensor, where 5 represent + (tl_x, tl_y, br_x, br_y, score) and the score between 0 and 1. + The shape of the second tensor in the tuple is (n,), and + each element represents the class label of the corresponding + box. + """ + assert len(cls_scores) == len(bbox_preds) == len(mlvl_anchors) + device = cls_scores[0].device + batch_size = cls_scores[0].shape[0] + # convert to tensor to keep tracing + nms_pre_tensor = torch.tensor( + cfg.get('nms_pre', -1), device=device, dtype=torch.long) + mlvl_bboxes = [] + mlvl_scores = [] + mlvl_centerness = [] + for cls_score, bbox_pred, centerness, anchors in zip( + cls_scores, bbox_preds, centernesses, mlvl_anchors): + assert cls_score.size()[-2:] == bbox_pred.size()[-2:] + scores = cls_score.permute(0, 2, 3, 1).reshape( + batch_size, -1, self.cls_out_channels).sigmoid() + centerness = centerness.permute(0, 2, 3, + 1).reshape(batch_size, + -1).sigmoid() + bbox_pred = bbox_pred.permute(0, 2, 3, + 1).reshape(batch_size, -1, 4) + + # Always keep topk op for dynamic input in onnx + if nms_pre_tensor > 0 and (torch.onnx.is_in_onnx_export() + or scores.shape[-2] > nms_pre_tensor): + from torch import _shape_as_tensor + # keep shape as tensor and get k + num_anchor = _shape_as_tensor(scores)[-2].to(device) + nms_pre = torch.where(nms_pre_tensor < num_anchor, + nms_pre_tensor, num_anchor) + + max_scores, _ = (scores * centerness[..., None]).max(-1) + _, topk_inds = max_scores.topk(nms_pre) + anchors = anchors[topk_inds, :] + batch_inds = torch.arange(batch_size).view( + -1, 1).expand_as(topk_inds).long() + bbox_pred = bbox_pred[batch_inds, topk_inds, :] + scores = scores[batch_inds, topk_inds, :] + centerness = centerness[batch_inds, topk_inds] + else: + anchors = anchors.expand_as(bbox_pred) + + bboxes = self.bbox_coder.decode( + anchors, bbox_pred, max_shape=img_shapes) + mlvl_bboxes.append(bboxes) + mlvl_scores.append(scores) + mlvl_centerness.append(centerness) + + batch_mlvl_bboxes = torch.cat(mlvl_bboxes, dim=1) + if rescale: + batch_mlvl_bboxes /= batch_mlvl_bboxes.new_tensor( + scale_factors).unsqueeze(1) + batch_mlvl_scores = torch.cat(mlvl_scores, dim=1) + batch_mlvl_centerness = torch.cat(mlvl_centerness, dim=1) + + # Set max number of box to be feed into nms in deployment + deploy_nms_pre = cfg.get('deploy_nms_pre', -1) + if deploy_nms_pre > 0 and torch.onnx.is_in_onnx_export(): + batch_mlvl_scores, _ = ( + batch_mlvl_scores * + batch_mlvl_centerness.unsqueeze(2).expand_as(batch_mlvl_scores) + ).max(-1) + _, topk_inds = batch_mlvl_scores.topk(deploy_nms_pre) + batch_inds = torch.arange(batch_size).view(-1, + 1).expand_as(topk_inds) + batch_mlvl_scores = batch_mlvl_scores[batch_inds, topk_inds, :] + batch_mlvl_bboxes = batch_mlvl_bboxes[batch_inds, topk_inds, :] + batch_mlvl_centerness = batch_mlvl_centerness[batch_inds, + topk_inds] + # remind that we set FG labels to [0, num_class-1] since mmdet v2.0 + # BG cat_id: num_class + padding = batch_mlvl_scores.new_zeros(batch_size, + batch_mlvl_scores.shape[1], 1) + batch_mlvl_scores = torch.cat([batch_mlvl_scores, padding], dim=-1) + + if with_nms: + det_results = [] + for (mlvl_bboxes, mlvl_scores, + mlvl_centerness) in zip(batch_mlvl_bboxes, batch_mlvl_scores, + batch_mlvl_centerness): + det_bbox, det_label = multiclass_nms( + mlvl_bboxes, + mlvl_scores, + cfg.score_thr, + cfg.nms, + cfg.max_per_img, + score_factors=mlvl_centerness) + det_results.append(tuple([det_bbox, det_label])) + else: + det_results = [ + tuple(mlvl_bs) + for mlvl_bs in zip(batch_mlvl_bboxes, batch_mlvl_scores, + batch_mlvl_centerness) + ] + return det_results + + def get_targets(self, + anchor_list, + valid_flag_list, + gt_bboxes_list, + img_metas, + gt_bboxes_ignore_list=None, + gt_labels_list=None, + label_channels=1, + unmap_outputs=True): + """Get targets for ATSS head. + + This method is almost the same as `AnchorHead.get_targets()`. Besides + returning the targets as the parent method does, it also returns the + anchors as the first element of the returned tuple. + """ + num_imgs = len(img_metas) + assert len(anchor_list) == len(valid_flag_list) == num_imgs + + # anchor number of multi levels + num_level_anchors = [anchors.size(0) for anchors in anchor_list[0]] + num_level_anchors_list = [num_level_anchors] * num_imgs + + # concat all level anchors and flags to a single tensor + for i in range(num_imgs): + assert len(anchor_list[i]) == len(valid_flag_list[i]) + anchor_list[i] = torch.cat(anchor_list[i]) + valid_flag_list[i] = torch.cat(valid_flag_list[i]) + + # compute targets for each image + if gt_bboxes_ignore_list is None: + gt_bboxes_ignore_list = [None for _ in range(num_imgs)] + if gt_labels_list is None: + gt_labels_list = [None for _ in range(num_imgs)] + (all_anchors, all_labels, all_label_weights, all_bbox_targets, + all_bbox_weights, pos_inds_list, neg_inds_list) = multi_apply( + self._get_target_single, + anchor_list, + valid_flag_list, + num_level_anchors_list, + gt_bboxes_list, + gt_bboxes_ignore_list, + gt_labels_list, + img_metas, + label_channels=label_channels, + unmap_outputs=unmap_outputs) + # no valid anchors + if any([labels is None for labels in all_labels]): + return None + # sampled anchors of all images + num_total_pos = sum([max(inds.numel(), 1) for inds in pos_inds_list]) + num_total_neg = sum([max(inds.numel(), 1) for inds in neg_inds_list]) + # split targets to a list w.r.t. multiple levels + anchors_list = images_to_levels(all_anchors, num_level_anchors) + labels_list = images_to_levels(all_labels, num_level_anchors) + label_weights_list = images_to_levels(all_label_weights, + num_level_anchors) + bbox_targets_list = images_to_levels(all_bbox_targets, + num_level_anchors) + bbox_weights_list = images_to_levels(all_bbox_weights, + num_level_anchors) + return (anchors_list, labels_list, label_weights_list, + bbox_targets_list, bbox_weights_list, num_total_pos, + num_total_neg) + + def _get_target_single(self, + flat_anchors, + valid_flags, + num_level_anchors, + gt_bboxes, + gt_bboxes_ignore, + gt_labels, + img_meta, + label_channels=1, + unmap_outputs=True): + """Compute regression, classification targets for anchors in a single + image. + + Args: + flat_anchors (Tensor): Multi-level anchors of the image, which are + concatenated into a single tensor of shape (num_anchors ,4) + valid_flags (Tensor): Multi level valid flags of the image, + which are concatenated into a single tensor of + shape (num_anchors,). + num_level_anchors Tensor): Number of anchors of each scale level. + gt_bboxes (Tensor): Ground truth bboxes of the image, + shape (num_gts, 4). + gt_bboxes_ignore (Tensor): Ground truth bboxes to be + ignored, shape (num_ignored_gts, 4). + gt_labels (Tensor): Ground truth labels of each box, + shape (num_gts,). + img_meta (dict): Meta info of the image. + label_channels (int): Channel of label. + unmap_outputs (bool): Whether to map outputs back to the original + set of anchors. + + Returns: + tuple: N is the number of total anchors in the image. + labels (Tensor): Labels of all anchors in the image with shape + (N,). + label_weights (Tensor): Label weights of all anchor in the + image with shape (N,). + bbox_targets (Tensor): BBox targets of all anchors in the + image with shape (N, 4). + bbox_weights (Tensor): BBox weights of all anchors in the + image with shape (N, 4) + pos_inds (Tensor): Indices of positive anchor with shape + (num_pos,). + neg_inds (Tensor): Indices of negative anchor with shape + (num_neg,). + """ + inside_flags = anchor_inside_flags(flat_anchors, valid_flags, + img_meta['img_shape'][:2], + self.train_cfg.allowed_border) + if not inside_flags.any(): + return (None, ) * 7 + # assign gt and sample anchors + anchors = flat_anchors[inside_flags, :] + + num_level_anchors_inside = self.get_num_level_anchors_inside( + num_level_anchors, inside_flags) + assign_result = self.assigner.assign(anchors, num_level_anchors_inside, + gt_bboxes, gt_bboxes_ignore, + gt_labels) + + sampling_result = self.sampler.sample(assign_result, anchors, + gt_bboxes) + + num_valid_anchors = anchors.shape[0] + bbox_targets = torch.zeros_like(anchors) + bbox_weights = torch.zeros_like(anchors) + labels = anchors.new_full((num_valid_anchors, ), + self.num_classes, + dtype=torch.long) + label_weights = anchors.new_zeros(num_valid_anchors, dtype=torch.float) + + pos_inds = sampling_result.pos_inds + neg_inds = sampling_result.neg_inds + if len(pos_inds) > 0: + if hasattr(self, 'bbox_coder'): + pos_bbox_targets = self.bbox_coder.encode( + sampling_result.pos_bboxes, sampling_result.pos_gt_bboxes) + else: + # used in VFNetHead + pos_bbox_targets = sampling_result.pos_gt_bboxes + bbox_targets[pos_inds, :] = pos_bbox_targets + bbox_weights[pos_inds, :] = 1.0 + if gt_labels is None: + # Only rpn gives gt_labels as None + # Foreground is the first class since v2.5.0 + labels[pos_inds] = 0 + else: + labels[pos_inds] = gt_labels[ + sampling_result.pos_assigned_gt_inds] + if self.train_cfg.pos_weight <= 0: + label_weights[pos_inds] = 1.0 + else: + label_weights[pos_inds] = self.train_cfg.pos_weight + if len(neg_inds) > 0: + label_weights[neg_inds] = 1.0 + + # map up to original set of anchors + if unmap_outputs: + num_total_anchors = flat_anchors.size(0) + anchors = unmap(anchors, num_total_anchors, inside_flags) + labels = unmap( + labels, num_total_anchors, inside_flags, fill=self.num_classes) + label_weights = unmap(label_weights, num_total_anchors, + inside_flags) + bbox_targets = unmap(bbox_targets, num_total_anchors, inside_flags) + bbox_weights = unmap(bbox_weights, num_total_anchors, inside_flags) + + return (anchors, labels, label_weights, bbox_targets, bbox_weights, + pos_inds, neg_inds) + + def get_num_level_anchors_inside(self, num_level_anchors, inside_flags): + split_inside_flags = torch.split(inside_flags, num_level_anchors) + num_level_anchors_inside = [ + int(flags.sum()) for flags in split_inside_flags + ] + return num_level_anchors_inside diff --git a/detection/mmdet/models/dense_heads/base_dense_head.py b/detection/mmdet/models/dense_heads/base_dense_head.py new file mode 100644 index 0000000..de11e4a --- /dev/null +++ b/detection/mmdet/models/dense_heads/base_dense_head.py @@ -0,0 +1,59 @@ +from abc import ABCMeta, abstractmethod + +import torch.nn as nn + + +class BaseDenseHead(nn.Module, metaclass=ABCMeta): + """Base class for DenseHeads.""" + + def __init__(self): + super(BaseDenseHead, self).__init__() + + @abstractmethod + def loss(self, **kwargs): + """Compute losses of the head.""" + pass + + @abstractmethod + def get_bboxes(self, **kwargs): + """Transform network output for a batch into bbox predictions.""" + pass + + def forward_train(self, + x, + img_metas, + gt_bboxes, + gt_labels=None, + gt_bboxes_ignore=None, + proposal_cfg=None, + **kwargs): + """ + Args: + x (list[Tensor]): Features from FPN. + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + gt_bboxes (Tensor): Ground truth bboxes of the image, + shape (num_gts, 4). + gt_labels (Tensor): Ground truth labels of each box, + shape (num_gts,). + gt_bboxes_ignore (Tensor): Ground truth bboxes to be + ignored, shape (num_ignored_gts, 4). + proposal_cfg (mmcv.Config): Test / postprocessing configuration, + if None, test_cfg would be used + + Returns: + tuple: + losses: (dict[str, Tensor]): A dictionary of loss components. + proposal_list (list[Tensor]): Proposals of each image. + """ + outs = self(x) + if gt_labels is None: + loss_inputs = outs + (gt_bboxes, img_metas) + else: + loss_inputs = outs + (gt_bboxes, gt_labels, img_metas) + losses = self.loss(*loss_inputs, gt_bboxes_ignore=gt_bboxes_ignore) + if proposal_cfg is None: + return losses + else: + proposal_list = self.get_bboxes(*outs, img_metas, cfg=proposal_cfg) + return losses, proposal_list diff --git a/detection/mmdet/models/dense_heads/cascade_rpn_head.py b/detection/mmdet/models/dense_heads/cascade_rpn_head.py new file mode 100644 index 0000000..e32ee46 --- /dev/null +++ b/detection/mmdet/models/dense_heads/cascade_rpn_head.py @@ -0,0 +1,784 @@ +from __future__ import division +import copy +import warnings + +import torch +import torch.nn as nn +from mmcv import ConfigDict +from mmcv.cnn import normal_init +from mmcv.ops import DeformConv2d, batched_nms + +from mmdet.core import (RegionAssigner, build_assigner, build_sampler, + images_to_levels, multi_apply) +from ..builder import HEADS, build_head +from .base_dense_head import BaseDenseHead +from .rpn_head import RPNHead + + +class AdaptiveConv(nn.Module): + """AdaptiveConv used to adapt the sampling location with the anchors. + + Args: + in_channels (int): Number of channels in the input image + out_channels (int): Number of channels produced by the convolution + kernel_size (int or tuple): Size of the conv kernel. Default: 3 + stride (int or tuple, optional): Stride of the convolution. Default: 1 + padding (int or tuple, optional): Zero-padding added to both sides of + the input. Default: 1 + dilation (int or tuple, optional): Spacing between kernel elements. + Default: 3 + groups (int, optional): Number of blocked connections from input + channels to output channels. Default: 1 + bias (bool, optional): If set True, adds a learnable bias to the + output. Default: False. + type (str, optional): Type of adaptive conv, can be either 'offset' + (arbitrary anchors) or 'dilation' (uniform anchor). + Default: 'dilation'. + """ + + def __init__(self, + in_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1, + dilation=3, + groups=1, + bias=False, + type='dilation'): + super(AdaptiveConv, self).__init__() + assert type in ['offset', 'dilation'] + self.adapt_type = type + + assert kernel_size == 3, 'Adaptive conv only supports kernels 3' + if self.adapt_type == 'offset': + assert stride == 1 and padding == 1 and groups == 1, \ + 'Adaptive conv offset mode only supports padding: {1}, ' \ + f'stride: {1}, groups: {1}' + self.conv = DeformConv2d( + in_channels, + out_channels, + kernel_size, + padding=padding, + stride=stride, + groups=groups, + bias=bias) + else: + self.conv = nn.Conv2d( + in_channels, + out_channels, + kernel_size, + padding=dilation, + dilation=dilation) + + def init_weights(self): + """Init weights.""" + normal_init(self.conv, std=0.01) + + def forward(self, x, offset): + """Forward function.""" + if self.adapt_type == 'offset': + N, _, H, W = x.shape + assert offset is not None + assert H * W == offset.shape[1] + # reshape [N, NA, 18] to (N, 18, H, W) + offset = offset.permute(0, 2, 1).reshape(N, -1, H, W) + offset = offset.contiguous() + x = self.conv(x, offset) + else: + assert offset is None + x = self.conv(x) + return x + + +@HEADS.register_module() +class StageCascadeRPNHead(RPNHead): + """Stage of CascadeRPNHead. + + Args: + in_channels (int): Number of channels in the input feature map. + anchor_generator (dict): anchor generator config. + adapt_cfg (dict): adaptation config. + bridged_feature (bool, optional): whether update rpn feature. + Default: False. + with_cls (bool, optional): wheather use classification branch. + Default: True. + sampling (bool, optional): wheather use sampling. Default: True. + """ + + def __init__(self, + in_channels, + anchor_generator=dict( + type='AnchorGenerator', + scales=[8], + ratios=[1.0], + strides=[4, 8, 16, 32, 64]), + adapt_cfg=dict(type='dilation', dilation=3), + bridged_feature=False, + with_cls=True, + sampling=True, + **kwargs): + self.with_cls = with_cls + self.anchor_strides = anchor_generator['strides'] + self.anchor_scales = anchor_generator['scales'] + self.bridged_feature = bridged_feature + self.adapt_cfg = adapt_cfg + super(StageCascadeRPNHead, self).__init__( + in_channels, anchor_generator=anchor_generator, **kwargs) + + # override sampling and sampler + self.sampling = sampling + if self.train_cfg: + self.assigner = build_assigner(self.train_cfg.assigner) + # use PseudoSampler when sampling is False + if self.sampling and hasattr(self.train_cfg, 'sampler'): + sampler_cfg = self.train_cfg.sampler + else: + sampler_cfg = dict(type='PseudoSampler') + self.sampler = build_sampler(sampler_cfg, context=self) + + def _init_layers(self): + """Init layers of a CascadeRPN stage.""" + self.rpn_conv = AdaptiveConv(self.in_channels, self.feat_channels, + **self.adapt_cfg) + if self.with_cls: + self.rpn_cls = nn.Conv2d(self.feat_channels, + self.num_anchors * self.cls_out_channels, + 1) + self.rpn_reg = nn.Conv2d(self.feat_channels, self.num_anchors * 4, 1) + self.relu = nn.ReLU(inplace=True) + + def init_weights(self): + """Init weights of a CascadeRPN stage.""" + self.rpn_conv.init_weights() + normal_init(self.rpn_reg, std=0.01) + if self.with_cls: + normal_init(self.rpn_cls, std=0.01) + + def forward_single(self, x, offset): + """Forward function of single scale.""" + bridged_x = x + x = self.relu(self.rpn_conv(x, offset)) + if self.bridged_feature: + bridged_x = x # update feature + cls_score = self.rpn_cls(x) if self.with_cls else None + bbox_pred = self.rpn_reg(x) + return bridged_x, cls_score, bbox_pred + + def forward(self, feats, offset_list=None): + """Forward function.""" + if offset_list is None: + offset_list = [None for _ in range(len(feats))] + return multi_apply(self.forward_single, feats, offset_list) + + def _region_targets_single(self, + anchors, + valid_flags, + gt_bboxes, + gt_bboxes_ignore, + gt_labels, + img_meta, + featmap_sizes, + label_channels=1): + """Get anchor targets based on region for single level.""" + assign_result = self.assigner.assign( + anchors, + valid_flags, + gt_bboxes, + img_meta, + featmap_sizes, + self.anchor_scales[0], + self.anchor_strides, + gt_bboxes_ignore=gt_bboxes_ignore, + gt_labels=None, + allowed_border=self.train_cfg.allowed_border) + flat_anchors = torch.cat(anchors) + sampling_result = self.sampler.sample(assign_result, flat_anchors, + gt_bboxes) + + num_anchors = flat_anchors.shape[0] + bbox_targets = torch.zeros_like(flat_anchors) + bbox_weights = torch.zeros_like(flat_anchors) + labels = flat_anchors.new_zeros(num_anchors, dtype=torch.long) + label_weights = flat_anchors.new_zeros(num_anchors, dtype=torch.float) + + pos_inds = sampling_result.pos_inds + neg_inds = sampling_result.neg_inds + if len(pos_inds) > 0: + if not self.reg_decoded_bbox: + pos_bbox_targets = self.bbox_coder.encode( + sampling_result.pos_bboxes, sampling_result.pos_gt_bboxes) + else: + pos_bbox_targets = sampling_result.pos_gt_bboxes + bbox_targets[pos_inds, :] = pos_bbox_targets + bbox_weights[pos_inds, :] = 1.0 + if gt_labels is None: + labels[pos_inds] = 1 + else: + labels[pos_inds] = gt_labels[ + sampling_result.pos_assigned_gt_inds] + if self.train_cfg.pos_weight <= 0: + label_weights[pos_inds] = 1.0 + else: + label_weights[pos_inds] = self.train_cfg.pos_weight + if len(neg_inds) > 0: + label_weights[neg_inds] = 1.0 + + return (labels, label_weights, bbox_targets, bbox_weights, pos_inds, + neg_inds) + + def region_targets(self, + anchor_list, + valid_flag_list, + gt_bboxes_list, + img_metas, + featmap_sizes, + gt_bboxes_ignore_list=None, + gt_labels_list=None, + label_channels=1, + unmap_outputs=True): + """See :func:`StageCascadeRPNHead.get_targets`.""" + num_imgs = len(img_metas) + assert len(anchor_list) == len(valid_flag_list) == num_imgs + + # anchor number of multi levels + num_level_anchors = [anchors.size(0) for anchors in anchor_list[0]] + + # compute targets for each image + if gt_bboxes_ignore_list is None: + gt_bboxes_ignore_list = [None for _ in range(num_imgs)] + if gt_labels_list is None: + gt_labels_list = [None for _ in range(num_imgs)] + (all_labels, all_label_weights, all_bbox_targets, all_bbox_weights, + pos_inds_list, neg_inds_list) = multi_apply( + self._region_targets_single, + anchor_list, + valid_flag_list, + gt_bboxes_list, + gt_bboxes_ignore_list, + gt_labels_list, + img_metas, + featmap_sizes=featmap_sizes, + label_channels=label_channels) + # no valid anchors + if any([labels is None for labels in all_labels]): + return None + # sampled anchors of all images + num_total_pos = sum([max(inds.numel(), 1) for inds in pos_inds_list]) + num_total_neg = sum([max(inds.numel(), 1) for inds in neg_inds_list]) + # split targets to a list w.r.t. multiple levels + labels_list = images_to_levels(all_labels, num_level_anchors) + label_weights_list = images_to_levels(all_label_weights, + num_level_anchors) + bbox_targets_list = images_to_levels(all_bbox_targets, + num_level_anchors) + bbox_weights_list = images_to_levels(all_bbox_weights, + num_level_anchors) + return (labels_list, label_weights_list, bbox_targets_list, + bbox_weights_list, num_total_pos, num_total_neg) + + def get_targets(self, + anchor_list, + valid_flag_list, + gt_bboxes, + img_metas, + featmap_sizes, + gt_bboxes_ignore=None, + label_channels=1): + """Compute regression and classification targets for anchors. + + Args: + anchor_list (list[list]): Multi level anchors of each image. + valid_flag_list (list[list]): Multi level valid flags of each + image. + gt_bboxes (list[Tensor]): Ground truth bboxes of each image. + img_metas (list[dict]): Meta info of each image. + featmap_sizes (list[Tensor]): Feature mapsize each level + gt_bboxes_ignore (list[Tensor]): Ignore bboxes of each images + label_channels (int): Channel of label. + + Returns: + cls_reg_targets (tuple) + """ + if isinstance(self.assigner, RegionAssigner): + cls_reg_targets = self.region_targets( + anchor_list, + valid_flag_list, + gt_bboxes, + img_metas, + featmap_sizes, + gt_bboxes_ignore_list=gt_bboxes_ignore, + label_channels=label_channels) + else: + cls_reg_targets = super(StageCascadeRPNHead, self).get_targets( + anchor_list, + valid_flag_list, + gt_bboxes, + img_metas, + gt_bboxes_ignore_list=gt_bboxes_ignore, + label_channels=label_channels) + return cls_reg_targets + + def anchor_offset(self, anchor_list, anchor_strides, featmap_sizes): + """ Get offest for deformable conv based on anchor shape + NOTE: currently support deformable kernel_size=3 and dilation=1 + + Args: + anchor_list (list[list[tensor])): [NI, NLVL, NA, 4] list of + multi-level anchors + anchor_strides (list[int]): anchor stride of each level + + Returns: + offset_list (list[tensor]): [NLVL, NA, 2, 18]: offset of DeformConv + kernel. + """ + + def _shape_offset(anchors, stride, ks=3, dilation=1): + # currently support kernel_size=3 and dilation=1 + assert ks == 3 and dilation == 1 + pad = (ks - 1) // 2 + idx = torch.arange(-pad, pad + 1, dtype=dtype, device=device) + yy, xx = torch.meshgrid(idx, idx) # return order matters + xx = xx.reshape(-1) + yy = yy.reshape(-1) + w = (anchors[:, 2] - anchors[:, 0]) / stride + h = (anchors[:, 3] - anchors[:, 1]) / stride + w = w / (ks - 1) - dilation + h = h / (ks - 1) - dilation + offset_x = w[:, None] * xx # (NA, ks**2) + offset_y = h[:, None] * yy # (NA, ks**2) + return offset_x, offset_y + + def _ctr_offset(anchors, stride, featmap_size): + feat_h, feat_w = featmap_size + assert len(anchors) == feat_h * feat_w + + x = (anchors[:, 0] + anchors[:, 2]) * 0.5 + y = (anchors[:, 1] + anchors[:, 3]) * 0.5 + # compute centers on feature map + x = x / stride + y = y / stride + # compute predefine centers + xx = torch.arange(0, feat_w, device=anchors.device) + yy = torch.arange(0, feat_h, device=anchors.device) + yy, xx = torch.meshgrid(yy, xx) + xx = xx.reshape(-1).type_as(x) + yy = yy.reshape(-1).type_as(y) + + offset_x = x - xx # (NA, ) + offset_y = y - yy # (NA, ) + return offset_x, offset_y + + num_imgs = len(anchor_list) + num_lvls = len(anchor_list[0]) + dtype = anchor_list[0][0].dtype + device = anchor_list[0][0].device + num_level_anchors = [anchors.size(0) for anchors in anchor_list[0]] + + offset_list = [] + for i in range(num_imgs): + mlvl_offset = [] + for lvl in range(num_lvls): + c_offset_x, c_offset_y = _ctr_offset(anchor_list[i][lvl], + anchor_strides[lvl], + featmap_sizes[lvl]) + s_offset_x, s_offset_y = _shape_offset(anchor_list[i][lvl], + anchor_strides[lvl]) + + # offset = ctr_offset + shape_offset + offset_x = s_offset_x + c_offset_x[:, None] + offset_y = s_offset_y + c_offset_y[:, None] + + # offset order (y0, x0, y1, x2, .., y8, x8, y9, x9) + offset = torch.stack([offset_y, offset_x], dim=-1) + offset = offset.reshape(offset.size(0), -1) # [NA, 2*ks**2] + mlvl_offset.append(offset) + offset_list.append(torch.cat(mlvl_offset)) # [totalNA, 2*ks**2] + offset_list = images_to_levels(offset_list, num_level_anchors) + return offset_list + + def loss_single(self, cls_score, bbox_pred, anchors, labels, label_weights, + bbox_targets, bbox_weights, num_total_samples): + """Loss function on single scale.""" + # classification loss + if self.with_cls: + labels = labels.reshape(-1) + label_weights = label_weights.reshape(-1) + cls_score = cls_score.permute(0, 2, 3, + 1).reshape(-1, self.cls_out_channels) + loss_cls = self.loss_cls( + cls_score, labels, label_weights, avg_factor=num_total_samples) + # regression loss + bbox_targets = bbox_targets.reshape(-1, 4) + bbox_weights = bbox_weights.reshape(-1, 4) + bbox_pred = bbox_pred.permute(0, 2, 3, 1).reshape(-1, 4) + if self.reg_decoded_bbox: + # When the regression loss (e.g. `IouLoss`, `GIouLoss`) + # is applied directly on the decoded bounding boxes, it + # decodes the already encoded coordinates to absolute format. + anchors = anchors.reshape(-1, 4) + bbox_pred = self.bbox_coder.decode(anchors, bbox_pred) + loss_reg = self.loss_bbox( + bbox_pred, + bbox_targets, + bbox_weights, + avg_factor=num_total_samples) + if self.with_cls: + return loss_cls, loss_reg + return None, loss_reg + + def loss(self, + anchor_list, + valid_flag_list, + cls_scores, + bbox_preds, + gt_bboxes, + img_metas, + gt_bboxes_ignore=None): + """Compute losses of the head. + + Args: + anchor_list (list[list]): Multi level anchors of each image. + cls_scores (list[Tensor]): Box scores for each scale level + Has shape (N, num_anchors * num_classes, H, W) + bbox_preds (list[Tensor]): Box energies / deltas for each scale + level with shape (N, num_anchors * 4, H, W) + gt_bboxes (list[Tensor]): Ground truth bboxes for each image with + shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + gt_bboxes_ignore (None | list[Tensor]): specify which bounding + boxes can be ignored when computing the loss. Default: None + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + featmap_sizes = [featmap.size()[-2:] for featmap in bbox_preds] + label_channels = self.cls_out_channels if self.use_sigmoid_cls else 1 + cls_reg_targets = self.get_targets( + anchor_list, + valid_flag_list, + gt_bboxes, + img_metas, + featmap_sizes, + gt_bboxes_ignore=gt_bboxes_ignore, + label_channels=label_channels) + if cls_reg_targets is None: + return None + (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, + num_total_pos, num_total_neg) = cls_reg_targets + if self.sampling: + num_total_samples = num_total_pos + num_total_neg + else: + # 200 is hard-coded average factor, + # which follows guided anchoring. + num_total_samples = sum([label.numel() + for label in labels_list]) / 200.0 + + # change per image, per level anchor_list to per_level, per_image + mlvl_anchor_list = list(zip(*anchor_list)) + # concat mlvl_anchor_list + mlvl_anchor_list = [ + torch.cat(anchors, dim=0) for anchors in mlvl_anchor_list + ] + + losses = multi_apply( + self.loss_single, + cls_scores, + bbox_preds, + mlvl_anchor_list, + labels_list, + label_weights_list, + bbox_targets_list, + bbox_weights_list, + num_total_samples=num_total_samples) + if self.with_cls: + return dict(loss_rpn_cls=losses[0], loss_rpn_reg=losses[1]) + return dict(loss_rpn_reg=losses[1]) + + def get_bboxes(self, + anchor_list, + cls_scores, + bbox_preds, + img_metas, + cfg, + rescale=False): + """Get proposal predict.""" + assert len(cls_scores) == len(bbox_preds) + num_levels = len(cls_scores) + + result_list = [] + for img_id in range(len(img_metas)): + cls_score_list = [ + cls_scores[i][img_id].detach() for i in range(num_levels) + ] + bbox_pred_list = [ + bbox_preds[i][img_id].detach() for i in range(num_levels) + ] + img_shape = img_metas[img_id]['img_shape'] + scale_factor = img_metas[img_id]['scale_factor'] + proposals = self._get_bboxes_single(cls_score_list, bbox_pred_list, + anchor_list[img_id], img_shape, + scale_factor, cfg, rescale) + result_list.append(proposals) + return result_list + + def refine_bboxes(self, anchor_list, bbox_preds, img_metas): + """Refine bboxes through stages.""" + num_levels = len(bbox_preds) + new_anchor_list = [] + for img_id in range(len(img_metas)): + mlvl_anchors = [] + for i in range(num_levels): + bbox_pred = bbox_preds[i][img_id].detach() + bbox_pred = bbox_pred.permute(1, 2, 0).reshape(-1, 4) + img_shape = img_metas[img_id]['img_shape'] + bboxes = self.bbox_coder.decode(anchor_list[img_id][i], + bbox_pred, img_shape) + mlvl_anchors.append(bboxes) + new_anchor_list.append(mlvl_anchors) + return new_anchor_list + + # TODO: temporary plan + def _get_bboxes_single(self, + cls_scores, + bbox_preds, + mlvl_anchors, + img_shape, + scale_factor, + cfg, + rescale=False): + """Transform outputs for a single batch item into bbox predictions. + + Args: + cls_scores (list[Tensor]): Box scores for each scale level + Has shape (num_anchors * num_classes, H, W). + bbox_preds (list[Tensor]): Box energies / deltas for each scale + level with shape (num_anchors * 4, H, W). + mlvl_anchors (list[Tensor]): Box reference for each scale level + with shape (num_total_anchors, 4). + img_shape (tuple[int]): Shape of the input image, + (height, width, 3). + scale_factor (ndarray): Scale factor of the image arange as + (w_scale, h_scale, w_scale, h_scale). + cfg (mmcv.Config): Test / postprocessing configuration, + if None, test_cfg would be used. + rescale (bool): If True, return boxes in original image space. + + Returns: + Tensor: Labeled boxes have the shape of (n,5), where the + first 4 columns are bounding box positions + (tl_x, tl_y, br_x, br_y) and the 5-th column is a score + between 0 and 1. + """ + cfg = self.test_cfg if cfg is None else cfg + cfg = copy.deepcopy(cfg) + # bboxes from different level should be independent during NMS, + # level_ids are used as labels for batched NMS to separate them + level_ids = [] + mlvl_scores = [] + mlvl_bbox_preds = [] + mlvl_valid_anchors = [] + for idx in range(len(cls_scores)): + rpn_cls_score = cls_scores[idx] + rpn_bbox_pred = bbox_preds[idx] + assert rpn_cls_score.size()[-2:] == rpn_bbox_pred.size()[-2:] + rpn_cls_score = rpn_cls_score.permute(1, 2, 0) + if self.use_sigmoid_cls: + rpn_cls_score = rpn_cls_score.reshape(-1) + scores = rpn_cls_score.sigmoid() + else: + rpn_cls_score = rpn_cls_score.reshape(-1, 2) + # We set FG labels to [0, num_class-1] and BG label to + # num_class in RPN head since mmdet v2.5, which is unified to + # be consistent with other head since mmdet v2.0. In mmdet v2.0 + # to v2.4 we keep BG label as 0 and FG label as 1 in rpn head. + scores = rpn_cls_score.softmax(dim=1)[:, 0] + rpn_bbox_pred = rpn_bbox_pred.permute(1, 2, 0).reshape(-1, 4) + anchors = mlvl_anchors[idx] + if cfg.nms_pre > 0 and scores.shape[0] > cfg.nms_pre: + # sort is faster than topk + # _, topk_inds = scores.topk(cfg.nms_pre) + if torch.onnx.is_in_onnx_export(): + # sort op will be converted to TopK in onnx + # and k<=3480 in TensorRT + _, topk_inds = scores.topk(cfg.nms_pre) + scores = scores[topk_inds] + else: + ranked_scores, rank_inds = scores.sort(descending=True) + topk_inds = rank_inds[:cfg.nms_pre] + scores = ranked_scores[:cfg.nms_pre] + rpn_bbox_pred = rpn_bbox_pred[topk_inds, :] + anchors = anchors[topk_inds, :] + mlvl_scores.append(scores) + mlvl_bbox_preds.append(rpn_bbox_pred) + mlvl_valid_anchors.append(anchors) + level_ids.append( + scores.new_full((scores.size(0), ), idx, dtype=torch.long)) + + scores = torch.cat(mlvl_scores) + anchors = torch.cat(mlvl_valid_anchors) + rpn_bbox_pred = torch.cat(mlvl_bbox_preds) + proposals = self.bbox_coder.decode( + anchors, rpn_bbox_pred, max_shape=img_shape) + ids = torch.cat(level_ids) + + # Skip nonzero op while exporting to ONNX + if cfg.min_bbox_size > 0 and (not torch.onnx.is_in_onnx_export()): + w = proposals[:, 2] - proposals[:, 0] + h = proposals[:, 3] - proposals[:, 1] + valid_inds = torch.nonzero( + (w >= cfg.min_bbox_size) + & (h >= cfg.min_bbox_size), + as_tuple=False).squeeze() + if valid_inds.sum().item() != len(proposals): + proposals = proposals[valid_inds, :] + scores = scores[valid_inds] + ids = ids[valid_inds] + + # deprecate arguments warning + if 'nms' not in cfg or 'max_num' in cfg or 'nms_thr' in cfg: + warnings.warn( + 'In rpn_proposal or test_cfg, ' + 'nms_thr has been moved to a dict named nms as ' + 'iou_threshold, max_num has been renamed as max_per_img, ' + 'name of original arguments and the way to specify ' + 'iou_threshold of NMS will be deprecated.') + if 'nms' not in cfg: + cfg.nms = ConfigDict(dict(type='nms', iou_threshold=cfg.nms_thr)) + if 'max_num' in cfg: + if 'max_per_img' in cfg: + assert cfg.max_num == cfg.max_per_img, f'You ' \ + f'set max_num and ' \ + f'max_per_img at the same time, but get {cfg.max_num} ' \ + f'and {cfg.max_per_img} respectively' \ + 'Please delete max_num which will be deprecated.' + else: + cfg.max_per_img = cfg.max_num + if 'nms_thr' in cfg: + assert cfg.nms.iou_threshold == cfg.nms_thr, f'You set' \ + f' iou_threshold in nms and ' \ + f'nms_thr at the same time, but get' \ + f' {cfg.nms.iou_threshold} and {cfg.nms_thr}' \ + f' respectively. Please delete the nms_thr ' \ + f'which will be deprecated.' + + dets, keep = batched_nms(proposals, scores, ids, cfg.nms) + return dets[:cfg.max_per_img] + + +@HEADS.register_module() +class CascadeRPNHead(BaseDenseHead): + """The CascadeRPNHead will predict more accurate region proposals, which is + required for two-stage detectors (such as Fast/Faster R-CNN). CascadeRPN + consists of a sequence of RPNStage to progressively improve the accuracy of + the detected proposals. + + More details can be found in ``https://arxiv.org/abs/1909.06720``. + + Args: + num_stages (int): number of CascadeRPN stages. + stages (list[dict]): list of configs to build the stages. + train_cfg (list[dict]): list of configs at training time each stage. + test_cfg (dict): config at testing time. + """ + + def __init__(self, num_stages, stages, train_cfg, test_cfg): + super(CascadeRPNHead, self).__init__() + assert num_stages == len(stages) + self.num_stages = num_stages + self.stages = nn.ModuleList() + for i in range(len(stages)): + train_cfg_i = train_cfg[i] if train_cfg is not None else None + stages[i].update(train_cfg=train_cfg_i) + stages[i].update(test_cfg=test_cfg) + self.stages.append(build_head(stages[i])) + self.train_cfg = train_cfg + self.test_cfg = test_cfg + + def init_weights(self): + """Init weight of CascadeRPN.""" + for i in range(self.num_stages): + self.stages[i].init_weights() + + def loss(self): + """loss() is implemented in StageCascadeRPNHead.""" + pass + + def get_bboxes(self): + """get_bboxes() is implemented in StageCascadeRPNHead.""" + pass + + def forward_train(self, + x, + img_metas, + gt_bboxes, + gt_labels=None, + gt_bboxes_ignore=None, + proposal_cfg=None): + """Forward train function.""" + assert gt_labels is None, 'RPN does not require gt_labels' + + featmap_sizes = [featmap.size()[-2:] for featmap in x] + device = x[0].device + anchor_list, valid_flag_list = self.stages[0].get_anchors( + featmap_sizes, img_metas, device=device) + + losses = dict() + + for i in range(self.num_stages): + stage = self.stages[i] + + if stage.adapt_cfg['type'] == 'offset': + offset_list = stage.anchor_offset(anchor_list, + stage.anchor_strides, + featmap_sizes) + else: + offset_list = None + x, cls_score, bbox_pred = stage(x, offset_list) + rpn_loss_inputs = (anchor_list, valid_flag_list, cls_score, + bbox_pred, gt_bboxes, img_metas) + stage_loss = stage.loss(*rpn_loss_inputs) + for name, value in stage_loss.items(): + losses['s{}.{}'.format(i, name)] = value + + # refine boxes + if i < self.num_stages - 1: + anchor_list = stage.refine_bboxes(anchor_list, bbox_pred, + img_metas) + if proposal_cfg is None: + return losses + else: + proposal_list = self.stages[-1].get_bboxes(anchor_list, cls_score, + bbox_pred, img_metas, + self.test_cfg) + return losses, proposal_list + + def simple_test_rpn(self, x, img_metas): + """Simple forward test function.""" + featmap_sizes = [featmap.size()[-2:] for featmap in x] + device = x[0].device + anchor_list, _ = self.stages[0].get_anchors( + featmap_sizes, img_metas, device=device) + + for i in range(self.num_stages): + stage = self.stages[i] + if stage.adapt_cfg['type'] == 'offset': + offset_list = stage.anchor_offset(anchor_list, + stage.anchor_strides, + featmap_sizes) + else: + offset_list = None + x, cls_score, bbox_pred = stage(x, offset_list) + if i < self.num_stages - 1: + anchor_list = stage.refine_bboxes(anchor_list, bbox_pred, + img_metas) + + proposal_list = self.stages[-1].get_bboxes(anchor_list, cls_score, + bbox_pred, img_metas, + self.test_cfg) + return proposal_list + + def aug_test_rpn(self, x, img_metas): + """Augmented forward test function.""" + raise NotImplementedError diff --git a/detection/mmdet/models/dense_heads/centripetal_head.py b/detection/mmdet/models/dense_heads/centripetal_head.py new file mode 100644 index 0000000..6728218 --- /dev/null +++ b/detection/mmdet/models/dense_heads/centripetal_head.py @@ -0,0 +1,421 @@ +import torch.nn as nn +from mmcv.cnn import ConvModule, normal_init +from mmcv.ops import DeformConv2d + +from mmdet.core import multi_apply +from ..builder import HEADS, build_loss +from .corner_head import CornerHead + + +@HEADS.register_module() +class CentripetalHead(CornerHead): + """Head of CentripetalNet: Pursuing High-quality Keypoint Pairs for Object + Detection. + + CentripetalHead inherits from :class:`CornerHead`. It removes the + embedding branch and adds guiding shift and centripetal shift branches. + More details can be found in the `paper + `_ . + + Args: + num_classes (int): Number of categories excluding the background + category. + in_channels (int): Number of channels in the input feature map. + num_feat_levels (int): Levels of feature from the previous module. 2 + for HourglassNet-104 and 1 for HourglassNet-52. HourglassNet-104 + outputs the final feature and intermediate supervision feature and + HourglassNet-52 only outputs the final feature. Default: 2. + corner_emb_channels (int): Channel of embedding vector. Default: 1. + train_cfg (dict | None): Training config. Useless in CornerHead, + but we keep this variable for SingleStageDetector. Default: None. + test_cfg (dict | None): Testing config of CornerHead. Default: None. + loss_heatmap (dict | None): Config of corner heatmap loss. Default: + GaussianFocalLoss. + loss_embedding (dict | None): Config of corner embedding loss. Default: + AssociativeEmbeddingLoss. + loss_offset (dict | None): Config of corner offset loss. Default: + SmoothL1Loss. + loss_guiding_shift (dict): Config of guiding shift loss. Default: + SmoothL1Loss. + loss_centripetal_shift (dict): Config of centripetal shift loss. + Default: SmoothL1Loss. + """ + + def __init__(self, + *args, + centripetal_shift_channels=2, + guiding_shift_channels=2, + feat_adaption_conv_kernel=3, + loss_guiding_shift=dict( + type='SmoothL1Loss', beta=1.0, loss_weight=0.05), + loss_centripetal_shift=dict( + type='SmoothL1Loss', beta=1.0, loss_weight=1), + **kwargs): + assert centripetal_shift_channels == 2, ( + 'CentripetalHead only support centripetal_shift_channels == 2') + self.centripetal_shift_channels = centripetal_shift_channels + assert guiding_shift_channels == 2, ( + 'CentripetalHead only support guiding_shift_channels == 2') + self.guiding_shift_channels = guiding_shift_channels + self.feat_adaption_conv_kernel = feat_adaption_conv_kernel + super(CentripetalHead, self).__init__(*args, **kwargs) + self.loss_guiding_shift = build_loss(loss_guiding_shift) + self.loss_centripetal_shift = build_loss(loss_centripetal_shift) + + def _init_centripetal_layers(self): + """Initialize centripetal layers. + + Including feature adaption deform convs (feat_adaption), deform offset + prediction convs (dcn_off), guiding shift (guiding_shift) and + centripetal shift ( centripetal_shift). Each branch has two parts: + prefix `tl_` for top-left and `br_` for bottom-right. + """ + self.tl_feat_adaption = nn.ModuleList() + self.br_feat_adaption = nn.ModuleList() + self.tl_dcn_offset = nn.ModuleList() + self.br_dcn_offset = nn.ModuleList() + self.tl_guiding_shift = nn.ModuleList() + self.br_guiding_shift = nn.ModuleList() + self.tl_centripetal_shift = nn.ModuleList() + self.br_centripetal_shift = nn.ModuleList() + + for _ in range(self.num_feat_levels): + self.tl_feat_adaption.append( + DeformConv2d(self.in_channels, self.in_channels, + self.feat_adaption_conv_kernel, 1, 1)) + self.br_feat_adaption.append( + DeformConv2d(self.in_channels, self.in_channels, + self.feat_adaption_conv_kernel, 1, 1)) + + self.tl_guiding_shift.append( + self._make_layers( + out_channels=self.guiding_shift_channels, + in_channels=self.in_channels)) + self.br_guiding_shift.append( + self._make_layers( + out_channels=self.guiding_shift_channels, + in_channels=self.in_channels)) + + self.tl_dcn_offset.append( + ConvModule( + self.guiding_shift_channels, + self.feat_adaption_conv_kernel**2 * + self.guiding_shift_channels, + 1, + bias=False, + act_cfg=None)) + self.br_dcn_offset.append( + ConvModule( + self.guiding_shift_channels, + self.feat_adaption_conv_kernel**2 * + self.guiding_shift_channels, + 1, + bias=False, + act_cfg=None)) + + self.tl_centripetal_shift.append( + self._make_layers( + out_channels=self.centripetal_shift_channels, + in_channels=self.in_channels)) + self.br_centripetal_shift.append( + self._make_layers( + out_channels=self.centripetal_shift_channels, + in_channels=self.in_channels)) + + def _init_layers(self): + """Initialize layers for CentripetalHead. + + Including two parts: CornerHead layers and CentripetalHead layers + """ + super()._init_layers() # using _init_layers in CornerHead + self._init_centripetal_layers() + + def init_weights(self): + """Initialize weights of the head.""" + super().init_weights() + for i in range(self.num_feat_levels): + normal_init(self.tl_feat_adaption[i], std=0.01) + normal_init(self.br_feat_adaption[i], std=0.01) + normal_init(self.tl_dcn_offset[i].conv, std=0.1) + normal_init(self.br_dcn_offset[i].conv, std=0.1) + _ = [x.conv.reset_parameters() for x in self.tl_guiding_shift[i]] + _ = [x.conv.reset_parameters() for x in self.br_guiding_shift[i]] + _ = [ + x.conv.reset_parameters() for x in self.tl_centripetal_shift[i] + ] + _ = [ + x.conv.reset_parameters() for x in self.br_centripetal_shift[i] + ] + + def forward_single(self, x, lvl_ind): + """Forward feature of a single level. + + Args: + x (Tensor): Feature of a single level. + lvl_ind (int): Level index of current feature. + + Returns: + tuple[Tensor]: A tuple of CentripetalHead's output for current + feature level. Containing the following Tensors: + + - tl_heat (Tensor): Predicted top-left corner heatmap. + - br_heat (Tensor): Predicted bottom-right corner heatmap. + - tl_off (Tensor): Predicted top-left offset heatmap. + - br_off (Tensor): Predicted bottom-right offset heatmap. + - tl_guiding_shift (Tensor): Predicted top-left guiding shift + heatmap. + - br_guiding_shift (Tensor): Predicted bottom-right guiding + shift heatmap. + - tl_centripetal_shift (Tensor): Predicted top-left centripetal + shift heatmap. + - br_centripetal_shift (Tensor): Predicted bottom-right + centripetal shift heatmap. + """ + tl_heat, br_heat, _, _, tl_off, br_off, tl_pool, br_pool = super( + ).forward_single( + x, lvl_ind, return_pool=True) + + tl_guiding_shift = self.tl_guiding_shift[lvl_ind](tl_pool) + br_guiding_shift = self.br_guiding_shift[lvl_ind](br_pool) + + tl_dcn_offset = self.tl_dcn_offset[lvl_ind](tl_guiding_shift.detach()) + br_dcn_offset = self.br_dcn_offset[lvl_ind](br_guiding_shift.detach()) + + tl_feat_adaption = self.tl_feat_adaption[lvl_ind](tl_pool, + tl_dcn_offset) + br_feat_adaption = self.br_feat_adaption[lvl_ind](br_pool, + br_dcn_offset) + + tl_centripetal_shift = self.tl_centripetal_shift[lvl_ind]( + tl_feat_adaption) + br_centripetal_shift = self.br_centripetal_shift[lvl_ind]( + br_feat_adaption) + + result_list = [ + tl_heat, br_heat, tl_off, br_off, tl_guiding_shift, + br_guiding_shift, tl_centripetal_shift, br_centripetal_shift + ] + return result_list + + def loss(self, + tl_heats, + br_heats, + tl_offs, + br_offs, + tl_guiding_shifts, + br_guiding_shifts, + tl_centripetal_shifts, + br_centripetal_shifts, + gt_bboxes, + gt_labels, + img_metas, + gt_bboxes_ignore=None): + """Compute losses of the head. + + Args: + tl_heats (list[Tensor]): Top-left corner heatmaps for each level + with shape (N, num_classes, H, W). + br_heats (list[Tensor]): Bottom-right corner heatmaps for each + level with shape (N, num_classes, H, W). + tl_offs (list[Tensor]): Top-left corner offsets for each level + with shape (N, corner_offset_channels, H, W). + br_offs (list[Tensor]): Bottom-right corner offsets for each level + with shape (N, corner_offset_channels, H, W). + tl_guiding_shifts (list[Tensor]): Top-left guiding shifts for each + level with shape (N, guiding_shift_channels, H, W). + br_guiding_shifts (list[Tensor]): Bottom-right guiding shifts for + each level with shape (N, guiding_shift_channels, H, W). + tl_centripetal_shifts (list[Tensor]): Top-left centripetal shifts + for each level with shape (N, centripetal_shift_channels, H, + W). + br_centripetal_shifts (list[Tensor]): Bottom-right centripetal + shifts for each level with shape (N, + centripetal_shift_channels, H, W). + gt_bboxes (list[Tensor]): Ground truth bboxes for each image with + shape (num_gts, 4) in [left, top, right, bottom] format. + gt_labels (list[Tensor]): Class indices corresponding to each box. + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + gt_bboxes_ignore (list[Tensor] | None): Specify which bounding + boxes can be ignored when computing the loss. + + Returns: + dict[str, Tensor]: A dictionary of loss components. Containing the + following losses: + + - det_loss (list[Tensor]): Corner keypoint losses of all + feature levels. + - off_loss (list[Tensor]): Corner offset losses of all feature + levels. + - guiding_loss (list[Tensor]): Guiding shift losses of all + feature levels. + - centripetal_loss (list[Tensor]): Centripetal shift losses of + all feature levels. + """ + targets = self.get_targets( + gt_bboxes, + gt_labels, + tl_heats[-1].shape, + img_metas[0]['pad_shape'], + with_corner_emb=self.with_corner_emb, + with_guiding_shift=True, + with_centripetal_shift=True) + mlvl_targets = [targets for _ in range(self.num_feat_levels)] + [det_losses, off_losses, guiding_losses, centripetal_losses + ] = multi_apply(self.loss_single, tl_heats, br_heats, tl_offs, + br_offs, tl_guiding_shifts, br_guiding_shifts, + tl_centripetal_shifts, br_centripetal_shifts, + mlvl_targets) + loss_dict = dict( + det_loss=det_losses, + off_loss=off_losses, + guiding_loss=guiding_losses, + centripetal_loss=centripetal_losses) + return loss_dict + + def loss_single(self, tl_hmp, br_hmp, tl_off, br_off, tl_guiding_shift, + br_guiding_shift, tl_centripetal_shift, + br_centripetal_shift, targets): + """Compute losses for single level. + + Args: + tl_hmp (Tensor): Top-left corner heatmap for current level with + shape (N, num_classes, H, W). + br_hmp (Tensor): Bottom-right corner heatmap for current level with + shape (N, num_classes, H, W). + tl_off (Tensor): Top-left corner offset for current level with + shape (N, corner_offset_channels, H, W). + br_off (Tensor): Bottom-right corner offset for current level with + shape (N, corner_offset_channels, H, W). + tl_guiding_shift (Tensor): Top-left guiding shift for current level + with shape (N, guiding_shift_channels, H, W). + br_guiding_shift (Tensor): Bottom-right guiding shift for current + level with shape (N, guiding_shift_channels, H, W). + tl_centripetal_shift (Tensor): Top-left centripetal shift for + current level with shape (N, centripetal_shift_channels, H, W). + br_centripetal_shift (Tensor): Bottom-right centripetal shift for + current level with shape (N, centripetal_shift_channels, H, W). + targets (dict): Corner target generated by `get_targets`. + + Returns: + tuple[torch.Tensor]: Losses of the head's differnet branches + containing the following losses: + + - det_loss (Tensor): Corner keypoint loss. + - off_loss (Tensor): Corner offset loss. + - guiding_loss (Tensor): Guiding shift loss. + - centripetal_loss (Tensor): Centripetal shift loss. + """ + targets['corner_embedding'] = None + + det_loss, _, _, off_loss = super().loss_single(tl_hmp, br_hmp, None, + None, tl_off, br_off, + targets) + + gt_tl_guiding_shift = targets['topleft_guiding_shift'] + gt_br_guiding_shift = targets['bottomright_guiding_shift'] + gt_tl_centripetal_shift = targets['topleft_centripetal_shift'] + gt_br_centripetal_shift = targets['bottomright_centripetal_shift'] + + gt_tl_heatmap = targets['topleft_heatmap'] + gt_br_heatmap = targets['bottomright_heatmap'] + # We only compute the offset loss at the real corner position. + # The value of real corner would be 1 in heatmap ground truth. + # The mask is computed in class agnostic mode and its shape is + # batch * 1 * width * height. + tl_mask = gt_tl_heatmap.eq(1).sum(1).gt(0).unsqueeze(1).type_as( + gt_tl_heatmap) + br_mask = gt_br_heatmap.eq(1).sum(1).gt(0).unsqueeze(1).type_as( + gt_br_heatmap) + + # Guiding shift loss + tl_guiding_loss = self.loss_guiding_shift( + tl_guiding_shift, + gt_tl_guiding_shift, + tl_mask, + avg_factor=tl_mask.sum()) + br_guiding_loss = self.loss_guiding_shift( + br_guiding_shift, + gt_br_guiding_shift, + br_mask, + avg_factor=br_mask.sum()) + guiding_loss = (tl_guiding_loss + br_guiding_loss) / 2.0 + # Centripetal shift loss + tl_centripetal_loss = self.loss_centripetal_shift( + tl_centripetal_shift, + gt_tl_centripetal_shift, + tl_mask, + avg_factor=tl_mask.sum()) + br_centripetal_loss = self.loss_centripetal_shift( + br_centripetal_shift, + gt_br_centripetal_shift, + br_mask, + avg_factor=br_mask.sum()) + centripetal_loss = (tl_centripetal_loss + br_centripetal_loss) / 2.0 + + return det_loss, off_loss, guiding_loss, centripetal_loss + + def get_bboxes(self, + tl_heats, + br_heats, + tl_offs, + br_offs, + tl_guiding_shifts, + br_guiding_shifts, + tl_centripetal_shifts, + br_centripetal_shifts, + img_metas, + rescale=False, + with_nms=True): + """Transform network output for a batch into bbox predictions. + + Args: + tl_heats (list[Tensor]): Top-left corner heatmaps for each level + with shape (N, num_classes, H, W). + br_heats (list[Tensor]): Bottom-right corner heatmaps for each + level with shape (N, num_classes, H, W). + tl_offs (list[Tensor]): Top-left corner offsets for each level + with shape (N, corner_offset_channels, H, W). + br_offs (list[Tensor]): Bottom-right corner offsets for each level + with shape (N, corner_offset_channels, H, W). + tl_guiding_shifts (list[Tensor]): Top-left guiding shifts for each + level with shape (N, guiding_shift_channels, H, W). Useless in + this function, we keep this arg because it's the raw output + from CentripetalHead. + br_guiding_shifts (list[Tensor]): Bottom-right guiding shifts for + each level with shape (N, guiding_shift_channels, H, W). + Useless in this function, we keep this arg because it's the + raw output from CentripetalHead. + tl_centripetal_shifts (list[Tensor]): Top-left centripetal shifts + for each level with shape (N, centripetal_shift_channels, H, + W). + br_centripetal_shifts (list[Tensor]): Bottom-right centripetal + shifts for each level with shape (N, + centripetal_shift_channels, H, W). + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + rescale (bool): If True, return boxes in original image space. + Default: False. + with_nms (bool): If True, do nms before return boxes. + Default: True. + """ + assert tl_heats[-1].shape[0] == br_heats[-1].shape[0] == len(img_metas) + result_list = [] + for img_id in range(len(img_metas)): + result_list.append( + self._get_bboxes_single( + tl_heats[-1][img_id:img_id + 1, :], + br_heats[-1][img_id:img_id + 1, :], + tl_offs[-1][img_id:img_id + 1, :], + br_offs[-1][img_id:img_id + 1, :], + img_metas[img_id], + tl_emb=None, + br_emb=None, + tl_centripetal_shift=tl_centripetal_shifts[-1][ + img_id:img_id + 1, :], + br_centripetal_shift=br_centripetal_shifts[-1][ + img_id:img_id + 1, :], + rescale=rescale, + with_nms=with_nms)) + + return result_list diff --git a/detection/mmdet/models/dense_heads/corner_head.py b/detection/mmdet/models/dense_heads/corner_head.py new file mode 100644 index 0000000..50cdb49 --- /dev/null +++ b/detection/mmdet/models/dense_heads/corner_head.py @@ -0,0 +1,1074 @@ +from logging import warning +from math import ceil, log + +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import ConvModule, bias_init_with_prob +from mmcv.ops import CornerPool, batched_nms + +from mmdet.core import multi_apply +from ..builder import HEADS, build_loss +from ..utils import gaussian_radius, gen_gaussian_target +from .base_dense_head import BaseDenseHead + + +class BiCornerPool(nn.Module): + """Bidirectional Corner Pooling Module (TopLeft, BottomRight, etc.) + + Args: + in_channels (int): Input channels of module. + out_channels (int): Output channels of module. + feat_channels (int): Feature channels of module. + directions (list[str]): Directions of two CornerPools. + norm_cfg (dict): Dictionary to construct and config norm layer. + """ + + def __init__(self, + in_channels, + directions, + feat_channels=128, + out_channels=128, + norm_cfg=dict(type='BN', requires_grad=True)): + super(BiCornerPool, self).__init__() + self.direction1_conv = ConvModule( + in_channels, feat_channels, 3, padding=1, norm_cfg=norm_cfg) + self.direction2_conv = ConvModule( + in_channels, feat_channels, 3, padding=1, norm_cfg=norm_cfg) + + self.aftpool_conv = ConvModule( + feat_channels, + out_channels, + 3, + padding=1, + norm_cfg=norm_cfg, + act_cfg=None) + + self.conv1 = ConvModule( + in_channels, out_channels, 1, norm_cfg=norm_cfg, act_cfg=None) + self.conv2 = ConvModule( + in_channels, out_channels, 3, padding=1, norm_cfg=norm_cfg) + + self.direction1_pool = CornerPool(directions[0]) + self.direction2_pool = CornerPool(directions[1]) + self.relu = nn.ReLU(inplace=True) + + def forward(self, x): + """Forward features from the upstream network. + + Args: + x (tensor): Input feature of BiCornerPool. + + Returns: + conv2 (tensor): Output feature of BiCornerPool. + """ + direction1_conv = self.direction1_conv(x) + direction2_conv = self.direction2_conv(x) + direction1_feat = self.direction1_pool(direction1_conv) + direction2_feat = self.direction2_pool(direction2_conv) + aftpool_conv = self.aftpool_conv(direction1_feat + direction2_feat) + conv1 = self.conv1(x) + relu = self.relu(aftpool_conv + conv1) + conv2 = self.conv2(relu) + return conv2 + + +@HEADS.register_module() +class CornerHead(BaseDenseHead): + """Head of CornerNet: Detecting Objects as Paired Keypoints. + + Code is modified from the `official github repo + `_ . + + More details can be found in the `paper + `_ . + + Args: + num_classes (int): Number of categories excluding the background + category. + in_channels (int): Number of channels in the input feature map. + num_feat_levels (int): Levels of feature from the previous module. 2 + for HourglassNet-104 and 1 for HourglassNet-52. Because + HourglassNet-104 outputs the final feature and intermediate + supervision feature and HourglassNet-52 only outputs the final + feature. Default: 2. + corner_emb_channels (int): Channel of embedding vector. Default: 1. + train_cfg (dict | None): Training config. Useless in CornerHead, + but we keep this variable for SingleStageDetector. Default: None. + test_cfg (dict | None): Testing config of CornerHead. Default: None. + loss_heatmap (dict | None): Config of corner heatmap loss. Default: + GaussianFocalLoss. + loss_embedding (dict | None): Config of corner embedding loss. Default: + AssociativeEmbeddingLoss. + loss_offset (dict | None): Config of corner offset loss. Default: + SmoothL1Loss. + """ + + def __init__(self, + num_classes, + in_channels, + num_feat_levels=2, + corner_emb_channels=1, + train_cfg=None, + test_cfg=None, + loss_heatmap=dict( + type='GaussianFocalLoss', + alpha=2.0, + gamma=4.0, + loss_weight=1), + loss_embedding=dict( + type='AssociativeEmbeddingLoss', + pull_weight=0.25, + push_weight=0.25), + loss_offset=dict( + type='SmoothL1Loss', beta=1.0, loss_weight=1)): + super(CornerHead, self).__init__() + self.num_classes = num_classes + self.in_channels = in_channels + self.corner_emb_channels = corner_emb_channels + self.with_corner_emb = self.corner_emb_channels > 0 + self.corner_offset_channels = 2 + self.num_feat_levels = num_feat_levels + self.loss_heatmap = build_loss( + loss_heatmap) if loss_heatmap is not None else None + self.loss_embedding = build_loss( + loss_embedding) if loss_embedding is not None else None + self.loss_offset = build_loss( + loss_offset) if loss_offset is not None else None + self.train_cfg = train_cfg + self.test_cfg = test_cfg + + self._init_layers() + + def _make_layers(self, out_channels, in_channels=256, feat_channels=256): + """Initialize conv sequential for CornerHead.""" + return nn.Sequential( + ConvModule(in_channels, feat_channels, 3, padding=1), + ConvModule( + feat_channels, out_channels, 1, norm_cfg=None, act_cfg=None)) + + def _init_corner_kpt_layers(self): + """Initialize corner keypoint layers. + + Including corner heatmap branch and corner offset branch. Each branch + has two parts: prefix `tl_` for top-left and `br_` for bottom-right. + """ + self.tl_pool, self.br_pool = nn.ModuleList(), nn.ModuleList() + self.tl_heat, self.br_heat = nn.ModuleList(), nn.ModuleList() + self.tl_off, self.br_off = nn.ModuleList(), nn.ModuleList() + + for _ in range(self.num_feat_levels): + self.tl_pool.append( + BiCornerPool( + self.in_channels, ['top', 'left'], + out_channels=self.in_channels)) + self.br_pool.append( + BiCornerPool( + self.in_channels, ['bottom', 'right'], + out_channels=self.in_channels)) + + self.tl_heat.append( + self._make_layers( + out_channels=self.num_classes, + in_channels=self.in_channels)) + self.br_heat.append( + self._make_layers( + out_channels=self.num_classes, + in_channels=self.in_channels)) + + self.tl_off.append( + self._make_layers( + out_channels=self.corner_offset_channels, + in_channels=self.in_channels)) + self.br_off.append( + self._make_layers( + out_channels=self.corner_offset_channels, + in_channels=self.in_channels)) + + def _init_corner_emb_layers(self): + """Initialize corner embedding layers. + + Only include corner embedding branch with two parts: prefix `tl_` for + top-left and `br_` for bottom-right. + """ + self.tl_emb, self.br_emb = nn.ModuleList(), nn.ModuleList() + + for _ in range(self.num_feat_levels): + self.tl_emb.append( + self._make_layers( + out_channels=self.corner_emb_channels, + in_channels=self.in_channels)) + self.br_emb.append( + self._make_layers( + out_channels=self.corner_emb_channels, + in_channels=self.in_channels)) + + def _init_layers(self): + """Initialize layers for CornerHead. + + Including two parts: corner keypoint layers and corner embedding layers + """ + self._init_corner_kpt_layers() + if self.with_corner_emb: + self._init_corner_emb_layers() + + def init_weights(self): + """Initialize weights of the head.""" + bias_init = bias_init_with_prob(0.1) + for i in range(self.num_feat_levels): + # The initialization of parameters are different between nn.Conv2d + # and ConvModule. Our experiments show that using the original + # initialization of nn.Conv2d increases the final mAP by about 0.2% + self.tl_heat[i][-1].conv.reset_parameters() + self.tl_heat[i][-1].conv.bias.data.fill_(bias_init) + self.br_heat[i][-1].conv.reset_parameters() + self.br_heat[i][-1].conv.bias.data.fill_(bias_init) + self.tl_off[i][-1].conv.reset_parameters() + self.br_off[i][-1].conv.reset_parameters() + if self.with_corner_emb: + self.tl_emb[i][-1].conv.reset_parameters() + self.br_emb[i][-1].conv.reset_parameters() + + def forward(self, feats): + """Forward features from the upstream network. + + Args: + feats (tuple[Tensor]): Features from the upstream network, each is + a 4D-tensor. + + Returns: + tuple: Usually a tuple of corner heatmaps, offset heatmaps and + embedding heatmaps. + - tl_heats (list[Tensor]): Top-left corner heatmaps for all + levels, each is a 4D-tensor, the channels number is + num_classes. + - br_heats (list[Tensor]): Bottom-right corner heatmaps for all + levels, each is a 4D-tensor, the channels number is + num_classes. + - tl_embs (list[Tensor] | list[None]): Top-left embedding + heatmaps for all levels, each is a 4D-tensor or None. + If not None, the channels number is corner_emb_channels. + - br_embs (list[Tensor] | list[None]): Bottom-right embedding + heatmaps for all levels, each is a 4D-tensor or None. + If not None, the channels number is corner_emb_channels. + - tl_offs (list[Tensor]): Top-left offset heatmaps for all + levels, each is a 4D-tensor. The channels number is + corner_offset_channels. + - br_offs (list[Tensor]): Bottom-right offset heatmaps for all + levels, each is a 4D-tensor. The channels number is + corner_offset_channels. + """ + lvl_ind = list(range(self.num_feat_levels)) + return multi_apply(self.forward_single, feats, lvl_ind) + + def forward_single(self, x, lvl_ind, return_pool=False): + """Forward feature of a single level. + + Args: + x (Tensor): Feature of a single level. + lvl_ind (int): Level index of current feature. + return_pool (bool): Return corner pool feature or not. + + Returns: + tuple[Tensor]: A tuple of CornerHead's output for current feature + level. Containing the following Tensors: + + - tl_heat (Tensor): Predicted top-left corner heatmap. + - br_heat (Tensor): Predicted bottom-right corner heatmap. + - tl_emb (Tensor | None): Predicted top-left embedding heatmap. + None for `self.with_corner_emb == False`. + - br_emb (Tensor | None): Predicted bottom-right embedding + heatmap. None for `self.with_corner_emb == False`. + - tl_off (Tensor): Predicted top-left offset heatmap. + - br_off (Tensor): Predicted bottom-right offset heatmap. + - tl_pool (Tensor): Top-left corner pool feature. Not must + have. + - br_pool (Tensor): Bottom-right corner pool feature. Not must + have. + """ + tl_pool = self.tl_pool[lvl_ind](x) + tl_heat = self.tl_heat[lvl_ind](tl_pool) + br_pool = self.br_pool[lvl_ind](x) + br_heat = self.br_heat[lvl_ind](br_pool) + + tl_emb, br_emb = None, None + if self.with_corner_emb: + tl_emb = self.tl_emb[lvl_ind](tl_pool) + br_emb = self.br_emb[lvl_ind](br_pool) + + tl_off = self.tl_off[lvl_ind](tl_pool) + br_off = self.br_off[lvl_ind](br_pool) + + result_list = [tl_heat, br_heat, tl_emb, br_emb, tl_off, br_off] + if return_pool: + result_list.append(tl_pool) + result_list.append(br_pool) + + return result_list + + def get_targets(self, + gt_bboxes, + gt_labels, + feat_shape, + img_shape, + with_corner_emb=False, + with_guiding_shift=False, + with_centripetal_shift=False): + """Generate corner targets. + + Including corner heatmap, corner offset. + + Optional: corner embedding, corner guiding shift, centripetal shift. + + For CornerNet, we generate corner heatmap, corner offset and corner + embedding from this function. + + For CentripetalNet, we generate corner heatmap, corner offset, guiding + shift and centripetal shift from this function. + + Args: + gt_bboxes (list[Tensor]): Ground truth bboxes of each image, each + has shape (num_gt, 4). + gt_labels (list[Tensor]): Ground truth labels of each box, each has + shape (num_gt,). + feat_shape (list[int]): Shape of output feature, + [batch, channel, height, width]. + img_shape (list[int]): Shape of input image, + [height, width, channel]. + with_corner_emb (bool): Generate corner embedding target or not. + Default: False. + with_guiding_shift (bool): Generate guiding shift target or not. + Default: False. + with_centripetal_shift (bool): Generate centripetal shift target or + not. Default: False. + + Returns: + dict: Ground truth of corner heatmap, corner offset, corner + embedding, guiding shift and centripetal shift. Containing the + following keys: + + - topleft_heatmap (Tensor): Ground truth top-left corner + heatmap. + - bottomright_heatmap (Tensor): Ground truth bottom-right + corner heatmap. + - topleft_offset (Tensor): Ground truth top-left corner offset. + - bottomright_offset (Tensor): Ground truth bottom-right corner + offset. + - corner_embedding (list[list[list[int]]]): Ground truth corner + embedding. Not must have. + - topleft_guiding_shift (Tensor): Ground truth top-left corner + guiding shift. Not must have. + - bottomright_guiding_shift (Tensor): Ground truth bottom-right + corner guiding shift. Not must have. + - topleft_centripetal_shift (Tensor): Ground truth top-left + corner centripetal shift. Not must have. + - bottomright_centripetal_shift (Tensor): Ground truth + bottom-right corner centripetal shift. Not must have. + """ + batch_size, _, height, width = feat_shape + img_h, img_w = img_shape[:2] + + width_ratio = float(width / img_w) + height_ratio = float(height / img_h) + + gt_tl_heatmap = gt_bboxes[-1].new_zeros( + [batch_size, self.num_classes, height, width]) + gt_br_heatmap = gt_bboxes[-1].new_zeros( + [batch_size, self.num_classes, height, width]) + gt_tl_offset = gt_bboxes[-1].new_zeros([batch_size, 2, height, width]) + gt_br_offset = gt_bboxes[-1].new_zeros([batch_size, 2, height, width]) + + if with_corner_emb: + match = [] + + # Guiding shift is a kind of offset, from center to corner + if with_guiding_shift: + gt_tl_guiding_shift = gt_bboxes[-1].new_zeros( + [batch_size, 2, height, width]) + gt_br_guiding_shift = gt_bboxes[-1].new_zeros( + [batch_size, 2, height, width]) + # Centripetal shift is also a kind of offset, from center to corner + # and normalized by log. + if with_centripetal_shift: + gt_tl_centripetal_shift = gt_bboxes[-1].new_zeros( + [batch_size, 2, height, width]) + gt_br_centripetal_shift = gt_bboxes[-1].new_zeros( + [batch_size, 2, height, width]) + + for batch_id in range(batch_size): + # Ground truth of corner embedding per image is a list of coord set + corner_match = [] + for box_id in range(len(gt_labels[batch_id])): + left, top, right, bottom = gt_bboxes[batch_id][box_id] + center_x = (left + right) / 2.0 + center_y = (top + bottom) / 2.0 + label = gt_labels[batch_id][box_id] + + # Use coords in the feature level to generate ground truth + scale_left = left * width_ratio + scale_right = right * width_ratio + scale_top = top * height_ratio + scale_bottom = bottom * height_ratio + scale_center_x = center_x * width_ratio + scale_center_y = center_y * height_ratio + + # Int coords on feature map/ground truth tensor + left_idx = int(min(scale_left, width - 1)) + right_idx = int(min(scale_right, width - 1)) + top_idx = int(min(scale_top, height - 1)) + bottom_idx = int(min(scale_bottom, height - 1)) + + # Generate gaussian heatmap + scale_box_width = ceil(scale_right - scale_left) + scale_box_height = ceil(scale_bottom - scale_top) + radius = gaussian_radius((scale_box_height, scale_box_width), + min_overlap=0.3) + radius = max(0, int(radius)) + gt_tl_heatmap[batch_id, label] = gen_gaussian_target( + gt_tl_heatmap[batch_id, label], [left_idx, top_idx], + radius) + gt_br_heatmap[batch_id, label] = gen_gaussian_target( + gt_br_heatmap[batch_id, label], [right_idx, bottom_idx], + radius) + + # Generate corner offset + left_offset = scale_left - left_idx + top_offset = scale_top - top_idx + right_offset = scale_right - right_idx + bottom_offset = scale_bottom - bottom_idx + gt_tl_offset[batch_id, 0, top_idx, left_idx] = left_offset + gt_tl_offset[batch_id, 1, top_idx, left_idx] = top_offset + gt_br_offset[batch_id, 0, bottom_idx, right_idx] = right_offset + gt_br_offset[batch_id, 1, bottom_idx, + right_idx] = bottom_offset + + # Generate corner embedding + if with_corner_emb: + corner_match.append([[top_idx, left_idx], + [bottom_idx, right_idx]]) + # Generate guiding shift + if with_guiding_shift: + gt_tl_guiding_shift[batch_id, 0, top_idx, + left_idx] = scale_center_x - left_idx + gt_tl_guiding_shift[batch_id, 1, top_idx, + left_idx] = scale_center_y - top_idx + gt_br_guiding_shift[batch_id, 0, bottom_idx, + right_idx] = right_idx - scale_center_x + gt_br_guiding_shift[ + batch_id, 1, bottom_idx, + right_idx] = bottom_idx - scale_center_y + # Generate centripetal shift + if with_centripetal_shift: + gt_tl_centripetal_shift[batch_id, 0, top_idx, + left_idx] = log(scale_center_x - + scale_left) + gt_tl_centripetal_shift[batch_id, 1, top_idx, + left_idx] = log(scale_center_y - + scale_top) + gt_br_centripetal_shift[batch_id, 0, bottom_idx, + right_idx] = log(scale_right - + scale_center_x) + gt_br_centripetal_shift[batch_id, 1, bottom_idx, + right_idx] = log(scale_bottom - + scale_center_y) + + if with_corner_emb: + match.append(corner_match) + + target_result = dict( + topleft_heatmap=gt_tl_heatmap, + topleft_offset=gt_tl_offset, + bottomright_heatmap=gt_br_heatmap, + bottomright_offset=gt_br_offset) + + if with_corner_emb: + target_result.update(corner_embedding=match) + if with_guiding_shift: + target_result.update( + topleft_guiding_shift=gt_tl_guiding_shift, + bottomright_guiding_shift=gt_br_guiding_shift) + if with_centripetal_shift: + target_result.update( + topleft_centripetal_shift=gt_tl_centripetal_shift, + bottomright_centripetal_shift=gt_br_centripetal_shift) + + return target_result + + def loss(self, + tl_heats, + br_heats, + tl_embs, + br_embs, + tl_offs, + br_offs, + gt_bboxes, + gt_labels, + img_metas, + gt_bboxes_ignore=None): + """Compute losses of the head. + + Args: + tl_heats (list[Tensor]): Top-left corner heatmaps for each level + with shape (N, num_classes, H, W). + br_heats (list[Tensor]): Bottom-right corner heatmaps for each + level with shape (N, num_classes, H, W). + tl_embs (list[Tensor]): Top-left corner embeddings for each level + with shape (N, corner_emb_channels, H, W). + br_embs (list[Tensor]): Bottom-right corner embeddings for each + level with shape (N, corner_emb_channels, H, W). + tl_offs (list[Tensor]): Top-left corner offsets for each level + with shape (N, corner_offset_channels, H, W). + br_offs (list[Tensor]): Bottom-right corner offsets for each level + with shape (N, corner_offset_channels, H, W). + gt_bboxes (list[Tensor]): Ground truth bboxes for each image with + shape (num_gts, 4) in [left, top, right, bottom] format. + gt_labels (list[Tensor]): Class indices corresponding to each box. + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + gt_bboxes_ignore (list[Tensor] | None): Specify which bounding + boxes can be ignored when computing the loss. + + Returns: + dict[str, Tensor]: A dictionary of loss components. Containing the + following losses: + + - det_loss (list[Tensor]): Corner keypoint losses of all + feature levels. + - pull_loss (list[Tensor]): Part one of AssociativeEmbedding + losses of all feature levels. + - push_loss (list[Tensor]): Part two of AssociativeEmbedding + losses of all feature levels. + - off_loss (list[Tensor]): Corner offset losses of all feature + levels. + """ + targets = self.get_targets( + gt_bboxes, + gt_labels, + tl_heats[-1].shape, + img_metas[0]['pad_shape'], + with_corner_emb=self.with_corner_emb) + mlvl_targets = [targets for _ in range(self.num_feat_levels)] + det_losses, pull_losses, push_losses, off_losses = multi_apply( + self.loss_single, tl_heats, br_heats, tl_embs, br_embs, tl_offs, + br_offs, mlvl_targets) + loss_dict = dict(det_loss=det_losses, off_loss=off_losses) + if self.with_corner_emb: + loss_dict.update(pull_loss=pull_losses, push_loss=push_losses) + return loss_dict + + def loss_single(self, tl_hmp, br_hmp, tl_emb, br_emb, tl_off, br_off, + targets): + """Compute losses for single level. + + Args: + tl_hmp (Tensor): Top-left corner heatmap for current level with + shape (N, num_classes, H, W). + br_hmp (Tensor): Bottom-right corner heatmap for current level with + shape (N, num_classes, H, W). + tl_emb (Tensor): Top-left corner embedding for current level with + shape (N, corner_emb_channels, H, W). + br_emb (Tensor): Bottom-right corner embedding for current level + with shape (N, corner_emb_channels, H, W). + tl_off (Tensor): Top-left corner offset for current level with + shape (N, corner_offset_channels, H, W). + br_off (Tensor): Bottom-right corner offset for current level with + shape (N, corner_offset_channels, H, W). + targets (dict): Corner target generated by `get_targets`. + + Returns: + tuple[torch.Tensor]: Losses of the head's differnet branches + containing the following losses: + + - det_loss (Tensor): Corner keypoint loss. + - pull_loss (Tensor): Part one of AssociativeEmbedding loss. + - push_loss (Tensor): Part two of AssociativeEmbedding loss. + - off_loss (Tensor): Corner offset loss. + """ + gt_tl_hmp = targets['topleft_heatmap'] + gt_br_hmp = targets['bottomright_heatmap'] + gt_tl_off = targets['topleft_offset'] + gt_br_off = targets['bottomright_offset'] + gt_embedding = targets['corner_embedding'] + + # Detection loss + tl_det_loss = self.loss_heatmap( + tl_hmp.sigmoid(), + gt_tl_hmp, + avg_factor=max(1, + gt_tl_hmp.eq(1).sum())) + br_det_loss = self.loss_heatmap( + br_hmp.sigmoid(), + gt_br_hmp, + avg_factor=max(1, + gt_br_hmp.eq(1).sum())) + det_loss = (tl_det_loss + br_det_loss) / 2.0 + + # AssociativeEmbedding loss + if self.with_corner_emb and self.loss_embedding is not None: + pull_loss, push_loss = self.loss_embedding(tl_emb, br_emb, + gt_embedding) + else: + pull_loss, push_loss = None, None + + # Offset loss + # We only compute the offset loss at the real corner position. + # The value of real corner would be 1 in heatmap ground truth. + # The mask is computed in class agnostic mode and its shape is + # batch * 1 * width * height. + tl_off_mask = gt_tl_hmp.eq(1).sum(1).gt(0).unsqueeze(1).type_as( + gt_tl_hmp) + br_off_mask = gt_br_hmp.eq(1).sum(1).gt(0).unsqueeze(1).type_as( + gt_br_hmp) + tl_off_loss = self.loss_offset( + tl_off, + gt_tl_off, + tl_off_mask, + avg_factor=max(1, tl_off_mask.sum())) + br_off_loss = self.loss_offset( + br_off, + gt_br_off, + br_off_mask, + avg_factor=max(1, br_off_mask.sum())) + + off_loss = (tl_off_loss + br_off_loss) / 2.0 + + return det_loss, pull_loss, push_loss, off_loss + + def get_bboxes(self, + tl_heats, + br_heats, + tl_embs, + br_embs, + tl_offs, + br_offs, + img_metas, + rescale=False, + with_nms=True): + """Transform network output for a batch into bbox predictions. + + Args: + tl_heats (list[Tensor]): Top-left corner heatmaps for each level + with shape (N, num_classes, H, W). + br_heats (list[Tensor]): Bottom-right corner heatmaps for each + level with shape (N, num_classes, H, W). + tl_embs (list[Tensor]): Top-left corner embeddings for each level + with shape (N, corner_emb_channels, H, W). + br_embs (list[Tensor]): Bottom-right corner embeddings for each + level with shape (N, corner_emb_channels, H, W). + tl_offs (list[Tensor]): Top-left corner offsets for each level + with shape (N, corner_offset_channels, H, W). + br_offs (list[Tensor]): Bottom-right corner offsets for each level + with shape (N, corner_offset_channels, H, W). + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + rescale (bool): If True, return boxes in original image space. + Default: False. + with_nms (bool): If True, do nms before return boxes. + Default: True. + """ + assert tl_heats[-1].shape[0] == br_heats[-1].shape[0] == len(img_metas) + result_list = [] + for img_id in range(len(img_metas)): + result_list.append( + self._get_bboxes_single( + tl_heats[-1][img_id:img_id + 1, :], + br_heats[-1][img_id:img_id + 1, :], + tl_offs[-1][img_id:img_id + 1, :], + br_offs[-1][img_id:img_id + 1, :], + img_metas[img_id], + tl_emb=tl_embs[-1][img_id:img_id + 1, :], + br_emb=br_embs[-1][img_id:img_id + 1, :], + rescale=rescale, + with_nms=with_nms)) + + return result_list + + def _get_bboxes_single(self, + tl_heat, + br_heat, + tl_off, + br_off, + img_meta, + tl_emb=None, + br_emb=None, + tl_centripetal_shift=None, + br_centripetal_shift=None, + rescale=False, + with_nms=True): + """Transform outputs for a single batch item into bbox predictions. + + Args: + tl_heat (Tensor): Top-left corner heatmap for current level with + shape (N, num_classes, H, W). + br_heat (Tensor): Bottom-right corner heatmap for current level + with shape (N, num_classes, H, W). + tl_off (Tensor): Top-left corner offset for current level with + shape (N, corner_offset_channels, H, W). + br_off (Tensor): Bottom-right corner offset for current level with + shape (N, corner_offset_channels, H, W). + img_meta (dict): Meta information of current image, e.g., + image size, scaling factor, etc. + tl_emb (Tensor): Top-left corner embedding for current level with + shape (N, corner_emb_channels, H, W). + br_emb (Tensor): Bottom-right corner embedding for current level + with shape (N, corner_emb_channels, H, W). + tl_centripetal_shift: Top-left corner's centripetal shift for + current level with shape (N, 2, H, W). + br_centripetal_shift: Bottom-right corner's centripetal shift for + current level with shape (N, 2, H, W). + rescale (bool): If True, return boxes in original image space. + Default: False. + with_nms (bool): If True, do nms before return boxes. + Default: True. + """ + if isinstance(img_meta, (list, tuple)): + img_meta = img_meta[0] + + batch_bboxes, batch_scores, batch_clses = self.decode_heatmap( + tl_heat=tl_heat.sigmoid(), + br_heat=br_heat.sigmoid(), + tl_off=tl_off, + br_off=br_off, + tl_emb=tl_emb, + br_emb=br_emb, + tl_centripetal_shift=tl_centripetal_shift, + br_centripetal_shift=br_centripetal_shift, + img_meta=img_meta, + k=self.test_cfg.corner_topk, + kernel=self.test_cfg.local_maximum_kernel, + distance_threshold=self.test_cfg.distance_threshold) + + if rescale: + batch_bboxes /= batch_bboxes.new_tensor(img_meta['scale_factor']) + + bboxes = batch_bboxes.view([-1, 4]) + scores = batch_scores.view([-1, 1]) + clses = batch_clses.view([-1, 1]) + + idx = scores.argsort(dim=0, descending=True) + bboxes = bboxes[idx].view([-1, 4]) + scores = scores[idx].view(-1) + clses = clses[idx].view(-1) + + detections = torch.cat([bboxes, scores.unsqueeze(-1)], -1) + keepinds = (detections[:, -1] > -0.1) + detections = detections[keepinds] + labels = clses[keepinds] + + if with_nms: + detections, labels = self._bboxes_nms(detections, labels, + self.test_cfg) + + return detections, labels + + def _bboxes_nms(self, bboxes, labels, cfg): + if labels.numel() == 0: + return bboxes, labels + + if 'nms_cfg' in cfg: + warning.warn('nms_cfg in test_cfg will be deprecated. ' + 'Please rename it as nms') + if 'nms' not in cfg: + cfg.nms = cfg.nms_cfg + + out_bboxes, keep = batched_nms(bboxes[:, :4], bboxes[:, -1], labels, + cfg.nms) + out_labels = labels[keep] + + if len(out_bboxes) > 0: + idx = torch.argsort(out_bboxes[:, -1], descending=True) + idx = idx[:cfg.max_per_img] + out_bboxes = out_bboxes[idx] + out_labels = out_labels[idx] + + return out_bboxes, out_labels + + def _gather_feat(self, feat, ind, mask=None): + """Gather feature according to index. + + Args: + feat (Tensor): Target feature map. + ind (Tensor): Target coord index. + mask (Tensor | None): Mask of featuremap. Default: None. + + Returns: + feat (Tensor): Gathered feature. + """ + dim = feat.size(2) + ind = ind.unsqueeze(2).repeat(1, 1, dim) + feat = feat.gather(1, ind) + if mask is not None: + mask = mask.unsqueeze(2).expand_as(feat) + feat = feat[mask] + feat = feat.view(-1, dim) + return feat + + def _local_maximum(self, heat, kernel=3): + """Extract local maximum pixel with given kernel. + + Args: + heat (Tensor): Target heatmap. + kernel (int): Kernel size of max pooling. Default: 3. + + Returns: + heat (Tensor): A heatmap where local maximum pixels maintain its + own value and other positions are 0. + """ + pad = (kernel - 1) // 2 + hmax = F.max_pool2d(heat, kernel, stride=1, padding=pad) + keep = (hmax == heat).float() + return heat * keep + + def _transpose_and_gather_feat(self, feat, ind): + """Transpose and gather feature according to index. + + Args: + feat (Tensor): Target feature map. + ind (Tensor): Target coord index. + + Returns: + feat (Tensor): Transposed and gathered feature. + """ + feat = feat.permute(0, 2, 3, 1).contiguous() + feat = feat.view(feat.size(0), -1, feat.size(3)) + feat = self._gather_feat(feat, ind) + return feat + + def _topk(self, scores, k=20): + """Get top k positions from heatmap. + + Args: + scores (Tensor): Target heatmap with shape + [batch, num_classes, height, width]. + k (int): Target number. Default: 20. + + Returns: + tuple[torch.Tensor]: Scores, indexes, categories and coords of + topk keypoint. Containing following Tensors: + + - topk_scores (Tensor): Max scores of each topk keypoint. + - topk_inds (Tensor): Indexes of each topk keypoint. + - topk_clses (Tensor): Categories of each topk keypoint. + - topk_ys (Tensor): Y-coord of each topk keypoint. + - topk_xs (Tensor): X-coord of each topk keypoint. + """ + batch, _, height, width = scores.size() + topk_scores, topk_inds = torch.topk(scores.view(batch, -1), k) + topk_clses = topk_inds // (height * width) + topk_inds = topk_inds % (height * width) + topk_ys = topk_inds // width + topk_xs = (topk_inds % width).int().float() + return topk_scores, topk_inds, topk_clses, topk_ys, topk_xs + + def decode_heatmap(self, + tl_heat, + br_heat, + tl_off, + br_off, + tl_emb=None, + br_emb=None, + tl_centripetal_shift=None, + br_centripetal_shift=None, + img_meta=None, + k=100, + kernel=3, + distance_threshold=0.5, + num_dets=1000): + """Transform outputs for a single batch item into raw bbox predictions. + + Args: + tl_heat (Tensor): Top-left corner heatmap for current level with + shape (N, num_classes, H, W). + br_heat (Tensor): Bottom-right corner heatmap for current level + with shape (N, num_classes, H, W). + tl_off (Tensor): Top-left corner offset for current level with + shape (N, corner_offset_channels, H, W). + br_off (Tensor): Bottom-right corner offset for current level with + shape (N, corner_offset_channels, H, W). + tl_emb (Tensor | None): Top-left corner embedding for current + level with shape (N, corner_emb_channels, H, W). + br_emb (Tensor | None): Bottom-right corner embedding for current + level with shape (N, corner_emb_channels, H, W). + tl_centripetal_shift (Tensor | None): Top-left centripetal shift + for current level with shape (N, 2, H, W). + br_centripetal_shift (Tensor | None): Bottom-right centripetal + shift for current level with shape (N, 2, H, W). + img_meta (dict): Meta information of current image, e.g., + image size, scaling factor, etc. + k (int): Get top k corner keypoints from heatmap. + kernel (int): Max pooling kernel for extract local maximum pixels. + distance_threshold (float): Distance threshold. Top-left and + bottom-right corner keypoints with feature distance less than + the threshold will be regarded as keypoints from same object. + num_dets (int): Num of raw boxes before doing nms. + + Returns: + tuple[torch.Tensor]: Decoded output of CornerHead, containing the + following Tensors: + + - bboxes (Tensor): Coords of each box. + - scores (Tensor): Scores of each box. + - clses (Tensor): Categories of each box. + """ + with_embedding = tl_emb is not None and br_emb is not None + with_centripetal_shift = ( + tl_centripetal_shift is not None + and br_centripetal_shift is not None) + assert with_embedding + with_centripetal_shift == 1 + batch, _, height, width = tl_heat.size() + inp_h, inp_w, _ = img_meta['pad_shape'] + + # perform nms on heatmaps + tl_heat = self._local_maximum(tl_heat, kernel=kernel) + br_heat = self._local_maximum(br_heat, kernel=kernel) + + tl_scores, tl_inds, tl_clses, tl_ys, tl_xs = self._topk(tl_heat, k=k) + br_scores, br_inds, br_clses, br_ys, br_xs = self._topk(br_heat, k=k) + + # We use repeat instead of expand here because expand is a + # shallow-copy function. Thus it could cause unexpected testing result + # sometimes. Using expand will decrease about 10% mAP during testing + # compared to repeat. + tl_ys = tl_ys.view(batch, k, 1).repeat(1, 1, k) + tl_xs = tl_xs.view(batch, k, 1).repeat(1, 1, k) + br_ys = br_ys.view(batch, 1, k).repeat(1, k, 1) + br_xs = br_xs.view(batch, 1, k).repeat(1, k, 1) + + tl_off = self._transpose_and_gather_feat(tl_off, tl_inds) + tl_off = tl_off.view(batch, k, 1, 2) + br_off = self._transpose_and_gather_feat(br_off, br_inds) + br_off = br_off.view(batch, 1, k, 2) + + tl_xs = tl_xs + tl_off[..., 0] + tl_ys = tl_ys + tl_off[..., 1] + br_xs = br_xs + br_off[..., 0] + br_ys = br_ys + br_off[..., 1] + + if with_centripetal_shift: + tl_centripetal_shift = self._transpose_and_gather_feat( + tl_centripetal_shift, tl_inds).view(batch, k, 1, 2).exp() + br_centripetal_shift = self._transpose_and_gather_feat( + br_centripetal_shift, br_inds).view(batch, 1, k, 2).exp() + + tl_ctxs = tl_xs + tl_centripetal_shift[..., 0] + tl_ctys = tl_ys + tl_centripetal_shift[..., 1] + br_ctxs = br_xs - br_centripetal_shift[..., 0] + br_ctys = br_ys - br_centripetal_shift[..., 1] + + # all possible boxes based on top k corners (ignoring class) + tl_xs *= (inp_w / width) + tl_ys *= (inp_h / height) + br_xs *= (inp_w / width) + br_ys *= (inp_h / height) + + if with_centripetal_shift: + tl_ctxs *= (inp_w / width) + tl_ctys *= (inp_h / height) + br_ctxs *= (inp_w / width) + br_ctys *= (inp_h / height) + + x_off = img_meta['border'][2] + y_off = img_meta['border'][0] + + tl_xs -= x_off + tl_ys -= y_off + br_xs -= x_off + br_ys -= y_off + + tl_xs *= tl_xs.gt(0.0).type_as(tl_xs) + tl_ys *= tl_ys.gt(0.0).type_as(tl_ys) + br_xs *= br_xs.gt(0.0).type_as(br_xs) + br_ys *= br_ys.gt(0.0).type_as(br_ys) + + bboxes = torch.stack((tl_xs, tl_ys, br_xs, br_ys), dim=3) + area_bboxes = ((br_xs - tl_xs) * (br_ys - tl_ys)).abs() + + if with_centripetal_shift: + tl_ctxs -= x_off + tl_ctys -= y_off + br_ctxs -= x_off + br_ctys -= y_off + + tl_ctxs *= tl_ctxs.gt(0.0).type_as(tl_ctxs) + tl_ctys *= tl_ctys.gt(0.0).type_as(tl_ctys) + br_ctxs *= br_ctxs.gt(0.0).type_as(br_ctxs) + br_ctys *= br_ctys.gt(0.0).type_as(br_ctys) + + ct_bboxes = torch.stack((tl_ctxs, tl_ctys, br_ctxs, br_ctys), + dim=3) + area_ct_bboxes = ((br_ctxs - tl_ctxs) * (br_ctys - tl_ctys)).abs() + + rcentral = torch.zeros_like(ct_bboxes) + # magic nums from paper section 4.1 + mu = torch.ones_like(area_bboxes) / 2.4 + mu[area_bboxes > 3500] = 1 / 2.1 # large bbox have smaller mu + + bboxes_center_x = (bboxes[..., 0] + bboxes[..., 2]) / 2 + bboxes_center_y = (bboxes[..., 1] + bboxes[..., 3]) / 2 + rcentral[..., 0] = bboxes_center_x - mu * (bboxes[..., 2] - + bboxes[..., 0]) / 2 + rcentral[..., 1] = bboxes_center_y - mu * (bboxes[..., 3] - + bboxes[..., 1]) / 2 + rcentral[..., 2] = bboxes_center_x + mu * (bboxes[..., 2] - + bboxes[..., 0]) / 2 + rcentral[..., 3] = bboxes_center_y + mu * (bboxes[..., 3] - + bboxes[..., 1]) / 2 + area_rcentral = ((rcentral[..., 2] - rcentral[..., 0]) * + (rcentral[..., 3] - rcentral[..., 1])).abs() + dists = area_ct_bboxes / area_rcentral + + tl_ctx_inds = (ct_bboxes[..., 0] <= rcentral[..., 0]) | ( + ct_bboxes[..., 0] >= rcentral[..., 2]) + tl_cty_inds = (ct_bboxes[..., 1] <= rcentral[..., 1]) | ( + ct_bboxes[..., 1] >= rcentral[..., 3]) + br_ctx_inds = (ct_bboxes[..., 2] <= rcentral[..., 0]) | ( + ct_bboxes[..., 2] >= rcentral[..., 2]) + br_cty_inds = (ct_bboxes[..., 3] <= rcentral[..., 1]) | ( + ct_bboxes[..., 3] >= rcentral[..., 3]) + + if with_embedding: + tl_emb = self._transpose_and_gather_feat(tl_emb, tl_inds) + tl_emb = tl_emb.view(batch, k, 1) + br_emb = self._transpose_and_gather_feat(br_emb, br_inds) + br_emb = br_emb.view(batch, 1, k) + dists = torch.abs(tl_emb - br_emb) + + tl_scores = tl_scores.view(batch, k, 1).repeat(1, 1, k) + br_scores = br_scores.view(batch, 1, k).repeat(1, k, 1) + + scores = (tl_scores + br_scores) / 2 # scores for all possible boxes + + # tl and br should have same class + tl_clses = tl_clses.view(batch, k, 1).repeat(1, 1, k) + br_clses = br_clses.view(batch, 1, k).repeat(1, k, 1) + cls_inds = (tl_clses != br_clses) + + # reject boxes based on distances + dist_inds = dists > distance_threshold + + # reject boxes based on widths and heights + width_inds = (br_xs <= tl_xs) + height_inds = (br_ys <= tl_ys) + + scores[cls_inds] = -1 + scores[width_inds] = -1 + scores[height_inds] = -1 + scores[dist_inds] = -1 + if with_centripetal_shift: + scores[tl_ctx_inds] = -1 + scores[tl_cty_inds] = -1 + scores[br_ctx_inds] = -1 + scores[br_cty_inds] = -1 + + scores = scores.view(batch, -1) + scores, inds = torch.topk(scores, num_dets) + scores = scores.unsqueeze(2) + + bboxes = bboxes.view(batch, -1, 4) + bboxes = self._gather_feat(bboxes, inds) + + clses = tl_clses.contiguous().view(batch, -1, 1) + clses = self._gather_feat(clses, inds).float() + + return bboxes, scores, clses diff --git a/detection/mmdet/models/dense_heads/dense_test_mixins.py b/detection/mmdet/models/dense_heads/dense_test_mixins.py new file mode 100644 index 0000000..dd81364 --- /dev/null +++ b/detection/mmdet/models/dense_heads/dense_test_mixins.py @@ -0,0 +1,100 @@ +from inspect import signature + +import torch + +from mmdet.core import bbox2result, bbox_mapping_back, multiclass_nms + + +class BBoxTestMixin(object): + """Mixin class for test time augmentation of bboxes.""" + + def merge_aug_bboxes(self, aug_bboxes, aug_scores, img_metas): + """Merge augmented detection bboxes and scores. + + Args: + aug_bboxes (list[Tensor]): shape (n, 4*#class) + aug_scores (list[Tensor] or None): shape (n, #class) + img_shapes (list[Tensor]): shape (3, ). + + Returns: + tuple: (bboxes, scores) + """ + recovered_bboxes = [] + for bboxes, img_info in zip(aug_bboxes, img_metas): + img_shape = img_info[0]['img_shape'] + scale_factor = img_info[0]['scale_factor'] + flip = img_info[0]['flip'] + flip_direction = img_info[0]['flip_direction'] + bboxes = bbox_mapping_back(bboxes, img_shape, scale_factor, flip, + flip_direction) + recovered_bboxes.append(bboxes) + bboxes = torch.cat(recovered_bboxes, dim=0) + if aug_scores is None: + return bboxes + else: + scores = torch.cat(aug_scores, dim=0) + return bboxes, scores + + def aug_test_bboxes(self, feats, img_metas, rescale=False): + """Test det bboxes with test time augmentation. + + Args: + feats (list[Tensor]): the outer list indicates test-time + augmentations and inner Tensor should have a shape NxCxHxW, + which contains features for all images in the batch. + img_metas (list[list[dict]]): the outer list indicates test-time + augs (multiscale, flip, etc.) and the inner list indicates + images in a batch. each dict has image information. + rescale (bool, optional): Whether to rescale the results. + Defaults to False. + + Returns: + list[ndarray]: bbox results of each class + """ + # check with_nms argument + gb_sig = signature(self.get_bboxes) + gb_args = [p.name for p in gb_sig.parameters.values()] + if hasattr(self, '_get_bboxes'): + gbs_sig = signature(self._get_bboxes) + else: + gbs_sig = signature(self._get_bboxes_single) + gbs_args = [p.name for p in gbs_sig.parameters.values()] + assert ('with_nms' in gb_args) and ('with_nms' in gbs_args), \ + f'{self.__class__.__name__}' \ + ' does not support test-time augmentation' + + aug_bboxes = [] + aug_scores = [] + aug_factors = [] # score_factors for NMS + for x, img_meta in zip(feats, img_metas): + # only one image in the batch + outs = self.forward(x) + bbox_inputs = outs + (img_meta, self.test_cfg, False, False) + bbox_outputs = self.get_bboxes(*bbox_inputs)[0] + aug_bboxes.append(bbox_outputs[0]) + aug_scores.append(bbox_outputs[1]) + # bbox_outputs of some detectors (e.g., ATSS, FCOS, YOLOv3) + # contains additional element to adjust scores before NMS + if len(bbox_outputs) >= 3: + aug_factors.append(bbox_outputs[2]) + + # after merging, bboxes will be rescaled to the original image size + merged_bboxes, merged_scores = self.merge_aug_bboxes( + aug_bboxes, aug_scores, img_metas) + merged_factors = torch.cat(aug_factors, dim=0) if aug_factors else None + det_bboxes, det_labels = multiclass_nms( + merged_bboxes, + merged_scores, + self.test_cfg.score_thr, + self.test_cfg.nms, + self.test_cfg.max_per_img, + score_factors=merged_factors) + + if rescale: + _det_bboxes = det_bboxes + else: + _det_bboxes = det_bboxes.clone() + _det_bboxes[:, :4] *= det_bboxes.new_tensor( + img_metas[0][0]['scale_factor']) + bbox_results = bbox2result(_det_bboxes, det_labels, self.num_classes) + return bbox_results diff --git a/detection/mmdet/models/dense_heads/embedding_rpn_head.py b/detection/mmdet/models/dense_heads/embedding_rpn_head.py new file mode 100644 index 0000000..200ce8d --- /dev/null +++ b/detection/mmdet/models/dense_heads/embedding_rpn_head.py @@ -0,0 +1,100 @@ +import torch +import torch.nn as nn + +from mmdet.models.builder import HEADS +from ...core import bbox_cxcywh_to_xyxy + + +@HEADS.register_module() +class EmbeddingRPNHead(nn.Module): + """RPNHead in the `Sparse R-CNN `_ . + + Unlike traditional RPNHead, this module does not need FPN input, but just + decode `init_proposal_bboxes` and expand the first dimension of + `init_proposal_bboxes` and `init_proposal_features` to the batch_size. + + Args: + num_proposals (int): Number of init_proposals. Default 100. + proposal_feature_channel (int): Channel number of + init_proposal_feature. Defaults to 256. + """ + + def __init__(self, + num_proposals=100, + proposal_feature_channel=256, + **kwargs): + super(EmbeddingRPNHead, self).__init__() + self.num_proposals = num_proposals + self.proposal_feature_channel = proposal_feature_channel + self._init_layers() + + def _init_layers(self): + """Initialize a sparse set of proposal boxes and proposal features.""" + self.init_proposal_bboxes = nn.Embedding(self.num_proposals, 4) + self.init_proposal_features = nn.Embedding( + self.num_proposals, self.proposal_feature_channel) + + def init_weights(self): + """Initialize the init_proposal_bboxes as normalized. + + [c_x, c_y, w, h], and we initialize it to the size of the entire + image. + """ + nn.init.constant_(self.init_proposal_bboxes.weight[:, :2], 0.5) + nn.init.constant_(self.init_proposal_bboxes.weight[:, 2:], 1) + + def _decode_init_proposals(self, imgs, img_metas): + """Decode init_proposal_bboxes according to the size of images and + expand dimension of init_proposal_features to batch_size. + + Args: + imgs (list[Tensor]): List of FPN features. + img_metas (list[dict]): List of meta-information of + images. Need the img_shape to decode the init_proposals. + + Returns: + Tuple(Tensor): + + - proposals (Tensor): Decoded proposal bboxes, + has shape (batch_size, num_proposals, 4). + - init_proposal_features (Tensor): Expanded proposal + features, has shape + (batch_size, num_proposals, proposal_feature_channel). + - imgs_whwh (Tensor): Tensor with shape + (batch_size, 4), the dimension means + [img_width, img_height, img_width, img_height]. + """ + proposals = self.init_proposal_bboxes.weight.clone() + proposals = bbox_cxcywh_to_xyxy(proposals) + num_imgs = len(imgs[0]) + imgs_whwh = [] + for meta in img_metas: + h, w, _ = meta['img_shape'] + imgs_whwh.append(imgs[0].new_tensor([[w, h, w, h]])) + imgs_whwh = torch.cat(imgs_whwh, dim=0) + imgs_whwh = imgs_whwh[:, None, :] + + # imgs_whwh has shape (batch_size, 1, 4) + # The shape of proposals change from (num_proposals, 4) + # to (batch_size ,num_proposals, 4) + proposals = proposals * imgs_whwh + + init_proposal_features = self.init_proposal_features.weight.clone() + init_proposal_features = init_proposal_features[None].expand( + num_imgs, *init_proposal_features.size()) + return proposals, init_proposal_features, imgs_whwh + + def forward_dummy(self, img, img_metas): + """Dummy forward function. + + Used in flops calculation. + """ + return self._decode_init_proposals(img, img_metas) + + def forward_train(self, img, img_metas): + """Forward function in training stage.""" + return self._decode_init_proposals(img, img_metas) + + def simple_test_rpn(self, img, img_metas): + """Forward function in testing stage.""" + return self._decode_init_proposals(img, img_metas) diff --git a/detection/mmdet/models/dense_heads/fcos_head.py b/detection/mmdet/models/dense_heads/fcos_head.py new file mode 100644 index 0000000..905a703 --- /dev/null +++ b/detection/mmdet/models/dense_heads/fcos_head.py @@ -0,0 +1,629 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import Scale, normal_init +from mmcv.runner import force_fp32 + +from mmdet.core import distance2bbox, multi_apply, multiclass_nms, reduce_mean +from ..builder import HEADS, build_loss +from .anchor_free_head import AnchorFreeHead + +INF = 1e8 + + +@HEADS.register_module() +class FCOSHead(AnchorFreeHead): + """Anchor-free head used in `FCOS `_. + + The FCOS head does not use anchor boxes. Instead bounding boxes are + predicted at each pixel and a centerness measure is used to suppress + low-quality predictions. + Here norm_on_bbox, centerness_on_reg, dcn_on_last_conv are training + tricks used in official repo, which will bring remarkable mAP gains + of up to 4.9. Please see https://github.com/tianzhi0549/FCOS for + more detail. + + Args: + num_classes (int): Number of categories excluding the background + category. + in_channels (int): Number of channels in the input feature map. + strides (list[int] | list[tuple[int, int]]): Strides of points + in multiple feature levels. Default: (4, 8, 16, 32, 64). + regress_ranges (tuple[tuple[int, int]]): Regress range of multiple + level points. + center_sampling (bool): If true, use center sampling. Default: False. + center_sample_radius (float): Radius of center sampling. Default: 1.5. + norm_on_bbox (bool): If true, normalize the regression targets + with FPN strides. Default: False. + centerness_on_reg (bool): If true, position centerness on the + regress branch. Please refer to https://github.com/tianzhi0549/FCOS/issues/89#issuecomment-516877042. + Default: False. + conv_bias (bool | str): If specified as `auto`, it will be decided by the + norm_cfg. Bias of conv will be set as True if `norm_cfg` is None, otherwise + False. Default: "auto". + loss_cls (dict): Config of classification loss. + loss_bbox (dict): Config of localization loss. + loss_centerness (dict): Config of centerness loss. + norm_cfg (dict): dictionary to construct and config norm layer. + Default: norm_cfg=dict(type='GN', num_groups=32, requires_grad=True). + + Example: + >>> self = FCOSHead(11, 7) + >>> feats = [torch.rand(1, 7, s, s) for s in [4, 8, 16, 32, 64]] + >>> cls_score, bbox_pred, centerness = self.forward(feats) + >>> assert len(cls_score) == len(self.scales) + """ # noqa: E501 + + def __init__(self, + num_classes, + in_channels, + regress_ranges=((-1, 64), (64, 128), (128, 256), (256, 512), + (512, INF)), + center_sampling=False, + center_sample_radius=1.5, + norm_on_bbox=False, + centerness_on_reg=False, + loss_cls=dict( + type='FocalLoss', + use_sigmoid=True, + gamma=2.0, + alpha=0.25, + loss_weight=1.0), + loss_bbox=dict(type='IoULoss', loss_weight=1.0), + loss_centerness=dict( + type='CrossEntropyLoss', + use_sigmoid=True, + loss_weight=1.0), + norm_cfg=dict(type='GN', num_groups=32, requires_grad=True), + **kwargs): + self.regress_ranges = regress_ranges + self.center_sampling = center_sampling + self.center_sample_radius = center_sample_radius + self.norm_on_bbox = norm_on_bbox + self.centerness_on_reg = centerness_on_reg + super().__init__( + num_classes, + in_channels, + loss_cls=loss_cls, + loss_bbox=loss_bbox, + norm_cfg=norm_cfg, + **kwargs) + self.loss_centerness = build_loss(loss_centerness) + + def _init_layers(self): + """Initialize layers of the head.""" + super()._init_layers() + self.conv_centerness = nn.Conv2d(self.feat_channels, 1, 3, padding=1) + self.scales = nn.ModuleList([Scale(1.0) for _ in self.strides]) + + def init_weights(self): + """Initialize weights of the head.""" + super().init_weights() + normal_init(self.conv_centerness, std=0.01) + + def forward(self, feats): + """Forward features from the upstream network. + + Args: + feats (tuple[Tensor]): Features from the upstream network, each is + a 4D-tensor. + + Returns: + tuple: + cls_scores (list[Tensor]): Box scores for each scale level, \ + each is a 4D-tensor, the channel number is \ + num_points * num_classes. + bbox_preds (list[Tensor]): Box energies / deltas for each \ + scale level, each is a 4D-tensor, the channel number is \ + num_points * 4. + centernesses (list[Tensor]): centerness for each scale level, \ + each is a 4D-tensor, the channel number is num_points * 1. + """ + return multi_apply(self.forward_single, feats, self.scales, + self.strides) + + def forward_single(self, x, scale, stride): + """Forward features of a single scale level. + + Args: + x (Tensor): FPN feature maps of the specified stride. + scale (:obj: `mmcv.cnn.Scale`): Learnable scale module to resize + the bbox prediction. + stride (int): The corresponding stride for feature maps, only + used to normalize the bbox prediction when self.norm_on_bbox + is True. + + Returns: + tuple: scores for each class, bbox predictions and centerness \ + predictions of input feature maps. + """ + cls_score, bbox_pred, cls_feat, reg_feat = super().forward_single(x) + if self.centerness_on_reg: + centerness = self.conv_centerness(reg_feat) + else: + centerness = self.conv_centerness(cls_feat) + # scale the bbox_pred of different level + # float to avoid overflow when enabling FP16 + bbox_pred = scale(bbox_pred).float() + if self.norm_on_bbox: + bbox_pred = F.relu(bbox_pred) + if not self.training: + bbox_pred *= stride + else: + bbox_pred = bbox_pred.exp() + return cls_score, bbox_pred, centerness + + @force_fp32(apply_to=('cls_scores', 'bbox_preds', 'centernesses')) + def loss(self, + cls_scores, + bbox_preds, + centernesses, + gt_bboxes, + gt_labels, + img_metas, + gt_bboxes_ignore=None): + """Compute loss of the head. + + Args: + cls_scores (list[Tensor]): Box scores for each scale level, + each is a 4D-tensor, the channel number is + num_points * num_classes. + bbox_preds (list[Tensor]): Box energies / deltas for each scale + level, each is a 4D-tensor, the channel number is + num_points * 4. + centernesses (list[Tensor]): centerness for each scale level, each + is a 4D-tensor, the channel number is num_points * 1. + gt_bboxes (list[Tensor]): Ground truth bboxes for each image with + shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + gt_labels (list[Tensor]): class indices corresponding to each box + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + gt_bboxes_ignore (None | list[Tensor]): specify which bounding + boxes can be ignored when computing the loss. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + assert len(cls_scores) == len(bbox_preds) == len(centernesses) + featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] + all_level_points = self.get_points(featmap_sizes, bbox_preds[0].dtype, + bbox_preds[0].device) + labels, bbox_targets = self.get_targets(all_level_points, gt_bboxes, + gt_labels) + + num_imgs = cls_scores[0].size(0) + # flatten cls_scores, bbox_preds and centerness + flatten_cls_scores = [ + cls_score.permute(0, 2, 3, 1).reshape(-1, self.cls_out_channels) + for cls_score in cls_scores + ] + flatten_bbox_preds = [ + bbox_pred.permute(0, 2, 3, 1).reshape(-1, 4) + for bbox_pred in bbox_preds + ] + flatten_centerness = [ + centerness.permute(0, 2, 3, 1).reshape(-1) + for centerness in centernesses + ] + flatten_cls_scores = torch.cat(flatten_cls_scores) + flatten_bbox_preds = torch.cat(flatten_bbox_preds) + flatten_centerness = torch.cat(flatten_centerness) + flatten_labels = torch.cat(labels) + flatten_bbox_targets = torch.cat(bbox_targets) + # repeat points to align with bbox_preds + flatten_points = torch.cat( + [points.repeat(num_imgs, 1) for points in all_level_points]) + + # FG cat_id: [0, num_classes -1], BG cat_id: num_classes + bg_class_ind = self.num_classes + pos_inds = ((flatten_labels >= 0) + & (flatten_labels < bg_class_ind)).nonzero().reshape(-1) + num_pos = torch.tensor( + len(pos_inds), dtype=torch.float, device=bbox_preds[0].device) + num_pos = max(reduce_mean(num_pos), 1.0) + loss_cls = self.loss_cls( + flatten_cls_scores, flatten_labels, avg_factor=num_pos) + + pos_bbox_preds = flatten_bbox_preds[pos_inds] + pos_centerness = flatten_centerness[pos_inds] + + if len(pos_inds) > 0: + pos_bbox_targets = flatten_bbox_targets[pos_inds] + pos_centerness_targets = self.centerness_target(pos_bbox_targets) + pos_points = flatten_points[pos_inds] + pos_decoded_bbox_preds = distance2bbox(pos_points, pos_bbox_preds) + pos_decoded_target_preds = distance2bbox(pos_points, + pos_bbox_targets) + # centerness weighted iou loss + centerness_denorm = max( + reduce_mean(pos_centerness_targets.sum().detach()), 1e-6) + loss_bbox = self.loss_bbox( + pos_decoded_bbox_preds, + pos_decoded_target_preds, + weight=pos_centerness_targets, + avg_factor=centerness_denorm) + loss_centerness = self.loss_centerness( + pos_centerness, pos_centerness_targets, avg_factor=num_pos) + else: + loss_bbox = pos_bbox_preds.sum() + loss_centerness = pos_centerness.sum() + + return dict( + loss_cls=loss_cls, + loss_bbox=loss_bbox, + loss_centerness=loss_centerness) + + @force_fp32(apply_to=('cls_scores', 'bbox_preds', 'centernesses')) + def get_bboxes(self, + cls_scores, + bbox_preds, + centernesses, + img_metas, + cfg=None, + rescale=False, + with_nms=True): + """Transform network output for a batch into bbox predictions. + + Args: + cls_scores (list[Tensor]): Box scores for each scale level + with shape (N, num_points * num_classes, H, W). + bbox_preds (list[Tensor]): Box energies / deltas for each scale + level with shape (N, num_points * 4, H, W). + centernesses (list[Tensor]): Centerness for each scale level with + shape (N, num_points * 1, H, W). + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + cfg (mmcv.Config | None): Test / postprocessing configuration, + if None, test_cfg would be used. Default: None. + rescale (bool): If True, return boxes in original image space. + Default: False. + with_nms (bool): If True, do nms before return boxes. + Default: True. + + Returns: + list[tuple[Tensor, Tensor]]: Each item in result_list is 2-tuple. + The first item is an (n, 5) tensor, where 5 represent + (tl_x, tl_y, br_x, br_y, score) and the score between 0 and 1. + The shape of the second tensor in the tuple is (n,), and + each element represents the class label of the corresponding + box. + """ + assert len(cls_scores) == len(bbox_preds) + num_levels = len(cls_scores) + + featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] + mlvl_points = self.get_points(featmap_sizes, bbox_preds[0].dtype, + bbox_preds[0].device) + + cls_score_list = [cls_scores[i].detach() for i in range(num_levels)] + bbox_pred_list = [bbox_preds[i].detach() for i in range(num_levels)] + centerness_pred_list = [ + centernesses[i].detach() for i in range(num_levels) + ] + if torch.onnx.is_in_onnx_export(): + assert len( + img_metas + ) == 1, 'Only support one input image while in exporting to ONNX' + img_shapes = img_metas[0]['img_shape_for_onnx'] + else: + img_shapes = [ + img_metas[i]['img_shape'] + for i in range(cls_scores[0].shape[0]) + ] + scale_factors = [ + img_metas[i]['scale_factor'] for i in range(cls_scores[0].shape[0]) + ] + result_list = self._get_bboxes(cls_score_list, bbox_pred_list, + centerness_pred_list, mlvl_points, + img_shapes, scale_factors, cfg, rescale, + with_nms) + return result_list + + def _get_bboxes(self, + cls_scores, + bbox_preds, + centernesses, + mlvl_points, + img_shapes, + scale_factors, + cfg, + rescale=False, + with_nms=True): + """Transform outputs for a single batch item into bbox predictions. + + Args: + cls_scores (list[Tensor]): Box scores for a single scale level + with shape (N, num_points * num_classes, H, W). + bbox_preds (list[Tensor]): Box energies / deltas for a single scale + level with shape (N, num_points * 4, H, W). + centernesses (list[Tensor]): Centerness for a single scale level + with shape (N, num_points * 4, H, W). + mlvl_points (list[Tensor]): Box reference for a single scale level + with shape (num_total_points, 4). + img_shapes (list[tuple[int]]): Shape of the input image, + list[(height, width, 3)]. + scale_factors (list[ndarray]): Scale factor of the image arrange as + (w_scale, h_scale, w_scale, h_scale). + cfg (mmcv.Config | None): Test / postprocessing configuration, + if None, test_cfg would be used. + rescale (bool): If True, return boxes in original image space. + Default: False. + with_nms (bool): If True, do nms before return boxes. + Default: True. + + Returns: + tuple(Tensor): + det_bboxes (Tensor): BBox predictions in shape (n, 5), where + the first 4 columns are bounding box positions + (tl_x, tl_y, br_x, br_y) and the 5-th column is a score + between 0 and 1. + det_labels (Tensor): A (n,) tensor where each item is the + predicted class label of the corresponding box. + """ + cfg = self.test_cfg if cfg is None else cfg + assert len(cls_scores) == len(bbox_preds) == len(mlvl_points) + device = cls_scores[0].device + batch_size = cls_scores[0].shape[0] + # convert to tensor to keep tracing + nms_pre_tensor = torch.tensor( + cfg.get('nms_pre', -1), device=device, dtype=torch.long) + mlvl_bboxes = [] + mlvl_scores = [] + mlvl_centerness = [] + for cls_score, bbox_pred, centerness, points in zip( + cls_scores, bbox_preds, centernesses, mlvl_points): + assert cls_score.size()[-2:] == bbox_pred.size()[-2:] + scores = cls_score.permute(0, 2, 3, 1).reshape( + batch_size, -1, self.cls_out_channels).sigmoid() + centerness = centerness.permute(0, 2, 3, + 1).reshape(batch_size, + -1).sigmoid() + + bbox_pred = bbox_pred.permute(0, 2, 3, + 1).reshape(batch_size, -1, 4) + # Always keep topk op for dynamic input in onnx + if nms_pre_tensor > 0 and (torch.onnx.is_in_onnx_export() + or scores.shape[-2] > nms_pre_tensor): + from torch import _shape_as_tensor + # keep shape as tensor and get k + num_anchor = _shape_as_tensor(scores)[-2].to(device) + nms_pre = torch.where(nms_pre_tensor < num_anchor, + nms_pre_tensor, num_anchor) + + max_scores, _ = (scores * centerness[..., None]).max(-1) + _, topk_inds = max_scores.topk(nms_pre) + points = points[topk_inds, :] + batch_inds = torch.arange(batch_size).view( + -1, 1).expand_as(topk_inds).long() + bbox_pred = bbox_pred[batch_inds, topk_inds, :] + scores = scores[batch_inds, topk_inds, :] + centerness = centerness[batch_inds, topk_inds] + + bboxes = distance2bbox(points, bbox_pred, max_shape=img_shapes) + mlvl_bboxes.append(bboxes) + mlvl_scores.append(scores) + mlvl_centerness.append(centerness) + + batch_mlvl_bboxes = torch.cat(mlvl_bboxes, dim=1) + if rescale: + batch_mlvl_bboxes /= batch_mlvl_bboxes.new_tensor( + scale_factors).unsqueeze(1) + batch_mlvl_scores = torch.cat(mlvl_scores, dim=1) + batch_mlvl_centerness = torch.cat(mlvl_centerness, dim=1) + + # Set max number of box to be feed into nms in deployment + deploy_nms_pre = cfg.get('deploy_nms_pre', -1) + if deploy_nms_pre > 0 and torch.onnx.is_in_onnx_export(): + batch_mlvl_scores, _ = ( + batch_mlvl_scores * + batch_mlvl_centerness.unsqueeze(2).expand_as(batch_mlvl_scores) + ).max(-1) + _, topk_inds = batch_mlvl_scores.topk(deploy_nms_pre) + batch_inds = torch.arange(batch_mlvl_scores.shape[0]).view( + -1, 1).expand_as(topk_inds) + batch_mlvl_scores = batch_mlvl_scores[batch_inds, topk_inds, :] + batch_mlvl_bboxes = batch_mlvl_bboxes[batch_inds, topk_inds, :] + batch_mlvl_centerness = batch_mlvl_centerness[batch_inds, + topk_inds] + + # remind that we set FG labels to [0, num_class-1] since mmdet v2.0 + # BG cat_id: num_class + padding = batch_mlvl_scores.new_zeros(batch_size, + batch_mlvl_scores.shape[1], 1) + batch_mlvl_scores = torch.cat([batch_mlvl_scores, padding], dim=-1) + + if with_nms: + det_results = [] + for (mlvl_bboxes, mlvl_scores, + mlvl_centerness) in zip(batch_mlvl_bboxes, batch_mlvl_scores, + batch_mlvl_centerness): + det_bbox, det_label = multiclass_nms( + mlvl_bboxes, + mlvl_scores, + cfg.score_thr, + cfg.nms, + cfg.max_per_img, + score_factors=mlvl_centerness) + det_results.append(tuple([det_bbox, det_label])) + else: + det_results = [ + tuple(mlvl_bs) + for mlvl_bs in zip(batch_mlvl_bboxes, batch_mlvl_scores, + batch_mlvl_centerness) + ] + return det_results + + def _get_points_single(self, + featmap_size, + stride, + dtype, + device, + flatten=False): + """Get points according to feature map sizes.""" + y, x = super()._get_points_single(featmap_size, stride, dtype, device) + points = torch.stack((x.reshape(-1) * stride, y.reshape(-1) * stride), + dim=-1) + stride // 2 + return points + + def get_targets(self, points, gt_bboxes_list, gt_labels_list): + """Compute regression, classification and centerness targets for points + in multiple images. + + Args: + points (list[Tensor]): Points of each fpn level, each has shape + (num_points, 2). + gt_bboxes_list (list[Tensor]): Ground truth bboxes of each image, + each has shape (num_gt, 4). + gt_labels_list (list[Tensor]): Ground truth labels of each box, + each has shape (num_gt,). + + Returns: + tuple: + concat_lvl_labels (list[Tensor]): Labels of each level. \ + concat_lvl_bbox_targets (list[Tensor]): BBox targets of each \ + level. + """ + assert len(points) == len(self.regress_ranges) + num_levels = len(points) + # expand regress ranges to align with points + expanded_regress_ranges = [ + points[i].new_tensor(self.regress_ranges[i])[None].expand_as( + points[i]) for i in range(num_levels) + ] + # concat all levels points and regress ranges + concat_regress_ranges = torch.cat(expanded_regress_ranges, dim=0) + concat_points = torch.cat(points, dim=0) + + # the number of points per img, per lvl + num_points = [center.size(0) for center in points] + + # get labels and bbox_targets of each image + labels_list, bbox_targets_list = multi_apply( + self._get_target_single, + gt_bboxes_list, + gt_labels_list, + points=concat_points, + regress_ranges=concat_regress_ranges, + num_points_per_lvl=num_points) + + # split to per img, per level + labels_list = [labels.split(num_points, 0) for labels in labels_list] + bbox_targets_list = [ + bbox_targets.split(num_points, 0) + for bbox_targets in bbox_targets_list + ] + + # concat per level image + concat_lvl_labels = [] + concat_lvl_bbox_targets = [] + for i in range(num_levels): + concat_lvl_labels.append( + torch.cat([labels[i] for labels in labels_list])) + bbox_targets = torch.cat( + [bbox_targets[i] for bbox_targets in bbox_targets_list]) + if self.norm_on_bbox: + bbox_targets = bbox_targets / self.strides[i] + concat_lvl_bbox_targets.append(bbox_targets) + return concat_lvl_labels, concat_lvl_bbox_targets + + def _get_target_single(self, gt_bboxes, gt_labels, points, regress_ranges, + num_points_per_lvl): + """Compute regression and classification targets for a single image.""" + num_points = points.size(0) + num_gts = gt_labels.size(0) + if num_gts == 0: + return gt_labels.new_full((num_points,), self.num_classes), \ + gt_bboxes.new_zeros((num_points, 4)) + + areas = (gt_bboxes[:, 2] - gt_bboxes[:, 0]) * ( + gt_bboxes[:, 3] - gt_bboxes[:, 1]) + # TODO: figure out why these two are different + # areas = areas[None].expand(num_points, num_gts) + areas = areas[None].repeat(num_points, 1) + regress_ranges = regress_ranges[:, None, :].expand( + num_points, num_gts, 2) + gt_bboxes = gt_bboxes[None].expand(num_points, num_gts, 4) + xs, ys = points[:, 0], points[:, 1] + xs = xs[:, None].expand(num_points, num_gts) + ys = ys[:, None].expand(num_points, num_gts) + + left = xs - gt_bboxes[..., 0] + right = gt_bboxes[..., 2] - xs + top = ys - gt_bboxes[..., 1] + bottom = gt_bboxes[..., 3] - ys + bbox_targets = torch.stack((left, top, right, bottom), -1) + + if self.center_sampling: + # condition1: inside a `center bbox` + radius = self.center_sample_radius + center_xs = (gt_bboxes[..., 0] + gt_bboxes[..., 2]) / 2 + center_ys = (gt_bboxes[..., 1] + gt_bboxes[..., 3]) / 2 + center_gts = torch.zeros_like(gt_bboxes) + stride = center_xs.new_zeros(center_xs.shape) + + # project the points on current lvl back to the `original` sizes + lvl_begin = 0 + for lvl_idx, num_points_lvl in enumerate(num_points_per_lvl): + lvl_end = lvl_begin + num_points_lvl + stride[lvl_begin:lvl_end] = self.strides[lvl_idx] * radius + lvl_begin = lvl_end + + x_mins = center_xs - stride + y_mins = center_ys - stride + x_maxs = center_xs + stride + y_maxs = center_ys + stride + center_gts[..., 0] = torch.where(x_mins > gt_bboxes[..., 0], + x_mins, gt_bboxes[..., 0]) + center_gts[..., 1] = torch.where(y_mins > gt_bboxes[..., 1], + y_mins, gt_bboxes[..., 1]) + center_gts[..., 2] = torch.where(x_maxs > gt_bboxes[..., 2], + gt_bboxes[..., 2], x_maxs) + center_gts[..., 3] = torch.where(y_maxs > gt_bboxes[..., 3], + gt_bboxes[..., 3], y_maxs) + + cb_dist_left = xs - center_gts[..., 0] + cb_dist_right = center_gts[..., 2] - xs + cb_dist_top = ys - center_gts[..., 1] + cb_dist_bottom = center_gts[..., 3] - ys + center_bbox = torch.stack( + (cb_dist_left, cb_dist_top, cb_dist_right, cb_dist_bottom), -1) + inside_gt_bbox_mask = center_bbox.min(-1)[0] > 0 + else: + # condition1: inside a gt bbox + inside_gt_bbox_mask = bbox_targets.min(-1)[0] > 0 + + # condition2: limit the regression range for each location + max_regress_distance = bbox_targets.max(-1)[0] + inside_regress_range = ( + (max_regress_distance >= regress_ranges[..., 0]) + & (max_regress_distance <= regress_ranges[..., 1])) + + # if there are still more than one objects for a location, + # we choose the one with minimal area + areas[inside_gt_bbox_mask == 0] = INF + areas[inside_regress_range == 0] = INF + min_area, min_area_inds = areas.min(dim=1) + + labels = gt_labels[min_area_inds] + labels[min_area == INF] = self.num_classes # set as BG + bbox_targets = bbox_targets[range(num_points), min_area_inds] + + return labels, bbox_targets + + def centerness_target(self, pos_bbox_targets): + """Compute centerness targets. + + Args: + pos_bbox_targets (Tensor): BBox targets of positive bboxes in shape + (num_pos, 4) + + Returns: + Tensor: Centerness target. + """ + # only calculate pos centerness targets, otherwise there may be nan + left_right = pos_bbox_targets[:, [0, 2]] + top_bottom = pos_bbox_targets[:, [1, 3]] + centerness_targets = ( + left_right.min(dim=-1)[0] / left_right.max(dim=-1)[0]) * ( + top_bottom.min(dim=-1)[0] / top_bottom.max(dim=-1)[0]) + return torch.sqrt(centerness_targets) diff --git a/detection/mmdet/models/dense_heads/fovea_head.py b/detection/mmdet/models/dense_heads/fovea_head.py new file mode 100644 index 0000000..c8ccea7 --- /dev/null +++ b/detection/mmdet/models/dense_heads/fovea_head.py @@ -0,0 +1,341 @@ +import torch +import torch.nn as nn +from mmcv.cnn import ConvModule, normal_init +from mmcv.ops import DeformConv2d + +from mmdet.core import multi_apply, multiclass_nms +from ..builder import HEADS +from .anchor_free_head import AnchorFreeHead + +INF = 1e8 + + +class FeatureAlign(nn.Module): + + def __init__(self, + in_channels, + out_channels, + kernel_size=3, + deform_groups=4): + super(FeatureAlign, self).__init__() + offset_channels = kernel_size * kernel_size * 2 + self.conv_offset = nn.Conv2d( + 4, deform_groups * offset_channels, 1, bias=False) + self.conv_adaption = DeformConv2d( + in_channels, + out_channels, + kernel_size=kernel_size, + padding=(kernel_size - 1) // 2, + deform_groups=deform_groups) + self.relu = nn.ReLU(inplace=True) + + def init_weights(self): + normal_init(self.conv_offset, std=0.1) + normal_init(self.conv_adaption, std=0.01) + + def forward(self, x, shape): + offset = self.conv_offset(shape) + x = self.relu(self.conv_adaption(x, offset)) + return x + + +@HEADS.register_module() +class FoveaHead(AnchorFreeHead): + """FoveaBox: Beyond Anchor-based Object Detector + https://arxiv.org/abs/1904.03797 + """ + + def __init__(self, + num_classes, + in_channels, + base_edge_list=(16, 32, 64, 128, 256), + scale_ranges=((8, 32), (16, 64), (32, 128), (64, 256), (128, + 512)), + sigma=0.4, + with_deform=False, + deform_groups=4, + **kwargs): + self.base_edge_list = base_edge_list + self.scale_ranges = scale_ranges + self.sigma = sigma + self.with_deform = with_deform + self.deform_groups = deform_groups + super().__init__(num_classes, in_channels, **kwargs) + + def _init_layers(self): + # box branch + super()._init_reg_convs() + self.conv_reg = nn.Conv2d(self.feat_channels, 4, 3, padding=1) + + # cls branch + if not self.with_deform: + super()._init_cls_convs() + self.conv_cls = nn.Conv2d( + self.feat_channels, self.cls_out_channels, 3, padding=1) + else: + self.cls_convs = nn.ModuleList() + self.cls_convs.append( + ConvModule( + self.feat_channels, (self.feat_channels * 4), + 3, + stride=1, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + bias=self.norm_cfg is None)) + self.cls_convs.append( + ConvModule((self.feat_channels * 4), (self.feat_channels * 4), + 1, + stride=1, + padding=0, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + bias=self.norm_cfg is None)) + self.feature_adaption = FeatureAlign( + self.feat_channels, + self.feat_channels, + kernel_size=3, + deform_groups=self.deform_groups) + self.conv_cls = nn.Conv2d( + int(self.feat_channels * 4), + self.cls_out_channels, + 3, + padding=1) + + def init_weights(self): + super().init_weights() + if self.with_deform: + self.feature_adaption.init_weights() + + def forward_single(self, x): + cls_feat = x + reg_feat = x + for reg_layer in self.reg_convs: + reg_feat = reg_layer(reg_feat) + bbox_pred = self.conv_reg(reg_feat) + if self.with_deform: + cls_feat = self.feature_adaption(cls_feat, bbox_pred.exp()) + for cls_layer in self.cls_convs: + cls_feat = cls_layer(cls_feat) + cls_score = self.conv_cls(cls_feat) + return cls_score, bbox_pred + + def _get_points_single(self, *args, **kwargs): + y, x = super()._get_points_single(*args, **kwargs) + return y + 0.5, x + 0.5 + + def loss(self, + cls_scores, + bbox_preds, + gt_bbox_list, + gt_label_list, + img_metas, + gt_bboxes_ignore=None): + assert len(cls_scores) == len(bbox_preds) + + featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] + points = self.get_points(featmap_sizes, bbox_preds[0].dtype, + bbox_preds[0].device) + num_imgs = cls_scores[0].size(0) + flatten_cls_scores = [ + cls_score.permute(0, 2, 3, 1).reshape(-1, self.cls_out_channels) + for cls_score in cls_scores + ] + flatten_bbox_preds = [ + bbox_pred.permute(0, 2, 3, 1).reshape(-1, 4) + for bbox_pred in bbox_preds + ] + flatten_cls_scores = torch.cat(flatten_cls_scores) + flatten_bbox_preds = torch.cat(flatten_bbox_preds) + flatten_labels, flatten_bbox_targets = self.get_targets( + gt_bbox_list, gt_label_list, featmap_sizes, points) + + # FG cat_id: [0, num_classes -1], BG cat_id: num_classes + pos_inds = ((flatten_labels >= 0) + & (flatten_labels < self.num_classes)).nonzero().view(-1) + num_pos = len(pos_inds) + + loss_cls = self.loss_cls( + flatten_cls_scores, flatten_labels, avg_factor=num_pos + num_imgs) + if num_pos > 0: + pos_bbox_preds = flatten_bbox_preds[pos_inds] + pos_bbox_targets = flatten_bbox_targets[pos_inds] + pos_weights = pos_bbox_targets.new_zeros( + pos_bbox_targets.size()) + 1.0 + loss_bbox = self.loss_bbox( + pos_bbox_preds, + pos_bbox_targets, + pos_weights, + avg_factor=num_pos) + else: + loss_bbox = torch.tensor( + 0, + dtype=flatten_bbox_preds.dtype, + device=flatten_bbox_preds.device) + return dict(loss_cls=loss_cls, loss_bbox=loss_bbox) + + def get_targets(self, gt_bbox_list, gt_label_list, featmap_sizes, points): + label_list, bbox_target_list = multi_apply( + self._get_target_single, + gt_bbox_list, + gt_label_list, + featmap_size_list=featmap_sizes, + point_list=points) + flatten_labels = [ + torch.cat([ + labels_level_img.flatten() for labels_level_img in labels_level + ]) for labels_level in zip(*label_list) + ] + flatten_bbox_targets = [ + torch.cat([ + bbox_targets_level_img.reshape(-1, 4) + for bbox_targets_level_img in bbox_targets_level + ]) for bbox_targets_level in zip(*bbox_target_list) + ] + flatten_labels = torch.cat(flatten_labels) + flatten_bbox_targets = torch.cat(flatten_bbox_targets) + return flatten_labels, flatten_bbox_targets + + def _get_target_single(self, + gt_bboxes_raw, + gt_labels_raw, + featmap_size_list=None, + point_list=None): + + gt_areas = torch.sqrt((gt_bboxes_raw[:, 2] - gt_bboxes_raw[:, 0]) * + (gt_bboxes_raw[:, 3] - gt_bboxes_raw[:, 1])) + label_list = [] + bbox_target_list = [] + # for each pyramid, find the cls and box target + for base_len, (lower_bound, upper_bound), stride, featmap_size, \ + (y, x) in zip(self.base_edge_list, self.scale_ranges, + self.strides, featmap_size_list, point_list): + # FG cat_id: [0, num_classes -1], BG cat_id: num_classes + labels = gt_labels_raw.new_zeros(featmap_size) + self.num_classes + bbox_targets = gt_bboxes_raw.new(featmap_size[0], featmap_size[1], + 4) + 1 + # scale assignment + hit_indices = ((gt_areas >= lower_bound) & + (gt_areas <= upper_bound)).nonzero().flatten() + if len(hit_indices) == 0: + label_list.append(labels) + bbox_target_list.append(torch.log(bbox_targets)) + continue + _, hit_index_order = torch.sort(-gt_areas[hit_indices]) + hit_indices = hit_indices[hit_index_order] + gt_bboxes = gt_bboxes_raw[hit_indices, :] / stride + gt_labels = gt_labels_raw[hit_indices] + half_w = 0.5 * (gt_bboxes[:, 2] - gt_bboxes[:, 0]) + half_h = 0.5 * (gt_bboxes[:, 3] - gt_bboxes[:, 1]) + # valid fovea area: left, right, top, down + pos_left = torch.ceil( + gt_bboxes[:, 0] + (1 - self.sigma) * half_w - 0.5).long().\ + clamp(0, featmap_size[1] - 1) + pos_right = torch.floor( + gt_bboxes[:, 0] + (1 + self.sigma) * half_w - 0.5).long().\ + clamp(0, featmap_size[1] - 1) + pos_top = torch.ceil( + gt_bboxes[:, 1] + (1 - self.sigma) * half_h - 0.5).long().\ + clamp(0, featmap_size[0] - 1) + pos_down = torch.floor( + gt_bboxes[:, 1] + (1 + self.sigma) * half_h - 0.5).long().\ + clamp(0, featmap_size[0] - 1) + for px1, py1, px2, py2, label, (gt_x1, gt_y1, gt_x2, gt_y2) in \ + zip(pos_left, pos_top, pos_right, pos_down, gt_labels, + gt_bboxes_raw[hit_indices, :]): + labels[py1:py2 + 1, px1:px2 + 1] = label + bbox_targets[py1:py2 + 1, px1:px2 + 1, 0] = \ + (stride * x[py1:py2 + 1, px1:px2 + 1] - gt_x1) / base_len + bbox_targets[py1:py2 + 1, px1:px2 + 1, 1] = \ + (stride * y[py1:py2 + 1, px1:px2 + 1] - gt_y1) / base_len + bbox_targets[py1:py2 + 1, px1:px2 + 1, 2] = \ + (gt_x2 - stride * x[py1:py2 + 1, px1:px2 + 1]) / base_len + bbox_targets[py1:py2 + 1, px1:px2 + 1, 3] = \ + (gt_y2 - stride * y[py1:py2 + 1, px1:px2 + 1]) / base_len + bbox_targets = bbox_targets.clamp(min=1. / 16, max=16.) + label_list.append(labels) + bbox_target_list.append(torch.log(bbox_targets)) + return label_list, bbox_target_list + + def get_bboxes(self, + cls_scores, + bbox_preds, + img_metas, + cfg=None, + rescale=None): + assert len(cls_scores) == len(bbox_preds) + num_levels = len(cls_scores) + featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] + points = self.get_points( + featmap_sizes, + bbox_preds[0].dtype, + bbox_preds[0].device, + flatten=True) + result_list = [] + for img_id in range(len(img_metas)): + cls_score_list = [ + cls_scores[i][img_id].detach() for i in range(num_levels) + ] + bbox_pred_list = [ + bbox_preds[i][img_id].detach() for i in range(num_levels) + ] + img_shape = img_metas[img_id]['img_shape'] + scale_factor = img_metas[img_id]['scale_factor'] + det_bboxes = self._get_bboxes_single(cls_score_list, + bbox_pred_list, featmap_sizes, + points, img_shape, + scale_factor, cfg, rescale) + result_list.append(det_bboxes) + return result_list + + def _get_bboxes_single(self, + cls_scores, + bbox_preds, + featmap_sizes, + point_list, + img_shape, + scale_factor, + cfg, + rescale=False): + cfg = self.test_cfg if cfg is None else cfg + assert len(cls_scores) == len(bbox_preds) == len(point_list) + det_bboxes = [] + det_scores = [] + for cls_score, bbox_pred, featmap_size, stride, base_len, (y, x) \ + in zip(cls_scores, bbox_preds, featmap_sizes, self.strides, + self.base_edge_list, point_list): + assert cls_score.size()[-2:] == bbox_pred.size()[-2:] + scores = cls_score.permute(1, 2, 0).reshape( + -1, self.cls_out_channels).sigmoid() + bbox_pred = bbox_pred.permute(1, 2, 0).reshape(-1, 4).exp() + nms_pre = cfg.get('nms_pre', -1) + if (nms_pre > 0) and (scores.shape[0] > nms_pre): + max_scores, _ = scores.max(dim=1) + _, topk_inds = max_scores.topk(nms_pre) + bbox_pred = bbox_pred[topk_inds, :] + scores = scores[topk_inds, :] + y = y[topk_inds] + x = x[topk_inds] + x1 = (stride * x - base_len * bbox_pred[:, 0]).\ + clamp(min=0, max=img_shape[1] - 1) + y1 = (stride * y - base_len * bbox_pred[:, 1]).\ + clamp(min=0, max=img_shape[0] - 1) + x2 = (stride * x + base_len * bbox_pred[:, 2]).\ + clamp(min=0, max=img_shape[1] - 1) + y2 = (stride * y + base_len * bbox_pred[:, 3]).\ + clamp(min=0, max=img_shape[0] - 1) + bboxes = torch.stack([x1, y1, x2, y2], -1) + det_bboxes.append(bboxes) + det_scores.append(scores) + det_bboxes = torch.cat(det_bboxes) + if rescale: + det_bboxes /= det_bboxes.new_tensor(scale_factor) + det_scores = torch.cat(det_scores) + padding = det_scores.new_zeros(det_scores.shape[0], 1) + # remind that we set FG labels to [0, num_class-1] since mmdet v2.0 + # BG cat_id: num_class + det_scores = torch.cat([det_scores, padding], dim=1) + det_bboxes, det_labels = multiclass_nms(det_bboxes, det_scores, + cfg.score_thr, cfg.nms, + cfg.max_per_img) + return det_bboxes, det_labels diff --git a/detection/mmdet/models/dense_heads/free_anchor_retina_head.py b/detection/mmdet/models/dense_heads/free_anchor_retina_head.py new file mode 100644 index 0000000..79879fd --- /dev/null +++ b/detection/mmdet/models/dense_heads/free_anchor_retina_head.py @@ -0,0 +1,270 @@ +import torch +import torch.nn.functional as F + +from mmdet.core import bbox_overlaps +from ..builder import HEADS +from .retina_head import RetinaHead + +EPS = 1e-12 + + +@HEADS.register_module() +class FreeAnchorRetinaHead(RetinaHead): + """FreeAnchor RetinaHead used in https://arxiv.org/abs/1909.02466. + + Args: + num_classes (int): Number of categories excluding the background + category. + in_channels (int): Number of channels in the input feature map. + stacked_convs (int): Number of conv layers in cls and reg tower. + Default: 4. + conv_cfg (dict): dictionary to construct and config conv layer. + Default: None. + norm_cfg (dict): dictionary to construct and config norm layer. + Default: norm_cfg=dict(type='GN', num_groups=32, + requires_grad=True). + pre_anchor_topk (int): Number of boxes that be token in each bag. + bbox_thr (float): The threshold of the saturated linear function. It is + usually the same with the IoU threshold used in NMS. + gamma (float): Gamma parameter in focal loss. + alpha (float): Alpha parameter in focal loss. + """ # noqa: W605 + + def __init__(self, + num_classes, + in_channels, + stacked_convs=4, + conv_cfg=None, + norm_cfg=None, + pre_anchor_topk=50, + bbox_thr=0.6, + gamma=2.0, + alpha=0.5, + **kwargs): + super(FreeAnchorRetinaHead, + self).__init__(num_classes, in_channels, stacked_convs, conv_cfg, + norm_cfg, **kwargs) + + self.pre_anchor_topk = pre_anchor_topk + self.bbox_thr = bbox_thr + self.gamma = gamma + self.alpha = alpha + + def loss(self, + cls_scores, + bbox_preds, + gt_bboxes, + gt_labels, + img_metas, + gt_bboxes_ignore=None): + """Compute losses of the head. + + Args: + cls_scores (list[Tensor]): Box scores for each scale level + Has shape (N, num_anchors * num_classes, H, W) + bbox_preds (list[Tensor]): Box energies / deltas for each scale + level with shape (N, num_anchors * 4, H, W) + gt_bboxes (list[Tensor]): each item are the truth boxes for each + image in [tl_x, tl_y, br_x, br_y] format. + gt_labels (list[Tensor]): class indices corresponding to each box + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + gt_bboxes_ignore (None | list[Tensor]): specify which bounding + boxes can be ignored when computing the loss. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] + assert len(featmap_sizes) == len(self.anchor_generator.base_anchors) + + anchor_list, _ = self.get_anchors(featmap_sizes, img_metas) + anchors = [torch.cat(anchor) for anchor in anchor_list] + + # concatenate each level + cls_scores = [ + cls.permute(0, 2, 3, + 1).reshape(cls.size(0), -1, self.cls_out_channels) + for cls in cls_scores + ] + bbox_preds = [ + bbox_pred.permute(0, 2, 3, 1).reshape(bbox_pred.size(0), -1, 4) + for bbox_pred in bbox_preds + ] + cls_scores = torch.cat(cls_scores, dim=1) + bbox_preds = torch.cat(bbox_preds, dim=1) + + cls_prob = torch.sigmoid(cls_scores) + box_prob = [] + num_pos = 0 + positive_losses = [] + for _, (anchors_, gt_labels_, gt_bboxes_, cls_prob_, + bbox_preds_) in enumerate( + zip(anchors, gt_labels, gt_bboxes, cls_prob, bbox_preds)): + + with torch.no_grad(): + if len(gt_bboxes_) == 0: + image_box_prob = torch.zeros( + anchors_.size(0), + self.cls_out_channels).type_as(bbox_preds_) + else: + # box_localization: a_{j}^{loc}, shape: [j, 4] + pred_boxes = self.bbox_coder.decode(anchors_, bbox_preds_) + + # object_box_iou: IoU_{ij}^{loc}, shape: [i, j] + object_box_iou = bbox_overlaps(gt_bboxes_, pred_boxes) + + # object_box_prob: P{a_{j} -> b_{i}}, shape: [i, j] + t1 = self.bbox_thr + t2 = object_box_iou.max( + dim=1, keepdim=True).values.clamp(min=t1 + 1e-12) + object_box_prob = ((object_box_iou - t1) / + (t2 - t1)).clamp( + min=0, max=1) + + # object_cls_box_prob: P{a_{j} -> b_{i}}, shape: [i, c, j] + num_obj = gt_labels_.size(0) + indices = torch.stack([ + torch.arange(num_obj).type_as(gt_labels_), gt_labels_ + ], + dim=0) + object_cls_box_prob = torch.sparse_coo_tensor( + indices, object_box_prob) + + # image_box_iou: P{a_{j} \in A_{+}}, shape: [c, j] + """ + from "start" to "end" implement: + image_box_iou = torch.sparse.max(object_cls_box_prob, + dim=0).t() + + """ + # start + box_cls_prob = torch.sparse.sum( + object_cls_box_prob, dim=0).to_dense() + + indices = torch.nonzero(box_cls_prob, as_tuple=False).t_() + if indices.numel() == 0: + image_box_prob = torch.zeros( + anchors_.size(0), + self.cls_out_channels).type_as(object_box_prob) + else: + nonzero_box_prob = torch.where( + (gt_labels_.unsqueeze(dim=-1) == indices[0]), + object_box_prob[:, indices[1]], + torch.tensor([ + 0 + ]).type_as(object_box_prob)).max(dim=0).values + + # upmap to shape [j, c] + image_box_prob = torch.sparse_coo_tensor( + indices.flip([0]), + nonzero_box_prob, + size=(anchors_.size(0), + self.cls_out_channels)).to_dense() + # end + + box_prob.append(image_box_prob) + + # construct bags for objects + match_quality_matrix = bbox_overlaps(gt_bboxes_, anchors_) + _, matched = torch.topk( + match_quality_matrix, + self.pre_anchor_topk, + dim=1, + sorted=False) + del match_quality_matrix + + # matched_cls_prob: P_{ij}^{cls} + matched_cls_prob = torch.gather( + cls_prob_[matched], 2, + gt_labels_.view(-1, 1, 1).repeat(1, self.pre_anchor_topk, + 1)).squeeze(2) + + # matched_box_prob: P_{ij}^{loc} + matched_anchors = anchors_[matched] + matched_object_targets = self.bbox_coder.encode( + matched_anchors, + gt_bboxes_.unsqueeze(dim=1).expand_as(matched_anchors)) + loss_bbox = self.loss_bbox( + bbox_preds_[matched], + matched_object_targets, + reduction_override='none').sum(-1) + matched_box_prob = torch.exp(-loss_bbox) + + # positive_losses: {-log( Mean-max(P_{ij}^{cls} * P_{ij}^{loc}) )} + num_pos += len(gt_bboxes_) + positive_losses.append( + self.positive_bag_loss(matched_cls_prob, matched_box_prob)) + positive_loss = torch.cat(positive_losses).sum() / max(1, num_pos) + + # box_prob: P{a_{j} \in A_{+}} + box_prob = torch.stack(box_prob, dim=0) + + # negative_loss: + # \sum_{j}{ FL((1 - P{a_{j} \in A_{+}}) * (1 - P_{j}^{bg})) } / n||B|| + negative_loss = self.negative_bag_loss(cls_prob, box_prob).sum() / max( + 1, num_pos * self.pre_anchor_topk) + + # avoid the absence of gradients in regression subnet + # when no ground-truth in a batch + if num_pos == 0: + positive_loss = bbox_preds.sum() * 0 + + losses = { + 'positive_bag_loss': positive_loss, + 'negative_bag_loss': negative_loss + } + return losses + + def positive_bag_loss(self, matched_cls_prob, matched_box_prob): + """Compute positive bag loss. + + :math:`-log( Mean-max(P_{ij}^{cls} * P_{ij}^{loc}) )`. + + :math:`P_{ij}^{cls}`: matched_cls_prob, classification probability of matched samples. + + :math:`P_{ij}^{loc}`: matched_box_prob, box probability of matched samples. + + Args: + matched_cls_prob (Tensor): Classification probabilty of matched + samples in shape (num_gt, pre_anchor_topk). + matched_box_prob (Tensor): BBox probability of matched samples, + in shape (num_gt, pre_anchor_topk). + + Returns: + Tensor: Positive bag loss in shape (num_gt,). + """ # noqa: E501, W605 + # bag_prob = Mean-max(matched_prob) + matched_prob = matched_cls_prob * matched_box_prob + weight = 1 / torch.clamp(1 - matched_prob, 1e-12, None) + weight /= weight.sum(dim=1).unsqueeze(dim=-1) + bag_prob = (weight * matched_prob).sum(dim=1) + # positive_bag_loss = -self.alpha * log(bag_prob) + return self.alpha * F.binary_cross_entropy( + bag_prob, torch.ones_like(bag_prob), reduction='none') + + def negative_bag_loss(self, cls_prob, box_prob): + """Compute negative bag loss. + + :math:`FL((1 - P_{a_{j} \in A_{+}}) * (1 - P_{j}^{bg}))`. + + :math:`P_{a_{j} \in A_{+}}`: Box_probability of matched samples. + + :math:`P_{j}^{bg}`: Classification probability of negative samples. + + Args: + cls_prob (Tensor): Classification probability, in shape + (num_img, num_anchors, num_classes). + box_prob (Tensor): Box probability, in shape + (num_img, num_anchors, num_classes). + + Returns: + Tensor: Negative bag loss in shape (num_img, num_anchors, num_classes). + """ # noqa: E501, W605 + prob = cls_prob * (1 - box_prob) + # There are some cases when neg_prob = 0. + # This will cause the neg_prob.log() to be inf without clamp. + prob = prob.clamp(min=EPS, max=1 - EPS) + negative_bag_loss = prob**self.gamma * F.binary_cross_entropy( + prob, torch.zeros_like(prob), reduction='none') + return (1 - self.alpha) * negative_bag_loss diff --git a/detection/mmdet/models/dense_heads/fsaf_head.py b/detection/mmdet/models/dense_heads/fsaf_head.py new file mode 100644 index 0000000..7183efc --- /dev/null +++ b/detection/mmdet/models/dense_heads/fsaf_head.py @@ -0,0 +1,422 @@ +import numpy as np +import torch +from mmcv.cnn import normal_init +from mmcv.runner import force_fp32 + +from mmdet.core import (anchor_inside_flags, images_to_levels, multi_apply, + unmap) +from ..builder import HEADS +from ..losses.accuracy import accuracy +from ..losses.utils import weight_reduce_loss +from .retina_head import RetinaHead + + +@HEADS.register_module() +class FSAFHead(RetinaHead): + """Anchor-free head used in `FSAF `_. + + The head contains two subnetworks. The first classifies anchor boxes and + the second regresses deltas for the anchors (num_anchors is 1 for anchor- + free methods) + + Args: + *args: Same as its base class in :class:`RetinaHead` + score_threshold (float, optional): The score_threshold to calculate + positive recall. If given, prediction scores lower than this value + is counted as incorrect prediction. Default to None. + **kwargs: Same as its base class in :class:`RetinaHead` + + Example: + >>> import torch + >>> self = FSAFHead(11, 7) + >>> x = torch.rand(1, 7, 32, 32) + >>> cls_score, bbox_pred = self.forward_single(x) + >>> # Each anchor predicts a score for each class except background + >>> cls_per_anchor = cls_score.shape[1] / self.num_anchors + >>> box_per_anchor = bbox_pred.shape[1] / self.num_anchors + >>> assert cls_per_anchor == self.num_classes + >>> assert box_per_anchor == 4 + """ + + def __init__(self, *args, score_threshold=None, **kwargs): + super().__init__(*args, **kwargs) + self.score_threshold = score_threshold + + def forward_single(self, x): + """Forward feature map of a single scale level. + + Args: + x (Tensor): Feature map of a single scale level. + + Returns: + tuple (Tensor): + cls_score (Tensor): Box scores for each scale level + Has shape (N, num_points * num_classes, H, W). + bbox_pred (Tensor): Box energies / deltas for each scale + level with shape (N, num_points * 4, H, W). + """ + cls_score, bbox_pred = super().forward_single(x) + # relu: TBLR encoder only accepts positive bbox_pred + return cls_score, self.relu(bbox_pred) + + def init_weights(self): + """Initialize weights of the head.""" + super(FSAFHead, self).init_weights() + # The positive bias in self.retina_reg conv is to prevent predicted \ + # bbox with 0 area + normal_init(self.retina_reg, std=0.01, bias=0.25) + + def _get_targets_single(self, + flat_anchors, + valid_flags, + gt_bboxes, + gt_bboxes_ignore, + gt_labels, + img_meta, + label_channels=1, + unmap_outputs=True): + """Compute regression and classification targets for anchors in a + single image. + + Most of the codes are the same with the base class + :obj: `AnchorHead`, except that it also collects and returns + the matched gt index in the image (from 0 to num_gt-1). If the + anchor bbox is not matched to any gt, the corresponding value in + pos_gt_inds is -1. + """ + inside_flags = anchor_inside_flags(flat_anchors, valid_flags, + img_meta['img_shape'][:2], + self.train_cfg.allowed_border) + if not inside_flags.any(): + return (None, ) * 7 + # Assign gt and sample anchors + anchors = flat_anchors[inside_flags.type(torch.bool), :] + assign_result = self.assigner.assign( + anchors, gt_bboxes, gt_bboxes_ignore, + None if self.sampling else gt_labels) + + sampling_result = self.sampler.sample(assign_result, anchors, + gt_bboxes) + + num_valid_anchors = anchors.shape[0] + bbox_targets = torch.zeros_like(anchors) + bbox_weights = torch.zeros_like(anchors) + labels = anchors.new_full((num_valid_anchors, ), + self.num_classes, + dtype=torch.long) + label_weights = anchors.new_zeros((num_valid_anchors, label_channels), + dtype=torch.float) + pos_gt_inds = anchors.new_full((num_valid_anchors, ), + -1, + dtype=torch.long) + + pos_inds = sampling_result.pos_inds + neg_inds = sampling_result.neg_inds + + if len(pos_inds) > 0: + if not self.reg_decoded_bbox: + pos_bbox_targets = self.bbox_coder.encode( + sampling_result.pos_bboxes, sampling_result.pos_gt_bboxes) + else: + # When the regression loss (e.g. `IouLoss`, `GIouLoss`) + # is applied directly on the decoded bounding boxes, both + # the predicted boxes and regression targets should be with + # absolute coordinate format. + pos_bbox_targets = sampling_result.pos_gt_bboxes + bbox_targets[pos_inds, :] = pos_bbox_targets + bbox_weights[pos_inds, :] = 1.0 + # The assigned gt_index for each anchor. (0-based) + pos_gt_inds[pos_inds] = sampling_result.pos_assigned_gt_inds + if gt_labels is None: + # Only rpn gives gt_labels as None + # Foreground is the first class + labels[pos_inds] = 0 + else: + labels[pos_inds] = gt_labels[ + sampling_result.pos_assigned_gt_inds] + if self.train_cfg.pos_weight <= 0: + label_weights[pos_inds] = 1.0 + else: + label_weights[pos_inds] = self.train_cfg.pos_weight + + if len(neg_inds) > 0: + label_weights[neg_inds] = 1.0 + + # shadowed_labels is a tensor composed of tuples + # (anchor_inds, class_label) that indicate those anchors lying in the + # outer region of a gt or overlapped by another gt with a smaller + # area. + # + # Therefore, only the shadowed labels are ignored for loss calculation. + # the key `shadowed_labels` is defined in :obj:`CenterRegionAssigner` + shadowed_labels = assign_result.get_extra_property('shadowed_labels') + if shadowed_labels is not None and shadowed_labels.numel(): + if len(shadowed_labels.shape) == 2: + idx_, label_ = shadowed_labels[:, 0], shadowed_labels[:, 1] + assert (labels[idx_] != label_).all(), \ + 'One label cannot be both positive and ignored' + label_weights[idx_, label_] = 0 + else: + label_weights[shadowed_labels] = 0 + + # map up to original set of anchors + if unmap_outputs: + num_total_anchors = flat_anchors.size(0) + labels = unmap(labels, num_total_anchors, inside_flags) + label_weights = unmap(label_weights, num_total_anchors, + inside_flags) + bbox_targets = unmap(bbox_targets, num_total_anchors, inside_flags) + bbox_weights = unmap(bbox_weights, num_total_anchors, inside_flags) + pos_gt_inds = unmap( + pos_gt_inds, num_total_anchors, inside_flags, fill=-1) + + return (labels, label_weights, bbox_targets, bbox_weights, pos_inds, + neg_inds, sampling_result, pos_gt_inds) + + @force_fp32(apply_to=('cls_scores', 'bbox_preds')) + def loss(self, + cls_scores, + bbox_preds, + gt_bboxes, + gt_labels, + img_metas, + gt_bboxes_ignore=None): + """Compute loss of the head. + + Args: + cls_scores (list[Tensor]): Box scores for each scale level + Has shape (N, num_points * num_classes, H, W). + bbox_preds (list[Tensor]): Box energies / deltas for each scale + level with shape (N, num_points * 4, H, W). + gt_bboxes (list[Tensor]): each item are the truth boxes for each + image in [tl_x, tl_y, br_x, br_y] format. + gt_labels (list[Tensor]): class indices corresponding to each box + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + gt_bboxes_ignore (None | list[Tensor]): specify which bounding + boxes can be ignored when computing the loss. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + for i in range(len(bbox_preds)): # loop over fpn level + # avoid 0 area of the predicted bbox + bbox_preds[i] = bbox_preds[i].clamp(min=1e-4) + # TODO: It may directly use the base-class loss function. + featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] + assert len(featmap_sizes) == self.anchor_generator.num_levels + batch_size = len(gt_bboxes) + device = cls_scores[0].device + anchor_list, valid_flag_list = self.get_anchors( + featmap_sizes, img_metas, device=device) + label_channels = self.cls_out_channels if self.use_sigmoid_cls else 1 + cls_reg_targets = self.get_targets( + anchor_list, + valid_flag_list, + gt_bboxes, + img_metas, + gt_bboxes_ignore_list=gt_bboxes_ignore, + gt_labels_list=gt_labels, + label_channels=label_channels) + if cls_reg_targets is None: + return None + (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, + num_total_pos, num_total_neg, + pos_assigned_gt_inds_list) = cls_reg_targets + + num_gts = np.array(list(map(len, gt_labels))) + num_total_samples = ( + num_total_pos + num_total_neg if self.sampling else num_total_pos) + # anchor number of multi levels + num_level_anchors = [anchors.size(0) for anchors in anchor_list[0]] + # concat all level anchors and flags to a single tensor + concat_anchor_list = [] + for i in range(len(anchor_list)): + concat_anchor_list.append(torch.cat(anchor_list[i])) + all_anchor_list = images_to_levels(concat_anchor_list, + num_level_anchors) + losses_cls, losses_bbox = multi_apply( + self.loss_single, + cls_scores, + bbox_preds, + all_anchor_list, + labels_list, + label_weights_list, + bbox_targets_list, + bbox_weights_list, + num_total_samples=num_total_samples) + + # `pos_assigned_gt_inds_list` (length: fpn_levels) stores the assigned + # gt index of each anchor bbox in each fpn level. + cum_num_gts = list(np.cumsum(num_gts)) # length of batch_size + for i, assign in enumerate(pos_assigned_gt_inds_list): + # loop over fpn levels + for j in range(1, batch_size): + # loop over batch size + # Convert gt indices in each img to those in the batch + assign[j][assign[j] >= 0] += int(cum_num_gts[j - 1]) + pos_assigned_gt_inds_list[i] = assign.flatten() + labels_list[i] = labels_list[i].flatten() + num_gts = sum(map(len, gt_labels)) # total number of gt in the batch + # The unique label index of each gt in the batch + label_sequence = torch.arange(num_gts, device=device) + # Collect the average loss of each gt in each level + with torch.no_grad(): + loss_levels, = multi_apply( + self.collect_loss_level_single, + losses_cls, + losses_bbox, + pos_assigned_gt_inds_list, + labels_seq=label_sequence) + # Shape: (fpn_levels, num_gts). Loss of each gt at each fpn level + loss_levels = torch.stack(loss_levels, dim=0) + # Locate the best fpn level for loss back-propagation + if loss_levels.numel() == 0: # zero gt + argmin = loss_levels.new_empty((num_gts, ), dtype=torch.long) + else: + _, argmin = loss_levels.min(dim=0) + + # Reweight the loss of each (anchor, label) pair, so that only those + # at the best gt level are back-propagated. + losses_cls, losses_bbox, pos_inds = multi_apply( + self.reweight_loss_single, + losses_cls, + losses_bbox, + pos_assigned_gt_inds_list, + labels_list, + list(range(len(losses_cls))), + min_levels=argmin) + num_pos = torch.cat(pos_inds, 0).sum().float() + pos_recall = self.calculate_pos_recall(cls_scores, labels_list, + pos_inds) + + if num_pos == 0: # No gt + avg_factor = num_pos + float(num_total_neg) + else: + avg_factor = num_pos + for i in range(len(losses_cls)): + losses_cls[i] /= avg_factor + losses_bbox[i] /= avg_factor + return dict( + loss_cls=losses_cls, + loss_bbox=losses_bbox, + num_pos=num_pos / batch_size, + pos_recall=pos_recall) + + def calculate_pos_recall(self, cls_scores, labels_list, pos_inds): + """Calculate positive recall with score threshold. + + Args: + cls_scores (list[Tensor]): Classification scores at all fpn levels. + Each tensor is in shape (N, num_classes * num_anchors, H, W) + labels_list (list[Tensor]): The label that each anchor is assigned + to. Shape (N * H * W * num_anchors, ) + pos_inds (list[Tensor]): List of bool tensors indicating whether + the anchor is assigned to a positive label. + Shape (N * H * W * num_anchors, ) + + Returns: + Tensor: A single float number indicating the positive recall. + """ + with torch.no_grad(): + num_class = self.num_classes + scores = [ + cls.permute(0, 2, 3, 1).reshape(-1, num_class)[pos] + for cls, pos in zip(cls_scores, pos_inds) + ] + labels = [ + label.reshape(-1)[pos] + for label, pos in zip(labels_list, pos_inds) + ] + scores = torch.cat(scores, dim=0) + labels = torch.cat(labels, dim=0) + if self.use_sigmoid_cls: + scores = scores.sigmoid() + else: + scores = scores.softmax(dim=1) + + return accuracy(scores, labels, thresh=self.score_threshold) + + def collect_loss_level_single(self, cls_loss, reg_loss, assigned_gt_inds, + labels_seq): + """Get the average loss in each FPN level w.r.t. each gt label. + + Args: + cls_loss (Tensor): Classification loss of each feature map pixel, + shape (num_anchor, num_class) + reg_loss (Tensor): Regression loss of each feature map pixel, + shape (num_anchor, 4) + assigned_gt_inds (Tensor): It indicates which gt the prior is + assigned to (0-based, -1: no assignment). shape (num_anchor), + labels_seq: The rank of labels. shape (num_gt) + + Returns: + shape: (num_gt), average loss of each gt in this level + """ + if len(reg_loss.shape) == 2: # iou loss has shape (num_prior, 4) + reg_loss = reg_loss.sum(dim=-1) # sum loss in tblr dims + if len(cls_loss.shape) == 2: + cls_loss = cls_loss.sum(dim=-1) # sum loss in class dims + loss = cls_loss + reg_loss + assert loss.size(0) == assigned_gt_inds.size(0) + # Default loss value is 1e6 for a layer where no anchor is positive + # to ensure it will not be chosen to back-propagate gradient + losses_ = loss.new_full(labels_seq.shape, 1e6) + for i, l in enumerate(labels_seq): + match = assigned_gt_inds == l + if match.any(): + losses_[i] = loss[match].mean() + return losses_, + + def reweight_loss_single(self, cls_loss, reg_loss, assigned_gt_inds, + labels, level, min_levels): + """Reweight loss values at each level. + + Reassign loss values at each level by masking those where the + pre-calculated loss is too large. Then return the reduced losses. + + Args: + cls_loss (Tensor): Element-wise classification loss. + Shape: (num_anchors, num_classes) + reg_loss (Tensor): Element-wise regression loss. + Shape: (num_anchors, 4) + assigned_gt_inds (Tensor): The gt indices that each anchor bbox + is assigned to. -1 denotes a negative anchor, otherwise it is the + gt index (0-based). Shape: (num_anchors, ), + labels (Tensor): Label assigned to anchors. Shape: (num_anchors, ). + level (int): The current level index in the pyramid + (0-4 for RetinaNet) + min_levels (Tensor): The best-matching level for each gt. + Shape: (num_gts, ), + + Returns: + tuple: + - cls_loss: Reduced corrected classification loss. Scalar. + - reg_loss: Reduced corrected regression loss. Scalar. + - pos_flags (Tensor): Corrected bool tensor indicating the + final positive anchors. Shape: (num_anchors, ). + """ + loc_weight = torch.ones_like(reg_loss) + cls_weight = torch.ones_like(cls_loss) + pos_flags = assigned_gt_inds >= 0 # positive pixel flag + pos_indices = torch.nonzero(pos_flags, as_tuple=False).flatten() + + if pos_flags.any(): # pos pixels exist + pos_assigned_gt_inds = assigned_gt_inds[pos_flags] + zeroing_indices = (min_levels[pos_assigned_gt_inds] != level) + neg_indices = pos_indices[zeroing_indices] + + if neg_indices.numel(): + pos_flags[neg_indices] = 0 + loc_weight[neg_indices] = 0 + # Only the weight corresponding to the label is + # zeroed out if not selected + zeroing_labels = labels[neg_indices] + assert (zeroing_labels >= 0).all() + cls_weight[neg_indices, zeroing_labels] = 0 + + # Weighted loss for both cls and reg loss + cls_loss = weight_reduce_loss(cls_loss, cls_weight, reduction='sum') + reg_loss = weight_reduce_loss(reg_loss, loc_weight, reduction='sum') + + return cls_loss, reg_loss, pos_flags diff --git a/detection/mmdet/models/dense_heads/ga_retina_head.py b/detection/mmdet/models/dense_heads/ga_retina_head.py new file mode 100644 index 0000000..8822d1c --- /dev/null +++ b/detection/mmdet/models/dense_heads/ga_retina_head.py @@ -0,0 +1,109 @@ +import torch.nn as nn +from mmcv.cnn import ConvModule, bias_init_with_prob, normal_init +from mmcv.ops import MaskedConv2d + +from ..builder import HEADS +from .guided_anchor_head import FeatureAdaption, GuidedAnchorHead + + +@HEADS.register_module() +class GARetinaHead(GuidedAnchorHead): + """Guided-Anchor-based RetinaNet head.""" + + def __init__(self, + num_classes, + in_channels, + stacked_convs=4, + conv_cfg=None, + norm_cfg=None, + **kwargs): + self.stacked_convs = stacked_convs + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + super(GARetinaHead, self).__init__(num_classes, in_channels, **kwargs) + + def _init_layers(self): + """Initialize layers of the head.""" + self.relu = nn.ReLU(inplace=True) + self.cls_convs = nn.ModuleList() + self.reg_convs = nn.ModuleList() + for i in range(self.stacked_convs): + chn = self.in_channels if i == 0 else self.feat_channels + self.cls_convs.append( + ConvModule( + chn, + self.feat_channels, + 3, + stride=1, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg)) + self.reg_convs.append( + ConvModule( + chn, + self.feat_channels, + 3, + stride=1, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg)) + + self.conv_loc = nn.Conv2d(self.feat_channels, 1, 1) + self.conv_shape = nn.Conv2d(self.feat_channels, self.num_anchors * 2, + 1) + self.feature_adaption_cls = FeatureAdaption( + self.feat_channels, + self.feat_channels, + kernel_size=3, + deform_groups=self.deform_groups) + self.feature_adaption_reg = FeatureAdaption( + self.feat_channels, + self.feat_channels, + kernel_size=3, + deform_groups=self.deform_groups) + self.retina_cls = MaskedConv2d( + self.feat_channels, + self.num_anchors * self.cls_out_channels, + 3, + padding=1) + self.retina_reg = MaskedConv2d( + self.feat_channels, self.num_anchors * 4, 3, padding=1) + + def init_weights(self): + """Initialize weights of the layer.""" + for m in self.cls_convs: + normal_init(m.conv, std=0.01) + for m in self.reg_convs: + normal_init(m.conv, std=0.01) + + self.feature_adaption_cls.init_weights() + self.feature_adaption_reg.init_weights() + + bias_cls = bias_init_with_prob(0.01) + normal_init(self.conv_loc, std=0.01, bias=bias_cls) + normal_init(self.conv_shape, std=0.01) + normal_init(self.retina_cls, std=0.01, bias=bias_cls) + normal_init(self.retina_reg, std=0.01) + + def forward_single(self, x): + """Forward feature map of a single scale level.""" + cls_feat = x + reg_feat = x + for cls_conv in self.cls_convs: + cls_feat = cls_conv(cls_feat) + for reg_conv in self.reg_convs: + reg_feat = reg_conv(reg_feat) + + loc_pred = self.conv_loc(cls_feat) + shape_pred = self.conv_shape(reg_feat) + + cls_feat = self.feature_adaption_cls(cls_feat, shape_pred) + reg_feat = self.feature_adaption_reg(reg_feat, shape_pred) + + if not self.training: + mask = loc_pred.sigmoid()[0] >= self.loc_filter_thr + else: + mask = None + cls_score = self.retina_cls(cls_feat, mask) + bbox_pred = self.retina_reg(reg_feat, mask) + return cls_score, bbox_pred, shape_pred, loc_pred diff --git a/detection/mmdet/models/dense_heads/ga_rpn_head.py b/detection/mmdet/models/dense_heads/ga_rpn_head.py new file mode 100644 index 0000000..2ec0d4f --- /dev/null +++ b/detection/mmdet/models/dense_heads/ga_rpn_head.py @@ -0,0 +1,171 @@ +import copy +import warnings + +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv import ConfigDict +from mmcv.cnn import normal_init +from mmcv.ops import nms + +from ..builder import HEADS +from .guided_anchor_head import GuidedAnchorHead +from .rpn_test_mixin import RPNTestMixin + + +@HEADS.register_module() +class GARPNHead(RPNTestMixin, GuidedAnchorHead): + """Guided-Anchor-based RPN head.""" + + def __init__(self, in_channels, **kwargs): + super(GARPNHead, self).__init__(1, in_channels, **kwargs) + + def _init_layers(self): + """Initialize layers of the head.""" + self.rpn_conv = nn.Conv2d( + self.in_channels, self.feat_channels, 3, padding=1) + super(GARPNHead, self)._init_layers() + + def init_weights(self): + """Initialize weights of the head.""" + normal_init(self.rpn_conv, std=0.01) + super(GARPNHead, self).init_weights() + + def forward_single(self, x): + """Forward feature of a single scale level.""" + + x = self.rpn_conv(x) + x = F.relu(x, inplace=True) + (cls_score, bbox_pred, shape_pred, + loc_pred) = super(GARPNHead, self).forward_single(x) + return cls_score, bbox_pred, shape_pred, loc_pred + + def loss(self, + cls_scores, + bbox_preds, + shape_preds, + loc_preds, + gt_bboxes, + img_metas, + gt_bboxes_ignore=None): + losses = super(GARPNHead, self).loss( + cls_scores, + bbox_preds, + shape_preds, + loc_preds, + gt_bboxes, + None, + img_metas, + gt_bboxes_ignore=gt_bboxes_ignore) + return dict( + loss_rpn_cls=losses['loss_cls'], + loss_rpn_bbox=losses['loss_bbox'], + loss_anchor_shape=losses['loss_shape'], + loss_anchor_loc=losses['loss_loc']) + + def _get_bboxes_single(self, + cls_scores, + bbox_preds, + mlvl_anchors, + mlvl_masks, + img_shape, + scale_factor, + cfg, + rescale=False): + cfg = self.test_cfg if cfg is None else cfg + + cfg = copy.deepcopy(cfg) + + # deprecate arguments warning + if 'nms' not in cfg or 'max_num' in cfg or 'nms_thr' in cfg: + warnings.warn( + 'In rpn_proposal or test_cfg, ' + 'nms_thr has been moved to a dict named nms as ' + 'iou_threshold, max_num has been renamed as max_per_img, ' + 'name of original arguments and the way to specify ' + 'iou_threshold of NMS will be deprecated.') + if 'nms' not in cfg: + cfg.nms = ConfigDict(dict(type='nms', iou_threshold=cfg.nms_thr)) + if 'max_num' in cfg: + if 'max_per_img' in cfg: + assert cfg.max_num == cfg.max_per_img, f'You ' \ + f'set max_num and max_per_img at the same time, ' \ + f'but get {cfg.max_num} ' \ + f'and {cfg.max_per_img} respectively' \ + 'Please delete max_num which will be deprecated.' + else: + cfg.max_per_img = cfg.max_num + if 'nms_thr' in cfg: + assert cfg.nms.iou_threshold == cfg.nms_thr, f'You set ' \ + f'iou_threshold in nms and ' \ + f'nms_thr at the same time, but get ' \ + f'{cfg.nms.iou_threshold} and {cfg.nms_thr}' \ + f' respectively. Please delete the ' \ + f'nms_thr which will be deprecated.' + + assert cfg.nms.get('type', 'nms') == 'nms', 'GARPNHead only support ' \ + 'naive nms.' + + mlvl_proposals = [] + for idx in range(len(cls_scores)): + rpn_cls_score = cls_scores[idx] + rpn_bbox_pred = bbox_preds[idx] + anchors = mlvl_anchors[idx] + mask = mlvl_masks[idx] + assert rpn_cls_score.size()[-2:] == rpn_bbox_pred.size()[-2:] + # if no location is kept, end. + if mask.sum() == 0: + continue + rpn_cls_score = rpn_cls_score.permute(1, 2, 0) + if self.use_sigmoid_cls: + rpn_cls_score = rpn_cls_score.reshape(-1) + scores = rpn_cls_score.sigmoid() + else: + rpn_cls_score = rpn_cls_score.reshape(-1, 2) + # remind that we set FG labels to [0, num_class-1] + # since mmdet v2.0 + # BG cat_id: num_class + scores = rpn_cls_score.softmax(dim=1)[:, :-1] + # filter scores, bbox_pred w.r.t. mask. + # anchors are filtered in get_anchors() beforehand. + scores = scores[mask] + rpn_bbox_pred = rpn_bbox_pred.permute(1, 2, 0).reshape(-1, + 4)[mask, :] + if scores.dim() == 0: + rpn_bbox_pred = rpn_bbox_pred.unsqueeze(0) + anchors = anchors.unsqueeze(0) + scores = scores.unsqueeze(0) + # filter anchors, bbox_pred, scores w.r.t. scores + if cfg.nms_pre > 0 and scores.shape[0] > cfg.nms_pre: + _, topk_inds = scores.topk(cfg.nms_pre) + rpn_bbox_pred = rpn_bbox_pred[topk_inds, :] + anchors = anchors[topk_inds, :] + scores = scores[topk_inds] + # get proposals w.r.t. anchors and rpn_bbox_pred + proposals = self.bbox_coder.decode( + anchors, rpn_bbox_pred, max_shape=img_shape) + # filter out too small bboxes + if cfg.min_bbox_size > 0: + w = proposals[:, 2] - proposals[:, 0] + h = proposals[:, 3] - proposals[:, 1] + valid_inds = torch.nonzero( + (w >= cfg.min_bbox_size) & (h >= cfg.min_bbox_size), + as_tuple=False).squeeze() + proposals = proposals[valid_inds, :] + scores = scores[valid_inds] + # NMS in current level + proposals, _ = nms(proposals, scores, cfg.nms.iou_threshold) + proposals = proposals[:cfg.nms_post, :] + mlvl_proposals.append(proposals) + proposals = torch.cat(mlvl_proposals, 0) + if cfg.get('nms_across_levels', False): + # NMS across multi levels + proposals, _ = nms(proposals[:, :4], proposals[:, -1], + cfg.nms.iou_threshold) + proposals = proposals[:cfg.max_per_img, :] + else: + scores = proposals[:, 4] + num = min(cfg.max_per_img, proposals.shape[0]) + _, topk_inds = scores.topk(num) + proposals = proposals[topk_inds, :] + return proposals diff --git a/detection/mmdet/models/dense_heads/gfl_head.py b/detection/mmdet/models/dense_heads/gfl_head.py new file mode 100644 index 0000000..961bc92 --- /dev/null +++ b/detection/mmdet/models/dense_heads/gfl_head.py @@ -0,0 +1,647 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import ConvModule, Scale, bias_init_with_prob, normal_init +from mmcv.runner import force_fp32 + +from mmdet.core import (anchor_inside_flags, bbox2distance, bbox_overlaps, + build_assigner, build_sampler, distance2bbox, + images_to_levels, multi_apply, multiclass_nms, + reduce_mean, unmap) +from ..builder import HEADS, build_loss +from .anchor_head import AnchorHead + + +class Integral(nn.Module): + """A fixed layer for calculating integral result from distribution. + + This layer calculates the target location by :math: `sum{P(y_i) * y_i}`, + P(y_i) denotes the softmax vector that represents the discrete distribution + y_i denotes the discrete set, usually {0, 1, 2, ..., reg_max} + + Args: + reg_max (int): The maximal value of the discrete set. Default: 16. You + may want to reset it according to your new dataset or related + settings. + """ + + def __init__(self, reg_max=16): + super(Integral, self).__init__() + self.reg_max = reg_max + self.register_buffer('project', + torch.linspace(0, self.reg_max, self.reg_max + 1)) + + def forward(self, x): + """Forward feature from the regression head to get integral result of + bounding box location. + + Args: + x (Tensor): Features of the regression head, shape (N, 4*(n+1)), + n is self.reg_max. + + Returns: + x (Tensor): Integral result of box locations, i.e., distance + offsets from the box center in four directions, shape (N, 4). + """ + x = F.softmax(x.reshape(-1, self.reg_max + 1), dim=1) + x = F.linear(x, self.project.type_as(x)).reshape(-1, 4) + return x + + +@HEADS.register_module() +class GFLHead(AnchorHead): + """Generalized Focal Loss: Learning Qualified and Distributed Bounding + Boxes for Dense Object Detection. + + GFL head structure is similar with ATSS, however GFL uses + 1) joint representation for classification and localization quality, and + 2) flexible General distribution for bounding box locations, + which are supervised by + Quality Focal Loss (QFL) and Distribution Focal Loss (DFL), respectively + + https://arxiv.org/abs/2006.04388 + + Args: + num_classes (int): Number of categories excluding the background + category. + in_channels (int): Number of channels in the input feature map. + stacked_convs (int): Number of conv layers in cls and reg tower. + Default: 4. + conv_cfg (dict): dictionary to construct and config conv layer. + Default: None. + norm_cfg (dict): dictionary to construct and config norm layer. + Default: dict(type='GN', num_groups=32, requires_grad=True). + loss_qfl (dict): Config of Quality Focal Loss (QFL). + reg_max (int): Max value of integral set :math: `{0, ..., reg_max}` + in QFL setting. Default: 16. + Example: + >>> self = GFLHead(11, 7) + >>> feats = [torch.rand(1, 7, s, s) for s in [4, 8, 16, 32, 64]] + >>> cls_quality_score, bbox_pred = self.forward(feats) + >>> assert len(cls_quality_score) == len(self.scales) + """ + + def __init__(self, + num_classes, + in_channels, + stacked_convs=4, + conv_cfg=None, + norm_cfg=dict(type='GN', num_groups=32, requires_grad=True), + loss_dfl=dict(type='DistributionFocalLoss', loss_weight=0.25), + reg_max=16, + **kwargs): + self.stacked_convs = stacked_convs + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.reg_max = reg_max + super(GFLHead, self).__init__(num_classes, in_channels, **kwargs) + + self.sampling = False + if self.train_cfg: + self.assigner = build_assigner(self.train_cfg.assigner) + # SSD sampling=False so use PseudoSampler + sampler_cfg = dict(type='PseudoSampler') + self.sampler = build_sampler(sampler_cfg, context=self) + + self.integral = Integral(self.reg_max) + self.loss_dfl = build_loss(loss_dfl) + + def _init_layers(self): + """Initialize layers of the head.""" + self.relu = nn.ReLU(inplace=True) + self.cls_convs = nn.ModuleList() + self.reg_convs = nn.ModuleList() + for i in range(self.stacked_convs): + chn = self.in_channels if i == 0 else self.feat_channels + self.cls_convs.append( + ConvModule( + chn, + self.feat_channels, + 3, + stride=1, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg)) + self.reg_convs.append( + ConvModule( + chn, + self.feat_channels, + 3, + stride=1, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg)) + assert self.num_anchors == 1, 'anchor free version' + self.gfl_cls = nn.Conv2d( + self.feat_channels, self.cls_out_channels, 3, padding=1) + self.gfl_reg = nn.Conv2d( + self.feat_channels, 4 * (self.reg_max + 1), 3, padding=1) + self.scales = nn.ModuleList( + [Scale(1.0) for _ in self.anchor_generator.strides]) + + def init_weights(self): + """Initialize weights of the head.""" + for m in self.cls_convs: + normal_init(m.conv, std=0.01) + for m in self.reg_convs: + normal_init(m.conv, std=0.01) + bias_cls = bias_init_with_prob(0.01) + normal_init(self.gfl_cls, std=0.01, bias=bias_cls) + normal_init(self.gfl_reg, std=0.01) + + def forward(self, feats): + """Forward features from the upstream network. + + Args: + feats (tuple[Tensor]): Features from the upstream network, each is + a 4D-tensor. + + Returns: + tuple: Usually a tuple of classification scores and bbox prediction + cls_scores (list[Tensor]): Classification and quality (IoU) + joint scores for all scale levels, each is a 4D-tensor, + the channel number is num_classes. + bbox_preds (list[Tensor]): Box distribution logits for all + scale levels, each is a 4D-tensor, the channel number is + 4*(n+1), n is max value of integral set. + """ + return multi_apply(self.forward_single, feats, self.scales) + + def forward_single(self, x, scale): + """Forward feature of a single scale level. + + Args: + x (Tensor): Features of a single scale level. + scale (:obj: `mmcv.cnn.Scale`): Learnable scale module to resize + the bbox prediction. + + Returns: + tuple: + cls_score (Tensor): Cls and quality joint scores for a single + scale level the channel number is num_classes. + bbox_pred (Tensor): Box distribution logits for a single scale + level, the channel number is 4*(n+1), n is max value of + integral set. + """ + cls_feat = x + reg_feat = x + for cls_conv in self.cls_convs: + cls_feat = cls_conv(cls_feat) + for reg_conv in self.reg_convs: + reg_feat = reg_conv(reg_feat) + cls_score = self.gfl_cls(cls_feat) + bbox_pred = scale(self.gfl_reg(reg_feat)).float() + return cls_score, bbox_pred + + def anchor_center(self, anchors): + """Get anchor centers from anchors. + + Args: + anchors (Tensor): Anchor list with shape (N, 4), "xyxy" format. + + Returns: + Tensor: Anchor centers with shape (N, 2), "xy" format. + """ + anchors_cx = (anchors[..., 2] + anchors[..., 0]) / 2 + anchors_cy = (anchors[..., 3] + anchors[..., 1]) / 2 + return torch.stack([anchors_cx, anchors_cy], dim=-1) + + def loss_single(self, anchors, cls_score, bbox_pred, labels, label_weights, + bbox_targets, stride, num_total_samples): + """Compute loss of a single scale level. + + Args: + anchors (Tensor): Box reference for each scale level with shape + (N, num_total_anchors, 4). + cls_score (Tensor): Cls and quality joint scores for each scale + level has shape (N, num_classes, H, W). + bbox_pred (Tensor): Box distribution logits for each scale + level with shape (N, 4*(n+1), H, W), n is max value of integral + set. + labels (Tensor): Labels of each anchors with shape + (N, num_total_anchors). + label_weights (Tensor): Label weights of each anchor with shape + (N, num_total_anchors) + bbox_targets (Tensor): BBox regression targets of each anchor wight + shape (N, num_total_anchors, 4). + stride (tuple): Stride in this scale level. + num_total_samples (int): Number of positive samples that is + reduced over all GPUs. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + assert stride[0] == stride[1], 'h stride is not equal to w stride!' + anchors = anchors.reshape(-1, 4) + cls_score = cls_score.permute(0, 2, 3, + 1).reshape(-1, self.cls_out_channels) + bbox_pred = bbox_pred.permute(0, 2, 3, + 1).reshape(-1, 4 * (self.reg_max + 1)) + bbox_targets = bbox_targets.reshape(-1, 4) + labels = labels.reshape(-1) + label_weights = label_weights.reshape(-1) + + # FG cat_id: [0, num_classes -1], BG cat_id: num_classes + bg_class_ind = self.num_classes + pos_inds = ((labels >= 0) + & (labels < bg_class_ind)).nonzero().squeeze(1) + score = label_weights.new_zeros(labels.shape) + + if len(pos_inds) > 0: + pos_bbox_targets = bbox_targets[pos_inds] + pos_bbox_pred = bbox_pred[pos_inds] + pos_anchors = anchors[pos_inds] + pos_anchor_centers = self.anchor_center(pos_anchors) / stride[0] + + weight_targets = cls_score.detach().sigmoid() + weight_targets = weight_targets.max(dim=1)[0][pos_inds] + pos_bbox_pred_corners = self.integral(pos_bbox_pred) + pos_decode_bbox_pred = distance2bbox(pos_anchor_centers, + pos_bbox_pred_corners) + pos_decode_bbox_targets = pos_bbox_targets / stride[0] + score[pos_inds] = bbox_overlaps( + pos_decode_bbox_pred.detach(), + pos_decode_bbox_targets, + is_aligned=True) + pred_corners = pos_bbox_pred.reshape(-1, self.reg_max + 1) + target_corners = bbox2distance(pos_anchor_centers, + pos_decode_bbox_targets, + self.reg_max).reshape(-1) + + # regression loss + loss_bbox = self.loss_bbox( + pos_decode_bbox_pred, + pos_decode_bbox_targets, + weight=weight_targets, + avg_factor=1.0) + + # dfl loss + loss_dfl = self.loss_dfl( + pred_corners, + target_corners, + weight=weight_targets[:, None].expand(-1, 4).reshape(-1), + avg_factor=4.0) + else: + loss_bbox = bbox_pred.sum() * 0 + loss_dfl = bbox_pred.sum() * 0 + weight_targets = bbox_pred.new_tensor(0) + + # cls (qfl) loss + loss_cls = self.loss_cls( + cls_score, (labels, score), + weight=label_weights, + avg_factor=num_total_samples) + + return loss_cls, loss_bbox, loss_dfl, weight_targets.sum() + + @force_fp32(apply_to=('cls_scores', 'bbox_preds')) + def loss(self, + cls_scores, + bbox_preds, + gt_bboxes, + gt_labels, + img_metas, + gt_bboxes_ignore=None): + """Compute losses of the head. + + Args: + cls_scores (list[Tensor]): Cls and quality scores for each scale + level has shape (N, num_classes, H, W). + bbox_preds (list[Tensor]): Box distribution logits for each scale + level with shape (N, 4*(n+1), H, W), n is max value of integral + set. + gt_bboxes (list[Tensor]): Ground truth bboxes for each image with + shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + gt_labels (list[Tensor]): class indices corresponding to each box + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + gt_bboxes_ignore (list[Tensor] | None): specify which bounding + boxes can be ignored when computing the loss. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + + featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] + assert len(featmap_sizes) == self.anchor_generator.num_levels + + device = cls_scores[0].device + anchor_list, valid_flag_list = self.get_anchors( + featmap_sizes, img_metas, device=device) + label_channels = self.cls_out_channels if self.use_sigmoid_cls else 1 + + cls_reg_targets = self.get_targets( + anchor_list, + valid_flag_list, + gt_bboxes, + img_metas, + gt_bboxes_ignore_list=gt_bboxes_ignore, + gt_labels_list=gt_labels, + label_channels=label_channels) + if cls_reg_targets is None: + return None + + (anchor_list, labels_list, label_weights_list, bbox_targets_list, + bbox_weights_list, num_total_pos, num_total_neg) = cls_reg_targets + + num_total_samples = reduce_mean( + torch.tensor(num_total_pos, dtype=torch.float, + device=device)).item() + num_total_samples = max(num_total_samples, 1.0) + + losses_cls, losses_bbox, losses_dfl,\ + avg_factor = multi_apply( + self.loss_single, + anchor_list, + cls_scores, + bbox_preds, + labels_list, + label_weights_list, + bbox_targets_list, + self.anchor_generator.strides, + num_total_samples=num_total_samples) + + avg_factor = sum(avg_factor) + avg_factor = reduce_mean(avg_factor).item() + losses_bbox = list(map(lambda x: x / avg_factor, losses_bbox)) + losses_dfl = list(map(lambda x: x / avg_factor, losses_dfl)) + return dict( + loss_cls=losses_cls, loss_bbox=losses_bbox, loss_dfl=losses_dfl) + + def _get_bboxes(self, + cls_scores, + bbox_preds, + mlvl_anchors, + img_shapes, + scale_factors, + cfg, + rescale=False, + with_nms=True): + """Transform outputs for a single batch item into labeled boxes. + + Args: + cls_scores (list[Tensor]): Box scores for a single scale level + has shape (N, num_classes, H, W). + bbox_preds (list[Tensor]): Box distribution logits for a single + scale level with shape (N, 4*(n+1), H, W), n is max value of + integral set. + mlvl_anchors (list[Tensor]): Box reference for a single scale level + with shape (num_total_anchors, 4). + img_shapes (list[tuple[int]]): Shape of the input image, + list[(height, width, 3)]. + scale_factors (list[ndarray]): Scale factor of the image arange as + (w_scale, h_scale, w_scale, h_scale). + cfg (mmcv.Config | None): Test / postprocessing configuration, + if None, test_cfg would be used. + rescale (bool): If True, return boxes in original image space. + Default: False. + with_nms (bool): If True, do nms before return boxes. + Default: True. + + Returns: + list[tuple[Tensor, Tensor]]: Each item in result_list is 2-tuple. + The first item is an (n, 5) tensor, where 5 represent + (tl_x, tl_y, br_x, br_y, score) and the score between 0 and 1. + The shape of the second tensor in the tuple is (n,), and + each element represents the class label of the corresponding + box. + """ + cfg = self.test_cfg if cfg is None else cfg + assert len(cls_scores) == len(bbox_preds) == len(mlvl_anchors) + batch_size = cls_scores[0].shape[0] + + mlvl_bboxes = [] + mlvl_scores = [] + for cls_score, bbox_pred, stride, anchors in zip( + cls_scores, bbox_preds, self.anchor_generator.strides, + mlvl_anchors): + assert cls_score.size()[-2:] == bbox_pred.size()[-2:] + assert stride[0] == stride[1] + scores = cls_score.permute(0, 2, 3, 1).reshape( + batch_size, -1, self.cls_out_channels).sigmoid() + bbox_pred = bbox_pred.permute(0, 2, 3, 1) + + bbox_pred = self.integral(bbox_pred) * stride[0] + bbox_pred = bbox_pred.reshape(batch_size, -1, 4) + + nms_pre = cfg.get('nms_pre', -1) + if nms_pre > 0 and scores.shape[1] > nms_pre: + max_scores, _ = scores.max(-1) + _, topk_inds = max_scores.topk(nms_pre) + batch_inds = torch.arange(batch_size).view( + -1, 1).expand_as(topk_inds).long() + anchors = anchors[topk_inds, :] + bbox_pred = bbox_pred[batch_inds, topk_inds, :] + scores = scores[batch_inds, topk_inds, :] + else: + anchors = anchors.expand_as(bbox_pred) + + bboxes = distance2bbox( + self.anchor_center(anchors), bbox_pred, max_shape=img_shapes) + mlvl_bboxes.append(bboxes) + mlvl_scores.append(scores) + + batch_mlvl_bboxes = torch.cat(mlvl_bboxes, dim=1) + if rescale: + batch_mlvl_bboxes /= batch_mlvl_bboxes.new_tensor( + scale_factors).unsqueeze(1) + + batch_mlvl_scores = torch.cat(mlvl_scores, dim=1) + # Add a dummy background class to the backend when using sigmoid + # remind that we set FG labels to [0, num_class-1] since mmdet v2.0 + # BG cat_id: num_class + padding = batch_mlvl_scores.new_zeros(batch_size, + batch_mlvl_scores.shape[1], 1) + batch_mlvl_scores = torch.cat([batch_mlvl_scores, padding], dim=-1) + + if with_nms: + det_results = [] + for (mlvl_bboxes, mlvl_scores) in zip(batch_mlvl_bboxes, + batch_mlvl_scores): + det_bbox, det_label = multiclass_nms(mlvl_bboxes, mlvl_scores, + cfg.score_thr, cfg.nms, + cfg.max_per_img) + det_results.append(tuple([det_bbox, det_label])) + else: + det_results = [ + tuple(mlvl_bs) + for mlvl_bs in zip(batch_mlvl_bboxes, batch_mlvl_scores) + ] + return det_results + + def get_targets(self, + anchor_list, + valid_flag_list, + gt_bboxes_list, + img_metas, + gt_bboxes_ignore_list=None, + gt_labels_list=None, + label_channels=1, + unmap_outputs=True): + """Get targets for GFL head. + + This method is almost the same as `AnchorHead.get_targets()`. Besides + returning the targets as the parent method does, it also returns the + anchors as the first element of the returned tuple. + """ + num_imgs = len(img_metas) + assert len(anchor_list) == len(valid_flag_list) == num_imgs + + # anchor number of multi levels + num_level_anchors = [anchors.size(0) for anchors in anchor_list[0]] + num_level_anchors_list = [num_level_anchors] * num_imgs + + # concat all level anchors and flags to a single tensor + for i in range(num_imgs): + assert len(anchor_list[i]) == len(valid_flag_list[i]) + anchor_list[i] = torch.cat(anchor_list[i]) + valid_flag_list[i] = torch.cat(valid_flag_list[i]) + + # compute targets for each image + if gt_bboxes_ignore_list is None: + gt_bboxes_ignore_list = [None for _ in range(num_imgs)] + if gt_labels_list is None: + gt_labels_list = [None for _ in range(num_imgs)] + (all_anchors, all_labels, all_label_weights, all_bbox_targets, + all_bbox_weights, pos_inds_list, neg_inds_list) = multi_apply( + self._get_target_single, + anchor_list, + valid_flag_list, + num_level_anchors_list, + gt_bboxes_list, + gt_bboxes_ignore_list, + gt_labels_list, + img_metas, + label_channels=label_channels, + unmap_outputs=unmap_outputs) + # no valid anchors + if any([labels is None for labels in all_labels]): + return None + # sampled anchors of all images + num_total_pos = sum([max(inds.numel(), 1) for inds in pos_inds_list]) + num_total_neg = sum([max(inds.numel(), 1) for inds in neg_inds_list]) + # split targets to a list w.r.t. multiple levels + anchors_list = images_to_levels(all_anchors, num_level_anchors) + labels_list = images_to_levels(all_labels, num_level_anchors) + label_weights_list = images_to_levels(all_label_weights, + num_level_anchors) + bbox_targets_list = images_to_levels(all_bbox_targets, + num_level_anchors) + bbox_weights_list = images_to_levels(all_bbox_weights, + num_level_anchors) + return (anchors_list, labels_list, label_weights_list, + bbox_targets_list, bbox_weights_list, num_total_pos, + num_total_neg) + + def _get_target_single(self, + flat_anchors, + valid_flags, + num_level_anchors, + gt_bboxes, + gt_bboxes_ignore, + gt_labels, + img_meta, + label_channels=1, + unmap_outputs=True): + """Compute regression, classification targets for anchors in a single + image. + + Args: + flat_anchors (Tensor): Multi-level anchors of the image, which are + concatenated into a single tensor of shape (num_anchors, 4) + valid_flags (Tensor): Multi level valid flags of the image, + which are concatenated into a single tensor of + shape (num_anchors,). + num_level_anchors Tensor): Number of anchors of each scale level. + gt_bboxes (Tensor): Ground truth bboxes of the image, + shape (num_gts, 4). + gt_bboxes_ignore (Tensor): Ground truth bboxes to be + ignored, shape (num_ignored_gts, 4). + gt_labels (Tensor): Ground truth labels of each box, + shape (num_gts,). + img_meta (dict): Meta info of the image. + label_channels (int): Channel of label. + unmap_outputs (bool): Whether to map outputs back to the original + set of anchors. + + Returns: + tuple: N is the number of total anchors in the image. + anchors (Tensor): All anchors in the image with shape (N, 4). + labels (Tensor): Labels of all anchors in the image with shape + (N,). + label_weights (Tensor): Label weights of all anchor in the + image with shape (N,). + bbox_targets (Tensor): BBox targets of all anchors in the + image with shape (N, 4). + bbox_weights (Tensor): BBox weights of all anchors in the + image with shape (N, 4). + pos_inds (Tensor): Indices of positive anchor with shape + (num_pos,). + neg_inds (Tensor): Indices of negative anchor with shape + (num_neg,). + """ + inside_flags = anchor_inside_flags(flat_anchors, valid_flags, + img_meta['img_shape'][:2], + self.train_cfg.allowed_border) + if not inside_flags.any(): + return (None, ) * 7 + # assign gt and sample anchors + anchors = flat_anchors[inside_flags, :] + + num_level_anchors_inside = self.get_num_level_anchors_inside( + num_level_anchors, inside_flags) + assign_result = self.assigner.assign(anchors, num_level_anchors_inside, + gt_bboxes, gt_bboxes_ignore, + gt_labels) + + sampling_result = self.sampler.sample(assign_result, anchors, + gt_bboxes) + + num_valid_anchors = anchors.shape[0] + bbox_targets = torch.zeros_like(anchors) + bbox_weights = torch.zeros_like(anchors) + labels = anchors.new_full((num_valid_anchors, ), + self.num_classes, + dtype=torch.long) + label_weights = anchors.new_zeros(num_valid_anchors, dtype=torch.float) + + pos_inds = sampling_result.pos_inds + neg_inds = sampling_result.neg_inds + if len(pos_inds) > 0: + pos_bbox_targets = sampling_result.pos_gt_bboxes + bbox_targets[pos_inds, :] = pos_bbox_targets + bbox_weights[pos_inds, :] = 1.0 + if gt_labels is None: + # Only rpn gives gt_labels as None + # Foreground is the first class + labels[pos_inds] = 0 + else: + labels[pos_inds] = gt_labels[ + sampling_result.pos_assigned_gt_inds] + if self.train_cfg.pos_weight <= 0: + label_weights[pos_inds] = 1.0 + else: + label_weights[pos_inds] = self.train_cfg.pos_weight + if len(neg_inds) > 0: + label_weights[neg_inds] = 1.0 + + # map up to original set of anchors + if unmap_outputs: + num_total_anchors = flat_anchors.size(0) + anchors = unmap(anchors, num_total_anchors, inside_flags) + labels = unmap( + labels, num_total_anchors, inside_flags, fill=self.num_classes) + label_weights = unmap(label_weights, num_total_anchors, + inside_flags) + bbox_targets = unmap(bbox_targets, num_total_anchors, inside_flags) + bbox_weights = unmap(bbox_weights, num_total_anchors, inside_flags) + + return (anchors, labels, label_weights, bbox_targets, bbox_weights, + pos_inds, neg_inds) + + def get_num_level_anchors_inside(self, num_level_anchors, inside_flags): + split_inside_flags = torch.split(inside_flags, num_level_anchors) + num_level_anchors_inside = [ + int(flags.sum()) for flags in split_inside_flags + ] + return num_level_anchors_inside diff --git a/detection/mmdet/models/dense_heads/guided_anchor_head.py b/detection/mmdet/models/dense_heads/guided_anchor_head.py new file mode 100644 index 0000000..997ebb7 --- /dev/null +++ b/detection/mmdet/models/dense_heads/guided_anchor_head.py @@ -0,0 +1,860 @@ +import torch +import torch.nn as nn +from mmcv.cnn import bias_init_with_prob, normal_init +from mmcv.ops import DeformConv2d, MaskedConv2d +from mmcv.runner import force_fp32 + +from mmdet.core import (anchor_inside_flags, build_anchor_generator, + build_assigner, build_bbox_coder, build_sampler, + calc_region, images_to_levels, multi_apply, + multiclass_nms, unmap) +from ..builder import HEADS, build_loss +from .anchor_head import AnchorHead + + +class FeatureAdaption(nn.Module): + """Feature Adaption Module. + + Feature Adaption Module is implemented based on DCN v1. + It uses anchor shape prediction rather than feature map to + predict offsets of deform conv layer. + + Args: + in_channels (int): Number of channels in the input feature map. + out_channels (int): Number of channels in the output feature map. + kernel_size (int): Deformable conv kernel size. + deform_groups (int): Deformable conv group size. + """ + + def __init__(self, + in_channels, + out_channels, + kernel_size=3, + deform_groups=4): + super(FeatureAdaption, self).__init__() + offset_channels = kernel_size * kernel_size * 2 + self.conv_offset = nn.Conv2d( + 2, deform_groups * offset_channels, 1, bias=False) + self.conv_adaption = DeformConv2d( + in_channels, + out_channels, + kernel_size=kernel_size, + padding=(kernel_size - 1) // 2, + deform_groups=deform_groups) + self.relu = nn.ReLU(inplace=True) + + def init_weights(self): + normal_init(self.conv_offset, std=0.1) + normal_init(self.conv_adaption, std=0.01) + + def forward(self, x, shape): + offset = self.conv_offset(shape.detach()) + x = self.relu(self.conv_adaption(x, offset)) + return x + + +@HEADS.register_module() +class GuidedAnchorHead(AnchorHead): + """Guided-Anchor-based head (GA-RPN, GA-RetinaNet, etc.). + + This GuidedAnchorHead will predict high-quality feature guided + anchors and locations where anchors will be kept in inference. + There are mainly 3 categories of bounding-boxes. + + - Sampled 9 pairs for target assignment. (approxes) + - The square boxes where the predicted anchors are based on. (squares) + - Guided anchors. + + Please refer to https://arxiv.org/abs/1901.03278 for more details. + + Args: + num_classes (int): Number of classes. + in_channels (int): Number of channels in the input feature map. + feat_channels (int): Number of hidden channels. + approx_anchor_generator (dict): Config dict for approx generator + square_anchor_generator (dict): Config dict for square generator + anchor_coder (dict): Config dict for anchor coder + bbox_coder (dict): Config dict for bbox coder + reg_decoded_bbox (bool): If true, the regression loss would be + applied directly on decoded bounding boxes, converting both + the predicted boxes and regression targets to absolute + coordinates format. Default False. It should be `True` when + using `IoULoss`, `GIoULoss`, or `DIoULoss` in the bbox head. + deform_groups: (int): Group number of DCN in + FeatureAdaption module. + loc_filter_thr (float): Threshold to filter out unconcerned regions. + loss_loc (dict): Config of location loss. + loss_shape (dict): Config of anchor shape loss. + loss_cls (dict): Config of classification loss. + loss_bbox (dict): Config of bbox regression loss. + """ + + def __init__( + self, + num_classes, + in_channels, + feat_channels=256, + approx_anchor_generator=dict( + type='AnchorGenerator', + octave_base_scale=8, + scales_per_octave=3, + ratios=[0.5, 1.0, 2.0], + strides=[4, 8, 16, 32, 64]), + square_anchor_generator=dict( + type='AnchorGenerator', + ratios=[1.0], + scales=[8], + strides=[4, 8, 16, 32, 64]), + anchor_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[.0, .0, .0, .0], + target_stds=[1.0, 1.0, 1.0, 1.0] + ), + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[.0, .0, .0, .0], + target_stds=[1.0, 1.0, 1.0, 1.0] + ), + reg_decoded_bbox=False, + deform_groups=4, + loc_filter_thr=0.01, + train_cfg=None, + test_cfg=None, + loss_loc=dict( + type='FocalLoss', + use_sigmoid=True, + gamma=2.0, + alpha=0.25, + loss_weight=1.0), + loss_shape=dict(type='BoundedIoULoss', beta=0.2, loss_weight=1.0), + loss_cls=dict( + type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), + loss_bbox=dict(type='SmoothL1Loss', beta=1.0, + loss_weight=1.0)): # yapf: disable + super(AnchorHead, self).__init__() + self.in_channels = in_channels + self.num_classes = num_classes + self.feat_channels = feat_channels + self.deform_groups = deform_groups + self.loc_filter_thr = loc_filter_thr + + # build approx_anchor_generator and square_anchor_generator + assert (approx_anchor_generator['octave_base_scale'] == + square_anchor_generator['scales'][0]) + assert (approx_anchor_generator['strides'] == + square_anchor_generator['strides']) + self.approx_anchor_generator = build_anchor_generator( + approx_anchor_generator) + self.square_anchor_generator = build_anchor_generator( + square_anchor_generator) + self.approxs_per_octave = self.approx_anchor_generator \ + .num_base_anchors[0] + + self.reg_decoded_bbox = reg_decoded_bbox + + # one anchor per location + self.num_anchors = 1 + self.use_sigmoid_cls = loss_cls.get('use_sigmoid', False) + self.loc_focal_loss = loss_loc['type'] in ['FocalLoss'] + self.sampling = loss_cls['type'] not in ['FocalLoss'] + self.ga_sampling = train_cfg is not None and hasattr( + train_cfg, 'ga_sampler') + if self.use_sigmoid_cls: + self.cls_out_channels = self.num_classes + else: + self.cls_out_channels = self.num_classes + 1 + + # build bbox_coder + self.anchor_coder = build_bbox_coder(anchor_coder) + self.bbox_coder = build_bbox_coder(bbox_coder) + + # build losses + self.loss_loc = build_loss(loss_loc) + self.loss_shape = build_loss(loss_shape) + self.loss_cls = build_loss(loss_cls) + self.loss_bbox = build_loss(loss_bbox) + + self.train_cfg = train_cfg + self.test_cfg = test_cfg + + if self.train_cfg: + self.assigner = build_assigner(self.train_cfg.assigner) + # use PseudoSampler when sampling is False + if self.sampling and hasattr(self.train_cfg, 'sampler'): + sampler_cfg = self.train_cfg.sampler + else: + sampler_cfg = dict(type='PseudoSampler') + self.sampler = build_sampler(sampler_cfg, context=self) + + self.ga_assigner = build_assigner(self.train_cfg.ga_assigner) + if self.ga_sampling: + ga_sampler_cfg = self.train_cfg.ga_sampler + else: + ga_sampler_cfg = dict(type='PseudoSampler') + self.ga_sampler = build_sampler(ga_sampler_cfg, context=self) + + self.fp16_enabled = False + + self._init_layers() + + def _init_layers(self): + self.relu = nn.ReLU(inplace=True) + self.conv_loc = nn.Conv2d(self.in_channels, 1, 1) + self.conv_shape = nn.Conv2d(self.in_channels, self.num_anchors * 2, 1) + self.feature_adaption = FeatureAdaption( + self.in_channels, + self.feat_channels, + kernel_size=3, + deform_groups=self.deform_groups) + self.conv_cls = MaskedConv2d(self.feat_channels, + self.num_anchors * self.cls_out_channels, + 1) + self.conv_reg = MaskedConv2d(self.feat_channels, self.num_anchors * 4, + 1) + + def init_weights(self): + normal_init(self.conv_cls, std=0.01) + normal_init(self.conv_reg, std=0.01) + + bias_cls = bias_init_with_prob(0.01) + normal_init(self.conv_loc, std=0.01, bias=bias_cls) + normal_init(self.conv_shape, std=0.01) + + self.feature_adaption.init_weights() + + def forward_single(self, x): + loc_pred = self.conv_loc(x) + shape_pred = self.conv_shape(x) + x = self.feature_adaption(x, shape_pred) + # masked conv is only used during inference for speed-up + if not self.training: + mask = loc_pred.sigmoid()[0] >= self.loc_filter_thr + else: + mask = None + cls_score = self.conv_cls(x, mask) + bbox_pred = self.conv_reg(x, mask) + return cls_score, bbox_pred, shape_pred, loc_pred + + def forward(self, feats): + return multi_apply(self.forward_single, feats) + + def get_sampled_approxs(self, featmap_sizes, img_metas, device='cuda'): + """Get sampled approxs and inside flags according to feature map sizes. + + Args: + featmap_sizes (list[tuple]): Multi-level feature map sizes. + img_metas (list[dict]): Image meta info. + device (torch.device | str): device for returned tensors + + Returns: + tuple: approxes of each image, inside flags of each image + """ + num_imgs = len(img_metas) + + # since feature map sizes of all images are the same, we only compute + # approxes for one time + multi_level_approxs = self.approx_anchor_generator.grid_anchors( + featmap_sizes, device=device) + approxs_list = [multi_level_approxs for _ in range(num_imgs)] + + # for each image, we compute inside flags of multi level approxes + inside_flag_list = [] + for img_id, img_meta in enumerate(img_metas): + multi_level_flags = [] + multi_level_approxs = approxs_list[img_id] + + # obtain valid flags for each approx first + multi_level_approx_flags = self.approx_anchor_generator \ + .valid_flags(featmap_sizes, + img_meta['pad_shape'], + device=device) + + for i, flags in enumerate(multi_level_approx_flags): + approxs = multi_level_approxs[i] + inside_flags_list = [] + for i in range(self.approxs_per_octave): + split_valid_flags = flags[i::self.approxs_per_octave] + split_approxs = approxs[i::self.approxs_per_octave, :] + inside_flags = anchor_inside_flags( + split_approxs, split_valid_flags, + img_meta['img_shape'][:2], + self.train_cfg.allowed_border) + inside_flags_list.append(inside_flags) + # inside_flag for a position is true if any anchor in this + # position is true + inside_flags = ( + torch.stack(inside_flags_list, 0).sum(dim=0) > 0) + multi_level_flags.append(inside_flags) + inside_flag_list.append(multi_level_flags) + return approxs_list, inside_flag_list + + def get_anchors(self, + featmap_sizes, + shape_preds, + loc_preds, + img_metas, + use_loc_filter=False, + device='cuda'): + """Get squares according to feature map sizes and guided anchors. + + Args: + featmap_sizes (list[tuple]): Multi-level feature map sizes. + shape_preds (list[tensor]): Multi-level shape predictions. + loc_preds (list[tensor]): Multi-level location predictions. + img_metas (list[dict]): Image meta info. + use_loc_filter (bool): Use loc filter or not. + device (torch.device | str): device for returned tensors + + Returns: + tuple: square approxs of each image, guided anchors of each image, + loc masks of each image + """ + num_imgs = len(img_metas) + num_levels = len(featmap_sizes) + + # since feature map sizes of all images are the same, we only compute + # squares for one time + multi_level_squares = self.square_anchor_generator.grid_anchors( + featmap_sizes, device=device) + squares_list = [multi_level_squares for _ in range(num_imgs)] + + # for each image, we compute multi level guided anchors + guided_anchors_list = [] + loc_mask_list = [] + for img_id, img_meta in enumerate(img_metas): + multi_level_guided_anchors = [] + multi_level_loc_mask = [] + for i in range(num_levels): + squares = squares_list[img_id][i] + shape_pred = shape_preds[i][img_id] + loc_pred = loc_preds[i][img_id] + guided_anchors, loc_mask = self._get_guided_anchors_single( + squares, + shape_pred, + loc_pred, + use_loc_filter=use_loc_filter) + multi_level_guided_anchors.append(guided_anchors) + multi_level_loc_mask.append(loc_mask) + guided_anchors_list.append(multi_level_guided_anchors) + loc_mask_list.append(multi_level_loc_mask) + return squares_list, guided_anchors_list, loc_mask_list + + def _get_guided_anchors_single(self, + squares, + shape_pred, + loc_pred, + use_loc_filter=False): + """Get guided anchors and loc masks for a single level. + + Args: + square (tensor): Squares of a single level. + shape_pred (tensor): Shape predections of a single level. + loc_pred (tensor): Loc predections of a single level. + use_loc_filter (list[tensor]): Use loc filter or not. + + Returns: + tuple: guided anchors, location masks + """ + # calculate location filtering mask + loc_pred = loc_pred.sigmoid().detach() + if use_loc_filter: + loc_mask = loc_pred >= self.loc_filter_thr + else: + loc_mask = loc_pred >= 0.0 + mask = loc_mask.permute(1, 2, 0).expand(-1, -1, self.num_anchors) + mask = mask.contiguous().view(-1) + # calculate guided anchors + squares = squares[mask] + anchor_deltas = shape_pred.permute(1, 2, 0).contiguous().view( + -1, 2).detach()[mask] + bbox_deltas = anchor_deltas.new_full(squares.size(), 0) + bbox_deltas[:, 2:] = anchor_deltas + guided_anchors = self.anchor_coder.decode( + squares, bbox_deltas, wh_ratio_clip=1e-6) + return guided_anchors, mask + + def ga_loc_targets(self, gt_bboxes_list, featmap_sizes): + """Compute location targets for guided anchoring. + + Each feature map is divided into positive, negative and ignore regions. + - positive regions: target 1, weight 1 + - ignore regions: target 0, weight 0 + - negative regions: target 0, weight 0.1 + + Args: + gt_bboxes_list (list[Tensor]): Gt bboxes of each image. + featmap_sizes (list[tuple]): Multi level sizes of each feature + maps. + + Returns: + tuple + """ + anchor_scale = self.approx_anchor_generator.octave_base_scale + anchor_strides = self.approx_anchor_generator.strides + # Currently only supports same stride in x and y direction. + for stride in anchor_strides: + assert (stride[0] == stride[1]) + anchor_strides = [stride[0] for stride in anchor_strides] + + center_ratio = self.train_cfg.center_ratio + ignore_ratio = self.train_cfg.ignore_ratio + img_per_gpu = len(gt_bboxes_list) + num_lvls = len(featmap_sizes) + r1 = (1 - center_ratio) / 2 + r2 = (1 - ignore_ratio) / 2 + all_loc_targets = [] + all_loc_weights = [] + all_ignore_map = [] + for lvl_id in range(num_lvls): + h, w = featmap_sizes[lvl_id] + loc_targets = torch.zeros( + img_per_gpu, + 1, + h, + w, + device=gt_bboxes_list[0].device, + dtype=torch.float32) + loc_weights = torch.full_like(loc_targets, -1) + ignore_map = torch.zeros_like(loc_targets) + all_loc_targets.append(loc_targets) + all_loc_weights.append(loc_weights) + all_ignore_map.append(ignore_map) + for img_id in range(img_per_gpu): + gt_bboxes = gt_bboxes_list[img_id] + scale = torch.sqrt((gt_bboxes[:, 2] - gt_bboxes[:, 0]) * + (gt_bboxes[:, 3] - gt_bboxes[:, 1])) + min_anchor_size = scale.new_full( + (1, ), float(anchor_scale * anchor_strides[0])) + # assign gt bboxes to different feature levels w.r.t. their scales + target_lvls = torch.floor( + torch.log2(scale) - torch.log2(min_anchor_size) + 0.5) + target_lvls = target_lvls.clamp(min=0, max=num_lvls - 1).long() + for gt_id in range(gt_bboxes.size(0)): + lvl = target_lvls[gt_id].item() + # rescaled to corresponding feature map + gt_ = gt_bboxes[gt_id, :4] / anchor_strides[lvl] + # calculate ignore regions + ignore_x1, ignore_y1, ignore_x2, ignore_y2 = calc_region( + gt_, r2, featmap_sizes[lvl]) + # calculate positive (center) regions + ctr_x1, ctr_y1, ctr_x2, ctr_y2 = calc_region( + gt_, r1, featmap_sizes[lvl]) + all_loc_targets[lvl][img_id, 0, ctr_y1:ctr_y2 + 1, + ctr_x1:ctr_x2 + 1] = 1 + all_loc_weights[lvl][img_id, 0, ignore_y1:ignore_y2 + 1, + ignore_x1:ignore_x2 + 1] = 0 + all_loc_weights[lvl][img_id, 0, ctr_y1:ctr_y2 + 1, + ctr_x1:ctr_x2 + 1] = 1 + # calculate ignore map on nearby low level feature + if lvl > 0: + d_lvl = lvl - 1 + # rescaled to corresponding feature map + gt_ = gt_bboxes[gt_id, :4] / anchor_strides[d_lvl] + ignore_x1, ignore_y1, ignore_x2, ignore_y2 = calc_region( + gt_, r2, featmap_sizes[d_lvl]) + all_ignore_map[d_lvl][img_id, 0, ignore_y1:ignore_y2 + 1, + ignore_x1:ignore_x2 + 1] = 1 + # calculate ignore map on nearby high level feature + if lvl < num_lvls - 1: + u_lvl = lvl + 1 + # rescaled to corresponding feature map + gt_ = gt_bboxes[gt_id, :4] / anchor_strides[u_lvl] + ignore_x1, ignore_y1, ignore_x2, ignore_y2 = calc_region( + gt_, r2, featmap_sizes[u_lvl]) + all_ignore_map[u_lvl][img_id, 0, ignore_y1:ignore_y2 + 1, + ignore_x1:ignore_x2 + 1] = 1 + for lvl_id in range(num_lvls): + # ignore negative regions w.r.t. ignore map + all_loc_weights[lvl_id][(all_loc_weights[lvl_id] < 0) + & (all_ignore_map[lvl_id] > 0)] = 0 + # set negative regions with weight 0.1 + all_loc_weights[lvl_id][all_loc_weights[lvl_id] < 0] = 0.1 + # loc average factor to balance loss + loc_avg_factor = sum( + [t.size(0) * t.size(-1) * t.size(-2) + for t in all_loc_targets]) / 200 + return all_loc_targets, all_loc_weights, loc_avg_factor + + def _ga_shape_target_single(self, + flat_approxs, + inside_flags, + flat_squares, + gt_bboxes, + gt_bboxes_ignore, + img_meta, + unmap_outputs=True): + """Compute guided anchoring targets. + + This function returns sampled anchors and gt bboxes directly + rather than calculates regression targets. + + Args: + flat_approxs (Tensor): flat approxs of a single image, + shape (n, 4) + inside_flags (Tensor): inside flags of a single image, + shape (n, ). + flat_squares (Tensor): flat squares of a single image, + shape (approxs_per_octave * n, 4) + gt_bboxes (Tensor): Ground truth bboxes of a single image. + img_meta (dict): Meta info of a single image. + approxs_per_octave (int): number of approxs per octave + cfg (dict): RPN train configs. + unmap_outputs (bool): unmap outputs or not. + + Returns: + tuple + """ + if not inside_flags.any(): + return (None, ) * 5 + # assign gt and sample anchors + expand_inside_flags = inside_flags[:, None].expand( + -1, self.approxs_per_octave).reshape(-1) + approxs = flat_approxs[expand_inside_flags, :] + squares = flat_squares[inside_flags, :] + + assign_result = self.ga_assigner.assign(approxs, squares, + self.approxs_per_octave, + gt_bboxes, gt_bboxes_ignore) + sampling_result = self.ga_sampler.sample(assign_result, squares, + gt_bboxes) + + bbox_anchors = torch.zeros_like(squares) + bbox_gts = torch.zeros_like(squares) + bbox_weights = torch.zeros_like(squares) + + pos_inds = sampling_result.pos_inds + neg_inds = sampling_result.neg_inds + if len(pos_inds) > 0: + bbox_anchors[pos_inds, :] = sampling_result.pos_bboxes + bbox_gts[pos_inds, :] = sampling_result.pos_gt_bboxes + bbox_weights[pos_inds, :] = 1.0 + + # map up to original set of anchors + if unmap_outputs: + num_total_anchors = flat_squares.size(0) + bbox_anchors = unmap(bbox_anchors, num_total_anchors, inside_flags) + bbox_gts = unmap(bbox_gts, num_total_anchors, inside_flags) + bbox_weights = unmap(bbox_weights, num_total_anchors, inside_flags) + + return (bbox_anchors, bbox_gts, bbox_weights, pos_inds, neg_inds) + + def ga_shape_targets(self, + approx_list, + inside_flag_list, + square_list, + gt_bboxes_list, + img_metas, + gt_bboxes_ignore_list=None, + unmap_outputs=True): + """Compute guided anchoring targets. + + Args: + approx_list (list[list]): Multi level approxs of each image. + inside_flag_list (list[list]): Multi level inside flags of each + image. + square_list (list[list]): Multi level squares of each image. + gt_bboxes_list (list[Tensor]): Ground truth bboxes of each image. + img_metas (list[dict]): Meta info of each image. + gt_bboxes_ignore_list (list[Tensor]): ignore list of gt bboxes. + unmap_outputs (bool): unmap outputs or not. + + Returns: + tuple + """ + num_imgs = len(img_metas) + assert len(approx_list) == len(inside_flag_list) == len( + square_list) == num_imgs + # anchor number of multi levels + num_level_squares = [squares.size(0) for squares in square_list[0]] + # concat all level anchors and flags to a single tensor + inside_flag_flat_list = [] + approx_flat_list = [] + square_flat_list = [] + for i in range(num_imgs): + assert len(square_list[i]) == len(inside_flag_list[i]) + inside_flag_flat_list.append(torch.cat(inside_flag_list[i])) + approx_flat_list.append(torch.cat(approx_list[i])) + square_flat_list.append(torch.cat(square_list[i])) + + # compute targets for each image + if gt_bboxes_ignore_list is None: + gt_bboxes_ignore_list = [None for _ in range(num_imgs)] + (all_bbox_anchors, all_bbox_gts, all_bbox_weights, pos_inds_list, + neg_inds_list) = multi_apply( + self._ga_shape_target_single, + approx_flat_list, + inside_flag_flat_list, + square_flat_list, + gt_bboxes_list, + gt_bboxes_ignore_list, + img_metas, + unmap_outputs=unmap_outputs) + # no valid anchors + if any([bbox_anchors is None for bbox_anchors in all_bbox_anchors]): + return None + # sampled anchors of all images + num_total_pos = sum([max(inds.numel(), 1) for inds in pos_inds_list]) + num_total_neg = sum([max(inds.numel(), 1) for inds in neg_inds_list]) + # split targets to a list w.r.t. multiple levels + bbox_anchors_list = images_to_levels(all_bbox_anchors, + num_level_squares) + bbox_gts_list = images_to_levels(all_bbox_gts, num_level_squares) + bbox_weights_list = images_to_levels(all_bbox_weights, + num_level_squares) + return (bbox_anchors_list, bbox_gts_list, bbox_weights_list, + num_total_pos, num_total_neg) + + def loss_shape_single(self, shape_pred, bbox_anchors, bbox_gts, + anchor_weights, anchor_total_num): + shape_pred = shape_pred.permute(0, 2, 3, 1).contiguous().view(-1, 2) + bbox_anchors = bbox_anchors.contiguous().view(-1, 4) + bbox_gts = bbox_gts.contiguous().view(-1, 4) + anchor_weights = anchor_weights.contiguous().view(-1, 4) + bbox_deltas = bbox_anchors.new_full(bbox_anchors.size(), 0) + bbox_deltas[:, 2:] += shape_pred + # filter out negative samples to speed-up weighted_bounded_iou_loss + inds = torch.nonzero( + anchor_weights[:, 0] > 0, as_tuple=False).squeeze(1) + bbox_deltas_ = bbox_deltas[inds] + bbox_anchors_ = bbox_anchors[inds] + bbox_gts_ = bbox_gts[inds] + anchor_weights_ = anchor_weights[inds] + pred_anchors_ = self.anchor_coder.decode( + bbox_anchors_, bbox_deltas_, wh_ratio_clip=1e-6) + loss_shape = self.loss_shape( + pred_anchors_, + bbox_gts_, + anchor_weights_, + avg_factor=anchor_total_num) + return loss_shape + + def loss_loc_single(self, loc_pred, loc_target, loc_weight, + loc_avg_factor): + loss_loc = self.loss_loc( + loc_pred.reshape(-1, 1), + loc_target.reshape(-1).long(), + loc_weight.reshape(-1), + avg_factor=loc_avg_factor) + return loss_loc + + @force_fp32( + apply_to=('cls_scores', 'bbox_preds', 'shape_preds', 'loc_preds')) + def loss(self, + cls_scores, + bbox_preds, + shape_preds, + loc_preds, + gt_bboxes, + gt_labels, + img_metas, + gt_bboxes_ignore=None): + featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] + assert len(featmap_sizes) == self.approx_anchor_generator.num_levels + + device = cls_scores[0].device + + # get loc targets + loc_targets, loc_weights, loc_avg_factor = self.ga_loc_targets( + gt_bboxes, featmap_sizes) + + # get sampled approxes + approxs_list, inside_flag_list = self.get_sampled_approxs( + featmap_sizes, img_metas, device=device) + # get squares and guided anchors + squares_list, guided_anchors_list, _ = self.get_anchors( + featmap_sizes, shape_preds, loc_preds, img_metas, device=device) + + # get shape targets + shape_targets = self.ga_shape_targets(approxs_list, inside_flag_list, + squares_list, gt_bboxes, + img_metas) + if shape_targets is None: + return None + (bbox_anchors_list, bbox_gts_list, anchor_weights_list, anchor_fg_num, + anchor_bg_num) = shape_targets + anchor_total_num = ( + anchor_fg_num if not self.ga_sampling else anchor_fg_num + + anchor_bg_num) + + # get anchor targets + label_channels = self.cls_out_channels if self.use_sigmoid_cls else 1 + cls_reg_targets = self.get_targets( + guided_anchors_list, + inside_flag_list, + gt_bboxes, + img_metas, + gt_bboxes_ignore_list=gt_bboxes_ignore, + gt_labels_list=gt_labels, + label_channels=label_channels) + if cls_reg_targets is None: + return None + (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, + num_total_pos, num_total_neg) = cls_reg_targets + num_total_samples = ( + num_total_pos + num_total_neg if self.sampling else num_total_pos) + + # anchor number of multi levels + num_level_anchors = [ + anchors.size(0) for anchors in guided_anchors_list[0] + ] + # concat all level anchors to a single tensor + concat_anchor_list = [] + for i in range(len(guided_anchors_list)): + concat_anchor_list.append(torch.cat(guided_anchors_list[i])) + all_anchor_list = images_to_levels(concat_anchor_list, + num_level_anchors) + + # get classification and bbox regression losses + losses_cls, losses_bbox = multi_apply( + self.loss_single, + cls_scores, + bbox_preds, + all_anchor_list, + labels_list, + label_weights_list, + bbox_targets_list, + bbox_weights_list, + num_total_samples=num_total_samples) + + # get anchor location loss + losses_loc = [] + for i in range(len(loc_preds)): + loss_loc = self.loss_loc_single( + loc_preds[i], + loc_targets[i], + loc_weights[i], + loc_avg_factor=loc_avg_factor) + losses_loc.append(loss_loc) + + # get anchor shape loss + losses_shape = [] + for i in range(len(shape_preds)): + loss_shape = self.loss_shape_single( + shape_preds[i], + bbox_anchors_list[i], + bbox_gts_list[i], + anchor_weights_list[i], + anchor_total_num=anchor_total_num) + losses_shape.append(loss_shape) + + return dict( + loss_cls=losses_cls, + loss_bbox=losses_bbox, + loss_shape=losses_shape, + loss_loc=losses_loc) + + @force_fp32( + apply_to=('cls_scores', 'bbox_preds', 'shape_preds', 'loc_preds')) + def get_bboxes(self, + cls_scores, + bbox_preds, + shape_preds, + loc_preds, + img_metas, + cfg=None, + rescale=False): + assert len(cls_scores) == len(bbox_preds) == len(shape_preds) == len( + loc_preds) + num_levels = len(cls_scores) + featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] + device = cls_scores[0].device + # get guided anchors + _, guided_anchors, loc_masks = self.get_anchors( + featmap_sizes, + shape_preds, + loc_preds, + img_metas, + use_loc_filter=not self.training, + device=device) + result_list = [] + for img_id in range(len(img_metas)): + cls_score_list = [ + cls_scores[i][img_id].detach() for i in range(num_levels) + ] + bbox_pred_list = [ + bbox_preds[i][img_id].detach() for i in range(num_levels) + ] + guided_anchor_list = [ + guided_anchors[img_id][i].detach() for i in range(num_levels) + ] + loc_mask_list = [ + loc_masks[img_id][i].detach() for i in range(num_levels) + ] + img_shape = img_metas[img_id]['img_shape'] + scale_factor = img_metas[img_id]['scale_factor'] + proposals = self._get_bboxes_single(cls_score_list, bbox_pred_list, + guided_anchor_list, + loc_mask_list, img_shape, + scale_factor, cfg, rescale) + result_list.append(proposals) + return result_list + + def _get_bboxes_single(self, + cls_scores, + bbox_preds, + mlvl_anchors, + mlvl_masks, + img_shape, + scale_factor, + cfg, + rescale=False): + cfg = self.test_cfg if cfg is None else cfg + assert len(cls_scores) == len(bbox_preds) == len(mlvl_anchors) + mlvl_bboxes = [] + mlvl_scores = [] + for cls_score, bbox_pred, anchors, mask in zip(cls_scores, bbox_preds, + mlvl_anchors, + mlvl_masks): + assert cls_score.size()[-2:] == bbox_pred.size()[-2:] + # if no location is kept, end. + if mask.sum() == 0: + continue + # reshape scores and bbox_pred + cls_score = cls_score.permute(1, 2, + 0).reshape(-1, self.cls_out_channels) + if self.use_sigmoid_cls: + scores = cls_score.sigmoid() + else: + scores = cls_score.softmax(-1) + bbox_pred = bbox_pred.permute(1, 2, 0).reshape(-1, 4) + # filter scores, bbox_pred w.r.t. mask. + # anchors are filtered in get_anchors() beforehand. + scores = scores[mask, :] + bbox_pred = bbox_pred[mask, :] + if scores.dim() == 0: + anchors = anchors.unsqueeze(0) + scores = scores.unsqueeze(0) + bbox_pred = bbox_pred.unsqueeze(0) + # filter anchors, bbox_pred, scores w.r.t. scores + nms_pre = cfg.get('nms_pre', -1) + if nms_pre > 0 and scores.shape[0] > nms_pre: + if self.use_sigmoid_cls: + max_scores, _ = scores.max(dim=1) + else: + # remind that we set FG labels to [0, num_class-1] + # since mmdet v2.0 + # BG cat_id: num_class + max_scores, _ = scores[:, :-1].max(dim=1) + _, topk_inds = max_scores.topk(nms_pre) + anchors = anchors[topk_inds, :] + bbox_pred = bbox_pred[topk_inds, :] + scores = scores[topk_inds, :] + bboxes = self.bbox_coder.decode( + anchors, bbox_pred, max_shape=img_shape) + mlvl_bboxes.append(bboxes) + mlvl_scores.append(scores) + mlvl_bboxes = torch.cat(mlvl_bboxes) + if rescale: + mlvl_bboxes /= mlvl_bboxes.new_tensor(scale_factor) + mlvl_scores = torch.cat(mlvl_scores) + if self.use_sigmoid_cls: + # Add a dummy background class to the backend when using sigmoid + # remind that we set FG labels to [0, num_class-1] since mmdet v2.0 + # BG cat_id: num_class + padding = mlvl_scores.new_zeros(mlvl_scores.shape[0], 1) + mlvl_scores = torch.cat([mlvl_scores, padding], dim=1) + # multi class NMS + det_bboxes, det_labels = multiclass_nms(mlvl_bboxes, mlvl_scores, + cfg.score_thr, cfg.nms, + cfg.max_per_img) + return det_bboxes, det_labels diff --git a/detection/mmdet/models/dense_heads/ld_head.py b/detection/mmdet/models/dense_heads/ld_head.py new file mode 100644 index 0000000..501e1f7 --- /dev/null +++ b/detection/mmdet/models/dense_heads/ld_head.py @@ -0,0 +1,261 @@ +import torch +from mmcv.runner import force_fp32 + +from mmdet.core import (bbox2distance, bbox_overlaps, distance2bbox, + multi_apply, reduce_mean) +from ..builder import HEADS, build_loss +from .gfl_head import GFLHead + + +@HEADS.register_module() +class LDHead(GFLHead): + """Localization distillation Head. (Short description) + + It utilizes the learned bbox distributions to transfer the localization + dark knowledge from teacher to student. Original paper: `Localization + Distillation for Object Detection. `_ + + Args: + num_classes (int): Number of categories excluding the background + category. + in_channels (int): Number of channels in the input feature map. + loss_ld (dict): Config of Localization Distillation Loss (LD), + T is the temperature for distillation. + """ + + def __init__(self, + num_classes, + in_channels, + loss_ld=dict( + type='LocalizationDistillationLoss', + loss_weight=0.25, + T=10), + **kwargs): + + super(LDHead, self).__init__(num_classes, in_channels, **kwargs) + self.loss_ld = build_loss(loss_ld) + + def loss_single(self, anchors, cls_score, bbox_pred, labels, label_weights, + bbox_targets, stride, soft_targets, num_total_samples): + """Compute loss of a single scale level. + + Args: + anchors (Tensor): Box reference for each scale level with shape + (N, num_total_anchors, 4). + cls_score (Tensor): Cls and quality joint scores for each scale + level has shape (N, num_classes, H, W). + bbox_pred (Tensor): Box distribution logits for each scale + level with shape (N, 4*(n+1), H, W), n is max value of integral + set. + labels (Tensor): Labels of each anchors with shape + (N, num_total_anchors). + label_weights (Tensor): Label weights of each anchor with shape + (N, num_total_anchors) + bbox_targets (Tensor): BBox regression targets of each anchor wight + shape (N, num_total_anchors, 4). + stride (tuple): Stride in this scale level. + num_total_samples (int): Number of positive samples that is + reduced over all GPUs. + + Returns: + dict[tuple, Tensor]: Loss components and weight targets. + """ + assert stride[0] == stride[1], 'h stride is not equal to w stride!' + anchors = anchors.reshape(-1, 4) + cls_score = cls_score.permute(0, 2, 3, + 1).reshape(-1, self.cls_out_channels) + bbox_pred = bbox_pred.permute(0, 2, 3, + 1).reshape(-1, 4 * (self.reg_max + 1)) + soft_targets = soft_targets.permute(0, 2, 3, + 1).reshape(-1, + 4 * (self.reg_max + 1)) + + bbox_targets = bbox_targets.reshape(-1, 4) + labels = labels.reshape(-1) + label_weights = label_weights.reshape(-1) + + # FG cat_id: [0, num_classes -1], BG cat_id: num_classes + bg_class_ind = self.num_classes + pos_inds = ((labels >= 0) + & (labels < bg_class_ind)).nonzero().squeeze(1) + score = label_weights.new_zeros(labels.shape) + + if len(pos_inds) > 0: + pos_bbox_targets = bbox_targets[pos_inds] + pos_bbox_pred = bbox_pred[pos_inds] + pos_anchors = anchors[pos_inds] + pos_anchor_centers = self.anchor_center(pos_anchors) / stride[0] + + weight_targets = cls_score.detach().sigmoid() + weight_targets = weight_targets.max(dim=1)[0][pos_inds] + pos_bbox_pred_corners = self.integral(pos_bbox_pred) + pos_decode_bbox_pred = distance2bbox(pos_anchor_centers, + pos_bbox_pred_corners) + pos_decode_bbox_targets = pos_bbox_targets / stride[0] + score[pos_inds] = bbox_overlaps( + pos_decode_bbox_pred.detach(), + pos_decode_bbox_targets, + is_aligned=True) + pred_corners = pos_bbox_pred.reshape(-1, self.reg_max + 1) + pos_soft_targets = soft_targets[pos_inds] + soft_corners = pos_soft_targets.reshape(-1, self.reg_max + 1) + + target_corners = bbox2distance(pos_anchor_centers, + pos_decode_bbox_targets, + self.reg_max).reshape(-1) + + # regression loss + loss_bbox = self.loss_bbox( + pos_decode_bbox_pred, + pos_decode_bbox_targets, + weight=weight_targets, + avg_factor=1.0) + + # dfl loss + loss_dfl = self.loss_dfl( + pred_corners, + target_corners, + weight=weight_targets[:, None].expand(-1, 4).reshape(-1), + avg_factor=4.0) + + # ld loss + loss_ld = self.loss_ld( + pred_corners, + soft_corners, + weight=weight_targets[:, None].expand(-1, 4).reshape(-1), + avg_factor=4.0) + + else: + loss_ld = bbox_pred.sum() * 0 + loss_bbox = bbox_pred.sum() * 0 + loss_dfl = bbox_pred.sum() * 0 + weight_targets = bbox_pred.new_tensor(0) + + # cls (qfl) loss + loss_cls = self.loss_cls( + cls_score, (labels, score), + weight=label_weights, + avg_factor=num_total_samples) + + return loss_cls, loss_bbox, loss_dfl, loss_ld, weight_targets.sum() + + def forward_train(self, + x, + out_teacher, + img_metas, + gt_bboxes, + gt_labels=None, + gt_bboxes_ignore=None, + proposal_cfg=None, + **kwargs): + """ + Args: + x (list[Tensor]): Features from FPN. + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + gt_bboxes (Tensor): Ground truth bboxes of the image, + shape (num_gts, 4). + gt_labels (Tensor): Ground truth labels of each box, + shape (num_gts,). + gt_bboxes_ignore (Tensor): Ground truth bboxes to be + ignored, shape (num_ignored_gts, 4). + proposal_cfg (mmcv.Config): Test / postprocessing configuration, + if None, test_cfg would be used + + Returns: + tuple[dict, list]: The loss components and proposals of each image. + + - losses (dict[str, Tensor]): A dictionary of loss components. + - proposal_list (list[Tensor]): Proposals of each image. + """ + outs = self(x) + soft_target = out_teacher[1] + if gt_labels is None: + loss_inputs = outs + (gt_bboxes, soft_target, img_metas) + else: + loss_inputs = outs + (gt_bboxes, gt_labels, soft_target, img_metas) + losses = self.loss(*loss_inputs, gt_bboxes_ignore=gt_bboxes_ignore) + if proposal_cfg is None: + return losses + else: + proposal_list = self.get_bboxes(*outs, img_metas, cfg=proposal_cfg) + return losses, proposal_list + + @force_fp32(apply_to=('cls_scores', 'bbox_preds')) + def loss(self, + cls_scores, + bbox_preds, + gt_bboxes, + gt_labels, + soft_target, + img_metas, + gt_bboxes_ignore=None): + """Compute losses of the head. + + Args: + cls_scores (list[Tensor]): Cls and quality scores for each scale + level has shape (N, num_classes, H, W). + bbox_preds (list[Tensor]): Box distribution logits for each scale + level with shape (N, 4*(n+1), H, W), n is max value of integral + set. + gt_bboxes (list[Tensor]): Ground truth bboxes for each image with + shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + gt_labels (list[Tensor]): class indices corresponding to each box + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + gt_bboxes_ignore (list[Tensor] | None): specify which bounding + boxes can be ignored when computing the loss. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + + featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] + assert len(featmap_sizes) == self.anchor_generator.num_levels + + device = cls_scores[0].device + anchor_list, valid_flag_list = self.get_anchors( + featmap_sizes, img_metas, device=device) + label_channels = self.cls_out_channels if self.use_sigmoid_cls else 1 + + cls_reg_targets = self.get_targets( + anchor_list, + valid_flag_list, + gt_bboxes, + img_metas, + gt_bboxes_ignore_list=gt_bboxes_ignore, + gt_labels_list=gt_labels, + label_channels=label_channels) + if cls_reg_targets is None: + return None + + (anchor_list, labels_list, label_weights_list, bbox_targets_list, + bbox_weights_list, num_total_pos, num_total_neg) = cls_reg_targets + + num_total_samples = reduce_mean( + torch.tensor(num_total_pos, dtype=torch.float, + device=device)).item() + num_total_samples = max(num_total_samples, 1.0) + + losses_cls, losses_bbox, losses_dfl, losses_ld, \ + avg_factor = multi_apply( + self.loss_single, + anchor_list, + cls_scores, + bbox_preds, + labels_list, + label_weights_list, + bbox_targets_list, + self.anchor_generator.strides, + soft_target, + num_total_samples=num_total_samples) + + avg_factor = sum(avg_factor) + 1e-6 + avg_factor = reduce_mean(avg_factor).item() + losses_bbox = [x / avg_factor for x in losses_bbox] + losses_dfl = [x / avg_factor for x in losses_dfl] + return dict( + loss_cls=losses_cls, + loss_bbox=losses_bbox, + loss_dfl=losses_dfl, + loss_ld=losses_ld) diff --git a/detection/mmdet/models/dense_heads/nasfcos_head.py b/detection/mmdet/models/dense_heads/nasfcos_head.py new file mode 100644 index 0000000..994ce04 --- /dev/null +++ b/detection/mmdet/models/dense_heads/nasfcos_head.py @@ -0,0 +1,75 @@ +import copy + +import torch.nn as nn +from mmcv.cnn import (ConvModule, Scale, bias_init_with_prob, + caffe2_xavier_init, normal_init) + +from mmdet.models.dense_heads.fcos_head import FCOSHead +from ..builder import HEADS + + +@HEADS.register_module() +class NASFCOSHead(FCOSHead): + """Anchor-free head used in `NASFCOS `_. + + It is quite similar with FCOS head, except for the searched structure of + classification branch and bbox regression branch, where a structure of + "dconv3x3, conv3x3, dconv3x3, conv1x1" is utilized instead. + """ + + def _init_layers(self): + """Initialize layers of the head.""" + dconv3x3_config = dict( + type='DCNv2', + kernel_size=3, + use_bias=True, + deform_groups=2, + padding=1) + conv3x3_config = dict(type='Conv', kernel_size=3, padding=1) + conv1x1_config = dict(type='Conv', kernel_size=1) + + self.arch_config = [ + dconv3x3_config, conv3x3_config, dconv3x3_config, conv1x1_config + ] + self.cls_convs = nn.ModuleList() + self.reg_convs = nn.ModuleList() + for i, op_ in enumerate(self.arch_config): + op = copy.deepcopy(op_) + chn = self.in_channels if i == 0 else self.feat_channels + assert isinstance(op, dict) + use_bias = op.pop('use_bias', False) + padding = op.pop('padding', 0) + kernel_size = op.pop('kernel_size') + module = ConvModule( + chn, + self.feat_channels, + kernel_size, + stride=1, + padding=padding, + norm_cfg=self.norm_cfg, + bias=use_bias, + conv_cfg=op) + + self.cls_convs.append(copy.deepcopy(module)) + self.reg_convs.append(copy.deepcopy(module)) + + self.conv_cls = nn.Conv2d( + self.feat_channels, self.cls_out_channels, 3, padding=1) + self.conv_reg = nn.Conv2d(self.feat_channels, 4, 3, padding=1) + self.conv_centerness = nn.Conv2d(self.feat_channels, 1, 3, padding=1) + + self.scales = nn.ModuleList([Scale(1.0) for _ in self.strides]) + + def init_weights(self): + """Initialize weights of the head.""" + # retinanet_bias_init + bias_cls = bias_init_with_prob(0.01) + normal_init(self.conv_reg, std=0.01) + normal_init(self.conv_centerness, std=0.01) + normal_init(self.conv_cls, std=0.01, bias=bias_cls) + + for branch in [self.cls_convs, self.reg_convs]: + for module in branch.modules(): + if isinstance(module, ConvModule) \ + and isinstance(module.conv, nn.Conv2d): + caffe2_xavier_init(module.conv) diff --git a/detection/mmdet/models/dense_heads/paa_head.py b/detection/mmdet/models/dense_heads/paa_head.py new file mode 100644 index 0000000..e067b01 --- /dev/null +++ b/detection/mmdet/models/dense_heads/paa_head.py @@ -0,0 +1,671 @@ +import numpy as np +import torch +from mmcv.runner import force_fp32 + +from mmdet.core import multi_apply, multiclass_nms +from mmdet.core.bbox.iou_calculators import bbox_overlaps +from mmdet.models import HEADS +from mmdet.models.dense_heads import ATSSHead + +EPS = 1e-12 +try: + import sklearn.mixture as skm +except ImportError: + skm = None + + +def levels_to_images(mlvl_tensor): + """Concat multi-level feature maps by image. + + [feature_level0, feature_level1...] -> [feature_image0, feature_image1...] + Convert the shape of each element in mlvl_tensor from (N, C, H, W) to + (N, H*W , C), then split the element to N elements with shape (H*W, C), and + concat elements in same image of all level along first dimension. + + Args: + mlvl_tensor (list[torch.Tensor]): list of Tensor which collect from + corresponding level. Each element is of shape (N, C, H, W) + + Returns: + list[torch.Tensor]: A list that contains N tensors and each tensor is + of shape (num_elements, C) + """ + batch_size = mlvl_tensor[0].size(0) + batch_list = [[] for _ in range(batch_size)] + channels = mlvl_tensor[0].size(1) + for t in mlvl_tensor: + t = t.permute(0, 2, 3, 1) + t = t.view(batch_size, -1, channels).contiguous() + for img in range(batch_size): + batch_list[img].append(t[img]) + return [torch.cat(item, 0) for item in batch_list] + + +@HEADS.register_module() +class PAAHead(ATSSHead): + """Head of PAAAssignment: Probabilistic Anchor Assignment with IoU + Prediction for Object Detection. + + Code is modified from the `official github repo + `_. + + More details can be found in the `paper + `_ . + + Args: + topk (int): Select topk samples with smallest loss in + each level. + score_voting (bool): Whether to use score voting in post-process. + covariance_type : String describing the type of covariance parameters + to be used in :class:`sklearn.mixture.GaussianMixture`. + It must be one of: + + - 'full': each component has its own general covariance matrix + - 'tied': all components share the same general covariance matrix + - 'diag': each component has its own diagonal covariance matrix + - 'spherical': each component has its own single variance + Default: 'diag'. From 'full' to 'spherical', the gmm fitting + process is faster yet the performance could be influenced. For most + cases, 'diag' should be a good choice. + """ + + def __init__(self, + *args, + topk=9, + score_voting=True, + covariance_type='diag', + **kwargs): + # topk used in paa reassign process + self.topk = topk + self.with_score_voting = score_voting + self.covariance_type = covariance_type + super(PAAHead, self).__init__(*args, **kwargs) + + @force_fp32(apply_to=('cls_scores', 'bbox_preds', 'iou_preds')) + def loss(self, + cls_scores, + bbox_preds, + iou_preds, + gt_bboxes, + gt_labels, + img_metas, + gt_bboxes_ignore=None): + """Compute losses of the head. + + Args: + cls_scores (list[Tensor]): Box scores for each scale level + Has shape (N, num_anchors * num_classes, H, W) + bbox_preds (list[Tensor]): Box energies / deltas for each scale + level with shape (N, num_anchors * 4, H, W) + iou_preds (list[Tensor]): iou_preds for each scale + level with shape (N, num_anchors * 1, H, W) + gt_bboxes (list[Tensor]): Ground truth bboxes for each image with + shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + gt_labels (list[Tensor]): class indices corresponding to each box + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + gt_bboxes_ignore (list[Tensor] | None): Specify which bounding + boxes can be ignored when are computing the loss. + + Returns: + dict[str, Tensor]: A dictionary of loss gmm_assignment. + """ + + featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] + assert len(featmap_sizes) == self.anchor_generator.num_levels + + device = cls_scores[0].device + anchor_list, valid_flag_list = self.get_anchors( + featmap_sizes, img_metas, device=device) + label_channels = self.cls_out_channels if self.use_sigmoid_cls else 1 + cls_reg_targets = self.get_targets( + anchor_list, + valid_flag_list, + gt_bboxes, + img_metas, + gt_bboxes_ignore_list=gt_bboxes_ignore, + gt_labels_list=gt_labels, + label_channels=label_channels, + ) + (labels, labels_weight, bboxes_target, bboxes_weight, pos_inds, + pos_gt_index) = cls_reg_targets + cls_scores = levels_to_images(cls_scores) + cls_scores = [ + item.reshape(-1, self.cls_out_channels) for item in cls_scores + ] + bbox_preds = levels_to_images(bbox_preds) + bbox_preds = [item.reshape(-1, 4) for item in bbox_preds] + iou_preds = levels_to_images(iou_preds) + iou_preds = [item.reshape(-1, 1) for item in iou_preds] + pos_losses_list, = multi_apply(self.get_pos_loss, anchor_list, + cls_scores, bbox_preds, labels, + labels_weight, bboxes_target, + bboxes_weight, pos_inds) + + with torch.no_grad(): + reassign_labels, reassign_label_weight, \ + reassign_bbox_weights, num_pos = multi_apply( + self.paa_reassign, + pos_losses_list, + labels, + labels_weight, + bboxes_weight, + pos_inds, + pos_gt_index, + anchor_list) + num_pos = sum(num_pos) + # convert all tensor list to a flatten tensor + cls_scores = torch.cat(cls_scores, 0).view(-1, cls_scores[0].size(-1)) + bbox_preds = torch.cat(bbox_preds, 0).view(-1, bbox_preds[0].size(-1)) + iou_preds = torch.cat(iou_preds, 0).view(-1, iou_preds[0].size(-1)) + labels = torch.cat(reassign_labels, 0).view(-1) + flatten_anchors = torch.cat( + [torch.cat(item, 0) for item in anchor_list]) + labels_weight = torch.cat(reassign_label_weight, 0).view(-1) + bboxes_target = torch.cat(bboxes_target, + 0).view(-1, bboxes_target[0].size(-1)) + + pos_inds_flatten = ((labels >= 0) + & + (labels < self.num_classes)).nonzero().reshape(-1) + + losses_cls = self.loss_cls( + cls_scores, + labels, + labels_weight, + avg_factor=max(num_pos, len(img_metas))) # avoid num_pos=0 + if num_pos: + pos_bbox_pred = self.bbox_coder.decode( + flatten_anchors[pos_inds_flatten], + bbox_preds[pos_inds_flatten]) + pos_bbox_target = bboxes_target[pos_inds_flatten] + iou_target = bbox_overlaps( + pos_bbox_pred.detach(), pos_bbox_target, is_aligned=True) + losses_iou = self.loss_centerness( + iou_preds[pos_inds_flatten], + iou_target.unsqueeze(-1), + avg_factor=num_pos) + losses_bbox = self.loss_bbox( + pos_bbox_pred, + pos_bbox_target, + iou_target.clamp(min=EPS), + avg_factor=iou_target.sum()) + else: + losses_iou = iou_preds.sum() * 0 + losses_bbox = bbox_preds.sum() * 0 + + return dict( + loss_cls=losses_cls, loss_bbox=losses_bbox, loss_iou=losses_iou) + + def get_pos_loss(self, anchors, cls_score, bbox_pred, label, label_weight, + bbox_target, bbox_weight, pos_inds): + """Calculate loss of all potential positive samples obtained from first + match process. + + Args: + anchors (list[Tensor]): Anchors of each scale. + cls_score (Tensor): Box scores of single image with shape + (num_anchors, num_classes) + bbox_pred (Tensor): Box energies / deltas of single image + with shape (num_anchors, 4) + label (Tensor): classification target of each anchor with + shape (num_anchors,) + label_weight (Tensor): Classification loss weight of each + anchor with shape (num_anchors). + bbox_target (dict): Regression target of each anchor with + shape (num_anchors, 4). + bbox_weight (Tensor): Bbox weight of each anchor with shape + (num_anchors, 4). + pos_inds (Tensor): Index of all positive samples got from + first assign process. + + Returns: + Tensor: Losses of all positive samples in single image. + """ + if not len(pos_inds): + return cls_score.new([]), + anchors_all_level = torch.cat(anchors, 0) + pos_scores = cls_score[pos_inds] + pos_bbox_pred = bbox_pred[pos_inds] + pos_label = label[pos_inds] + pos_label_weight = label_weight[pos_inds] + pos_bbox_target = bbox_target[pos_inds] + pos_bbox_weight = bbox_weight[pos_inds] + pos_anchors = anchors_all_level[pos_inds] + pos_bbox_pred = self.bbox_coder.decode(pos_anchors, pos_bbox_pred) + + # to keep loss dimension + loss_cls = self.loss_cls( + pos_scores, + pos_label, + pos_label_weight, + avg_factor=self.loss_cls.loss_weight, + reduction_override='none') + + loss_bbox = self.loss_bbox( + pos_bbox_pred, + pos_bbox_target, + pos_bbox_weight, + avg_factor=self.loss_cls.loss_weight, + reduction_override='none') + + loss_cls = loss_cls.sum(-1) + pos_loss = loss_bbox + loss_cls + return pos_loss, + + def paa_reassign(self, pos_losses, label, label_weight, bbox_weight, + pos_inds, pos_gt_inds, anchors): + """Fit loss to GMM distribution and separate positive, ignore, negative + samples again with GMM model. + + Args: + pos_losses (Tensor): Losses of all positive samples in + single image. + label (Tensor): classification target of each anchor with + shape (num_anchors,) + label_weight (Tensor): Classification loss weight of each + anchor with shape (num_anchors). + bbox_weight (Tensor): Bbox weight of each anchor with shape + (num_anchors, 4). + pos_inds (Tensor): Index of all positive samples got from + first assign process. + pos_gt_inds (Tensor): Gt_index of all positive samples got + from first assign process. + anchors (list[Tensor]): Anchors of each scale. + + Returns: + tuple: Usually returns a tuple containing learning targets. + + - label (Tensor): classification target of each anchor after + paa assign, with shape (num_anchors,) + - label_weight (Tensor): Classification loss weight of each + anchor after paa assign, with shape (num_anchors). + - bbox_weight (Tensor): Bbox weight of each anchor with shape + (num_anchors, 4). + - num_pos (int): The number of positive samples after paa + assign. + """ + if not len(pos_inds): + return label, label_weight, bbox_weight, 0 + label = label.clone() + label_weight = label_weight.clone() + bbox_weight = bbox_weight.clone() + num_gt = pos_gt_inds.max() + 1 + num_level = len(anchors) + num_anchors_each_level = [item.size(0) for item in anchors] + num_anchors_each_level.insert(0, 0) + inds_level_interval = np.cumsum(num_anchors_each_level) + pos_level_mask = [] + for i in range(num_level): + mask = (pos_inds >= inds_level_interval[i]) & ( + pos_inds < inds_level_interval[i + 1]) + pos_level_mask.append(mask) + pos_inds_after_paa = [label.new_tensor([])] + ignore_inds_after_paa = [label.new_tensor([])] + for gt_ind in range(num_gt): + pos_inds_gmm = [] + pos_loss_gmm = [] + gt_mask = pos_gt_inds == gt_ind + for level in range(num_level): + level_mask = pos_level_mask[level] + level_gt_mask = level_mask & gt_mask + value, topk_inds = pos_losses[level_gt_mask].topk( + min(level_gt_mask.sum(), self.topk), largest=False) + pos_inds_gmm.append(pos_inds[level_gt_mask][topk_inds]) + pos_loss_gmm.append(value) + pos_inds_gmm = torch.cat(pos_inds_gmm) + pos_loss_gmm = torch.cat(pos_loss_gmm) + # fix gmm need at least two sample + if len(pos_inds_gmm) < 2: + continue + device = pos_inds_gmm.device + pos_loss_gmm, sort_inds = pos_loss_gmm.sort() + pos_inds_gmm = pos_inds_gmm[sort_inds] + pos_loss_gmm = pos_loss_gmm.view(-1, 1).cpu().numpy() + min_loss, max_loss = pos_loss_gmm.min(), pos_loss_gmm.max() + means_init = np.array([min_loss, max_loss]).reshape(2, 1) + weights_init = np.array([0.5, 0.5]) + precisions_init = np.array([1.0, 1.0]).reshape(2, 1, 1) # full + if self.covariance_type == 'spherical': + precisions_init = precisions_init.reshape(2) + elif self.covariance_type == 'diag': + precisions_init = precisions_init.reshape(2, 1) + elif self.covariance_type == 'tied': + precisions_init = np.array([[1.0]]) + if skm is None: + raise ImportError('Please run "pip install sklearn" ' + 'to install sklearn first.') + gmm = skm.GaussianMixture( + 2, + weights_init=weights_init, + means_init=means_init, + precisions_init=precisions_init, + covariance_type=self.covariance_type) + gmm.fit(pos_loss_gmm) + gmm_assignment = gmm.predict(pos_loss_gmm) + scores = gmm.score_samples(pos_loss_gmm) + gmm_assignment = torch.from_numpy(gmm_assignment).to(device) + scores = torch.from_numpy(scores).to(device) + + pos_inds_temp, ignore_inds_temp = self.gmm_separation_scheme( + gmm_assignment, scores, pos_inds_gmm) + pos_inds_after_paa.append(pos_inds_temp) + ignore_inds_after_paa.append(ignore_inds_temp) + + pos_inds_after_paa = torch.cat(pos_inds_after_paa) + ignore_inds_after_paa = torch.cat(ignore_inds_after_paa) + reassign_mask = (pos_inds.unsqueeze(1) != pos_inds_after_paa).all(1) + reassign_ids = pos_inds[reassign_mask] + label[reassign_ids] = self.num_classes + label_weight[ignore_inds_after_paa] = 0 + bbox_weight[reassign_ids] = 0 + num_pos = len(pos_inds_after_paa) + return label, label_weight, bbox_weight, num_pos + + def gmm_separation_scheme(self, gmm_assignment, scores, pos_inds_gmm): + """A general separation scheme for gmm model. + + It separates a GMM distribution of candidate samples into three + parts, 0 1 and uncertain areas, and you can implement other + separation schemes by rewriting this function. + + Args: + gmm_assignment (Tensor): The prediction of GMM which is of shape + (num_samples,). The 0/1 value indicates the distribution + that each sample comes from. + scores (Tensor): The probability of sample coming from the + fit GMM distribution. The tensor is of shape (num_samples,). + pos_inds_gmm (Tensor): All the indexes of samples which are used + to fit GMM model. The tensor is of shape (num_samples,) + + Returns: + tuple[Tensor]: The indices of positive and ignored samples. + + - pos_inds_temp (Tensor): Indices of positive samples. + - ignore_inds_temp (Tensor): Indices of ignore samples. + """ + # The implementation is (c) in Fig.3 in origin paper instead of (b). + # You can refer to issues such as + # https://github.com/kkhoot/PAA/issues/8 and + # https://github.com/kkhoot/PAA/issues/9. + fgs = gmm_assignment == 0 + pos_inds_temp = fgs.new_tensor([], dtype=torch.long) + ignore_inds_temp = fgs.new_tensor([], dtype=torch.long) + if fgs.nonzero().numel(): + _, pos_thr_ind = scores[fgs].topk(1) + pos_inds_temp = pos_inds_gmm[fgs][:pos_thr_ind + 1] + ignore_inds_temp = pos_inds_gmm.new_tensor([]) + return pos_inds_temp, ignore_inds_temp + + def get_targets( + self, + anchor_list, + valid_flag_list, + gt_bboxes_list, + img_metas, + gt_bboxes_ignore_list=None, + gt_labels_list=None, + label_channels=1, + unmap_outputs=True, + ): + """Get targets for PAA head. + + This method is almost the same as `AnchorHead.get_targets()`. We direct + return the results from _get_targets_single instead map it to levels + by images_to_levels function. + + Args: + anchor_list (list[list[Tensor]]): Multi level anchors of each + image. The outer list indicates images, and the inner list + corresponds to feature levels of the image. Each element of + the inner list is a tensor of shape (num_anchors, 4). + valid_flag_list (list[list[Tensor]]): Multi level valid flags of + each image. The outer list indicates images, and the inner list + corresponds to feature levels of the image. Each element of + the inner list is a tensor of shape (num_anchors, ) + gt_bboxes_list (list[Tensor]): Ground truth bboxes of each image. + img_metas (list[dict]): Meta info of each image. + gt_bboxes_ignore_list (list[Tensor]): Ground truth bboxes to be + ignored. + gt_labels_list (list[Tensor]): Ground truth labels of each box. + label_channels (int): Channel of label. + unmap_outputs (bool): Whether to map outputs back to the original + set of anchors. + + Returns: + tuple: Usually returns a tuple containing learning targets. + + - labels (list[Tensor]): Labels of all anchors, each with + shape (num_anchors,). + - label_weights (list[Tensor]): Label weights of all anchor. + each with shape (num_anchors,). + - bbox_targets (list[Tensor]): BBox targets of all anchors. + each with shape (num_anchors, 4). + - bbox_weights (list[Tensor]): BBox weights of all anchors. + each with shape (num_anchors, 4). + - pos_inds (list[Tensor]): Contains all index of positive + sample in all anchor. + - gt_inds (list[Tensor]): Contains all gt_index of positive + sample in all anchor. + """ + + num_imgs = len(img_metas) + assert len(anchor_list) == len(valid_flag_list) == num_imgs + concat_anchor_list = [] + concat_valid_flag_list = [] + for i in range(num_imgs): + assert len(anchor_list[i]) == len(valid_flag_list[i]) + concat_anchor_list.append(torch.cat(anchor_list[i])) + concat_valid_flag_list.append(torch.cat(valid_flag_list[i])) + + # compute targets for each image + if gt_bboxes_ignore_list is None: + gt_bboxes_ignore_list = [None for _ in range(num_imgs)] + if gt_labels_list is None: + gt_labels_list = [None for _ in range(num_imgs)] + results = multi_apply( + self._get_targets_single, + concat_anchor_list, + concat_valid_flag_list, + gt_bboxes_list, + gt_bboxes_ignore_list, + gt_labels_list, + img_metas, + label_channels=label_channels, + unmap_outputs=unmap_outputs) + + (labels, label_weights, bbox_targets, bbox_weights, valid_pos_inds, + valid_neg_inds, sampling_result) = results + + # Due to valid flag of anchors, we have to calculate the real pos_inds + # in origin anchor set. + pos_inds = [] + for i, single_labels in enumerate(labels): + pos_mask = (0 <= single_labels) & ( + single_labels < self.num_classes) + pos_inds.append(pos_mask.nonzero().view(-1)) + + gt_inds = [item.pos_assigned_gt_inds for item in sampling_result] + return (labels, label_weights, bbox_targets, bbox_weights, pos_inds, + gt_inds) + + def _get_targets_single(self, + flat_anchors, + valid_flags, + gt_bboxes, + gt_bboxes_ignore, + gt_labels, + img_meta, + label_channels=1, + unmap_outputs=True): + """Compute regression and classification targets for anchors in a + single image. + + This method is same as `AnchorHead._get_targets_single()`. + """ + assert unmap_outputs, 'We must map outputs back to the original' \ + 'set of anchors in PAAhead' + return super(ATSSHead, self)._get_targets_single( + flat_anchors, + valid_flags, + gt_bboxes, + gt_bboxes_ignore, + gt_labels, + img_meta, + label_channels=1, + unmap_outputs=True) + + def _get_bboxes(self, + cls_scores, + bbox_preds, + iou_preds, + mlvl_anchors, + img_shapes, + scale_factors, + cfg, + rescale=False, + with_nms=True): + """Transform outputs for a single batch item into labeled boxes. + + This method is almost same as `ATSSHead._get_bboxes()`. + We use sqrt(iou_preds * cls_scores) in NMS process instead of just + cls_scores. Besides, score voting is used when `` score_voting`` + is set to True. + """ + assert with_nms, 'PAA only supports "with_nms=True" now' + assert len(cls_scores) == len(bbox_preds) == len(mlvl_anchors) + batch_size = cls_scores[0].shape[0] + + mlvl_bboxes = [] + mlvl_scores = [] + mlvl_iou_preds = [] + for cls_score, bbox_pred, iou_preds, anchors in zip( + cls_scores, bbox_preds, iou_preds, mlvl_anchors): + assert cls_score.size()[-2:] == bbox_pred.size()[-2:] + + scores = cls_score.permute(0, 2, 3, 1).reshape( + batch_size, -1, self.cls_out_channels).sigmoid() + bbox_pred = bbox_pred.permute(0, 2, 3, + 1).reshape(batch_size, -1, 4) + iou_preds = iou_preds.permute(0, 2, 3, 1).reshape(batch_size, + -1).sigmoid() + + nms_pre = cfg.get('nms_pre', -1) + if nms_pre > 0 and scores.shape[1] > nms_pre: + max_scores, _ = (scores * iou_preds[..., None]).sqrt().max(-1) + _, topk_inds = max_scores.topk(nms_pre) + batch_inds = torch.arange(batch_size).view( + -1, 1).expand_as(topk_inds).long() + anchors = anchors[topk_inds, :] + bbox_pred = bbox_pred[batch_inds, topk_inds, :] + scores = scores[batch_inds, topk_inds, :] + iou_preds = iou_preds[batch_inds, topk_inds] + else: + anchors = anchors.expand_as(bbox_pred) + + bboxes = self.bbox_coder.decode( + anchors, bbox_pred, max_shape=img_shapes) + mlvl_bboxes.append(bboxes) + mlvl_scores.append(scores) + mlvl_iou_preds.append(iou_preds) + + batch_mlvl_bboxes = torch.cat(mlvl_bboxes, dim=1) + if rescale: + batch_mlvl_bboxes /= batch_mlvl_bboxes.new_tensor( + scale_factors).unsqueeze(1) + batch_mlvl_scores = torch.cat(mlvl_scores, dim=1) + # Add a dummy background class to the backend when using sigmoid + # remind that we set FG labels to [0, num_class-1] since mmdet v2.0 + # BG cat_id: num_class + padding = batch_mlvl_scores.new_zeros(batch_size, + batch_mlvl_scores.shape[1], 1) + batch_mlvl_scores = torch.cat([batch_mlvl_scores, padding], dim=-1) + batch_mlvl_iou_preds = torch.cat(mlvl_iou_preds, dim=1) + batch_mlvl_nms_scores = (batch_mlvl_scores * + batch_mlvl_iou_preds[..., None]).sqrt() + + det_results = [] + for (mlvl_bboxes, mlvl_scores) in zip(batch_mlvl_bboxes, + batch_mlvl_nms_scores): + det_bbox, det_label = multiclass_nms( + mlvl_bboxes, + mlvl_scores, + cfg.score_thr, + cfg.nms, + cfg.max_per_img, + score_factors=None) + if self.with_score_voting and len(det_bbox) > 0: + det_bbox, det_label = self.score_voting( + det_bbox, det_label, mlvl_bboxes, mlvl_scores, + cfg.score_thr) + det_results.append(tuple([det_bbox, det_label])) + + return det_results + + def score_voting(self, det_bboxes, det_labels, mlvl_bboxes, + mlvl_nms_scores, score_thr): + """Implementation of score voting method works on each remaining boxes + after NMS procedure. + + Args: + det_bboxes (Tensor): Remaining boxes after NMS procedure, + with shape (k, 5), each dimension means + (x1, y1, x2, y2, score). + det_labels (Tensor): The label of remaining boxes, with shape + (k, 1),Labels are 0-based. + mlvl_bboxes (Tensor): All boxes before the NMS procedure, + with shape (num_anchors,4). + mlvl_nms_scores (Tensor): The scores of all boxes which is used + in the NMS procedure, with shape (num_anchors, num_class) + mlvl_iou_preds (Tensor): The predictions of IOU of all boxes + before the NMS procedure, with shape (num_anchors, 1) + score_thr (float): The score threshold of bboxes. + + Returns: + tuple: Usually returns a tuple containing voting results. + + - det_bboxes_voted (Tensor): Remaining boxes after + score voting procedure, with shape (k, 5), each + dimension means (x1, y1, x2, y2, score). + - det_labels_voted (Tensor): Label of remaining bboxes + after voting, with shape (num_anchors,). + """ + candidate_mask = mlvl_nms_scores > score_thr + candidate_mask_nonzeros = candidate_mask.nonzero() + candidate_inds = candidate_mask_nonzeros[:, 0] + candidate_labels = candidate_mask_nonzeros[:, 1] + candidate_bboxes = mlvl_bboxes[candidate_inds] + candidate_scores = mlvl_nms_scores[candidate_mask] + det_bboxes_voted = [] + det_labels_voted = [] + for cls in range(self.cls_out_channels): + candidate_cls_mask = candidate_labels == cls + if not candidate_cls_mask.any(): + continue + candidate_cls_scores = candidate_scores[candidate_cls_mask] + candidate_cls_bboxes = candidate_bboxes[candidate_cls_mask] + det_cls_mask = det_labels == cls + det_cls_bboxes = det_bboxes[det_cls_mask].view( + -1, det_bboxes.size(-1)) + det_candidate_ious = bbox_overlaps(det_cls_bboxes[:, :4], + candidate_cls_bboxes) + for det_ind in range(len(det_cls_bboxes)): + single_det_ious = det_candidate_ious[det_ind] + pos_ious_mask = single_det_ious > 0.01 + pos_ious = single_det_ious[pos_ious_mask] + pos_bboxes = candidate_cls_bboxes[pos_ious_mask] + pos_scores = candidate_cls_scores[pos_ious_mask] + pis = (torch.exp(-(1 - pos_ious)**2 / 0.025) * + pos_scores)[:, None] + voted_box = torch.sum( + pis * pos_bboxes, dim=0) / torch.sum( + pis, dim=0) + voted_score = det_cls_bboxes[det_ind][-1:][None, :] + det_bboxes_voted.append( + torch.cat((voted_box[None, :], voted_score), dim=1)) + det_labels_voted.append(cls) + + det_bboxes_voted = torch.cat(det_bboxes_voted, dim=0) + det_labels_voted = det_labels.new_tensor(det_labels_voted) + return det_bboxes_voted, det_labels_voted diff --git a/detection/mmdet/models/dense_heads/pisa_retinanet_head.py b/detection/mmdet/models/dense_heads/pisa_retinanet_head.py new file mode 100644 index 0000000..bd87b9a --- /dev/null +++ b/detection/mmdet/models/dense_heads/pisa_retinanet_head.py @@ -0,0 +1,154 @@ +import torch +from mmcv.runner import force_fp32 + +from mmdet.core import images_to_levels +from ..builder import HEADS +from ..losses import carl_loss, isr_p +from .retina_head import RetinaHead + + +@HEADS.register_module() +class PISARetinaHead(RetinaHead): + """PISA Retinanet Head. + + The head owns the same structure with Retinanet Head, but differs in two + aspects: + 1. Importance-based Sample Reweighting Positive (ISR-P) is applied to + change the positive loss weights. + 2. Classification-aware regression loss is adopted as a third loss. + """ + + @force_fp32(apply_to=('cls_scores', 'bbox_preds')) + def loss(self, + cls_scores, + bbox_preds, + gt_bboxes, + gt_labels, + img_metas, + gt_bboxes_ignore=None): + """Compute losses of the head. + + Args: + cls_scores (list[Tensor]): Box scores for each scale level + Has shape (N, num_anchors * num_classes, H, W) + bbox_preds (list[Tensor]): Box energies / deltas for each scale + level with shape (N, num_anchors * 4, H, W) + gt_bboxes (list[Tensor]): Ground truth bboxes of each image + with shape (num_obj, 4). + gt_labels (list[Tensor]): Ground truth labels of each image + with shape (num_obj, 4). + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + gt_bboxes_ignore (list[Tensor]): Ignored gt bboxes of each image. + Default: None. + + Returns: + dict: Loss dict, comprise classification loss, regression loss and + carl loss. + """ + featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] + assert len(featmap_sizes) == self.anchor_generator.num_levels + + device = cls_scores[0].device + + anchor_list, valid_flag_list = self.get_anchors( + featmap_sizes, img_metas, device=device) + label_channels = self.cls_out_channels if self.use_sigmoid_cls else 1 + cls_reg_targets = self.get_targets( + anchor_list, + valid_flag_list, + gt_bboxes, + img_metas, + gt_bboxes_ignore_list=gt_bboxes_ignore, + gt_labels_list=gt_labels, + label_channels=label_channels, + return_sampling_results=True) + if cls_reg_targets is None: + return None + (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, + num_total_pos, num_total_neg, sampling_results_list) = cls_reg_targets + num_total_samples = ( + num_total_pos + num_total_neg if self.sampling else num_total_pos) + + # anchor number of multi levels + num_level_anchors = [anchors.size(0) for anchors in anchor_list[0]] + # concat all level anchors and flags to a single tensor + concat_anchor_list = [] + for i in range(len(anchor_list)): + concat_anchor_list.append(torch.cat(anchor_list[i])) + all_anchor_list = images_to_levels(concat_anchor_list, + num_level_anchors) + + num_imgs = len(img_metas) + flatten_cls_scores = [ + cls_score.permute(0, 2, 3, 1).reshape(num_imgs, -1, label_channels) + for cls_score in cls_scores + ] + flatten_cls_scores = torch.cat( + flatten_cls_scores, dim=1).reshape(-1, + flatten_cls_scores[0].size(-1)) + flatten_bbox_preds = [ + bbox_pred.permute(0, 2, 3, 1).reshape(num_imgs, -1, 4) + for bbox_pred in bbox_preds + ] + flatten_bbox_preds = torch.cat( + flatten_bbox_preds, dim=1).view(-1, flatten_bbox_preds[0].size(-1)) + flatten_labels = torch.cat(labels_list, dim=1).reshape(-1) + flatten_label_weights = torch.cat( + label_weights_list, dim=1).reshape(-1) + flatten_anchors = torch.cat(all_anchor_list, dim=1).reshape(-1, 4) + flatten_bbox_targets = torch.cat( + bbox_targets_list, dim=1).reshape(-1, 4) + flatten_bbox_weights = torch.cat( + bbox_weights_list, dim=1).reshape(-1, 4) + + # Apply ISR-P + isr_cfg = self.train_cfg.get('isr', None) + if isr_cfg is not None: + all_targets = (flatten_labels, flatten_label_weights, + flatten_bbox_targets, flatten_bbox_weights) + with torch.no_grad(): + all_targets = isr_p( + flatten_cls_scores, + flatten_bbox_preds, + all_targets, + flatten_anchors, + sampling_results_list, + bbox_coder=self.bbox_coder, + loss_cls=self.loss_cls, + num_class=self.num_classes, + **self.train_cfg.isr) + (flatten_labels, flatten_label_weights, flatten_bbox_targets, + flatten_bbox_weights) = all_targets + + # For convenience we compute loss once instead separating by fpn level, + # so that we don't need to separate the weights by level again. + # The result should be the same + losses_cls = self.loss_cls( + flatten_cls_scores, + flatten_labels, + flatten_label_weights, + avg_factor=num_total_samples) + losses_bbox = self.loss_bbox( + flatten_bbox_preds, + flatten_bbox_targets, + flatten_bbox_weights, + avg_factor=num_total_samples) + loss_dict = dict(loss_cls=losses_cls, loss_bbox=losses_bbox) + + # CARL Loss + carl_cfg = self.train_cfg.get('carl', None) + if carl_cfg is not None: + loss_carl = carl_loss( + flatten_cls_scores, + flatten_labels, + flatten_bbox_preds, + flatten_bbox_targets, + self.loss_bbox, + **self.train_cfg.carl, + avg_factor=num_total_pos, + sigmoid=True, + num_class=self.num_classes) + loss_dict.update(loss_carl) + + return loss_dict diff --git a/detection/mmdet/models/dense_heads/pisa_ssd_head.py b/detection/mmdet/models/dense_heads/pisa_ssd_head.py new file mode 100644 index 0000000..90ef3c8 --- /dev/null +++ b/detection/mmdet/models/dense_heads/pisa_ssd_head.py @@ -0,0 +1,139 @@ +import torch + +from mmdet.core import multi_apply +from ..builder import HEADS +from ..losses import CrossEntropyLoss, SmoothL1Loss, carl_loss, isr_p +from .ssd_head import SSDHead + + +# TODO: add loss evaluator for SSD +@HEADS.register_module() +class PISASSDHead(SSDHead): + + def loss(self, + cls_scores, + bbox_preds, + gt_bboxes, + gt_labels, + img_metas, + gt_bboxes_ignore=None): + """Compute losses of the head. + + Args: + cls_scores (list[Tensor]): Box scores for each scale level + Has shape (N, num_anchors * num_classes, H, W) + bbox_preds (list[Tensor]): Box energies / deltas for each scale + level with shape (N, num_anchors * 4, H, W) + gt_bboxes (list[Tensor]): Ground truth bboxes of each image + with shape (num_obj, 4). + gt_labels (list[Tensor]): Ground truth labels of each image + with shape (num_obj, 4). + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + gt_bboxes_ignore (list[Tensor]): Ignored gt bboxes of each image. + Default: None. + + Returns: + dict: Loss dict, comprise classification loss regression loss and + carl loss. + """ + featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] + assert len(featmap_sizes) == self.anchor_generator.num_levels + + device = cls_scores[0].device + + anchor_list, valid_flag_list = self.get_anchors( + featmap_sizes, img_metas, device=device) + cls_reg_targets = self.get_targets( + anchor_list, + valid_flag_list, + gt_bboxes, + img_metas, + gt_bboxes_ignore_list=gt_bboxes_ignore, + gt_labels_list=gt_labels, + label_channels=1, + unmap_outputs=False, + return_sampling_results=True) + if cls_reg_targets is None: + return None + (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, + num_total_pos, num_total_neg, sampling_results_list) = cls_reg_targets + + num_images = len(img_metas) + all_cls_scores = torch.cat([ + s.permute(0, 2, 3, 1).reshape( + num_images, -1, self.cls_out_channels) for s in cls_scores + ], 1) + all_labels = torch.cat(labels_list, -1).view(num_images, -1) + all_label_weights = torch.cat(label_weights_list, + -1).view(num_images, -1) + all_bbox_preds = torch.cat([ + b.permute(0, 2, 3, 1).reshape(num_images, -1, 4) + for b in bbox_preds + ], -2) + all_bbox_targets = torch.cat(bbox_targets_list, + -2).view(num_images, -1, 4) + all_bbox_weights = torch.cat(bbox_weights_list, + -2).view(num_images, -1, 4) + + # concat all level anchors to a single tensor + all_anchors = [] + for i in range(num_images): + all_anchors.append(torch.cat(anchor_list[i])) + + isr_cfg = self.train_cfg.get('isr', None) + all_targets = (all_labels.view(-1), all_label_weights.view(-1), + all_bbox_targets.view(-1, + 4), all_bbox_weights.view(-1, 4)) + # apply ISR-P + if isr_cfg is not None: + all_targets = isr_p( + all_cls_scores.view(-1, all_cls_scores.size(-1)), + all_bbox_preds.view(-1, 4), + all_targets, + torch.cat(all_anchors), + sampling_results_list, + loss_cls=CrossEntropyLoss(), + bbox_coder=self.bbox_coder, + **self.train_cfg.isr, + num_class=self.num_classes) + (new_labels, new_label_weights, new_bbox_targets, + new_bbox_weights) = all_targets + all_labels = new_labels.view(all_labels.shape) + all_label_weights = new_label_weights.view(all_label_weights.shape) + all_bbox_targets = new_bbox_targets.view(all_bbox_targets.shape) + all_bbox_weights = new_bbox_weights.view(all_bbox_weights.shape) + + # add CARL loss + carl_loss_cfg = self.train_cfg.get('carl', None) + if carl_loss_cfg is not None: + loss_carl = carl_loss( + all_cls_scores.view(-1, all_cls_scores.size(-1)), + all_targets[0], + all_bbox_preds.view(-1, 4), + all_targets[2], + SmoothL1Loss(beta=1.), + **self.train_cfg.carl, + avg_factor=num_total_pos, + num_class=self.num_classes) + + # check NaN and Inf + assert torch.isfinite(all_cls_scores).all().item(), \ + 'classification scores become infinite or NaN!' + assert torch.isfinite(all_bbox_preds).all().item(), \ + 'bbox predications become infinite or NaN!' + + losses_cls, losses_bbox = multi_apply( + self.loss_single, + all_cls_scores, + all_bbox_preds, + all_anchors, + all_labels, + all_label_weights, + all_bbox_targets, + all_bbox_weights, + num_total_samples=num_total_pos) + loss_dict = dict(loss_cls=losses_cls, loss_bbox=losses_bbox) + if carl_loss_cfg is not None: + loss_dict.update(loss_carl) + return loss_dict diff --git a/detection/mmdet/models/dense_heads/reppoints_head.py b/detection/mmdet/models/dense_heads/reppoints_head.py new file mode 100644 index 0000000..499cc4f --- /dev/null +++ b/detection/mmdet/models/dense_heads/reppoints_head.py @@ -0,0 +1,763 @@ +import numpy as np +import torch +import torch.nn as nn +from mmcv.cnn import ConvModule, bias_init_with_prob, normal_init +from mmcv.ops import DeformConv2d + +from mmdet.core import (PointGenerator, build_assigner, build_sampler, + images_to_levels, multi_apply, multiclass_nms, unmap) +from ..builder import HEADS, build_loss +from .anchor_free_head import AnchorFreeHead + + +@HEADS.register_module() +class RepPointsHead(AnchorFreeHead): + """RepPoint head. + + Args: + point_feat_channels (int): Number of channels of points features. + gradient_mul (float): The multiplier to gradients from + points refinement and recognition. + point_strides (Iterable): points strides. + point_base_scale (int): bbox scale for assigning labels. + loss_cls (dict): Config of classification loss. + loss_bbox_init (dict): Config of initial points loss. + loss_bbox_refine (dict): Config of points loss in refinement. + use_grid_points (bool): If we use bounding box representation, the + reppoints is represented as grid points on the bounding box. + center_init (bool): Whether to use center point assignment. + transform_method (str): The methods to transform RepPoints to bbox. + """ # noqa: W605 + + def __init__(self, + num_classes, + in_channels, + point_feat_channels=256, + num_points=9, + gradient_mul=0.1, + point_strides=[8, 16, 32, 64, 128], + point_base_scale=4, + loss_cls=dict( + type='FocalLoss', + use_sigmoid=True, + gamma=2.0, + alpha=0.25, + loss_weight=1.0), + loss_bbox_init=dict( + type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=0.5), + loss_bbox_refine=dict( + type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0), + use_grid_points=False, + center_init=True, + transform_method='moment', + moment_mul=0.01, + **kwargs): + self.num_points = num_points + self.point_feat_channels = point_feat_channels + self.use_grid_points = use_grid_points + self.center_init = center_init + + # we use deform conv to extract points features + self.dcn_kernel = int(np.sqrt(num_points)) + self.dcn_pad = int((self.dcn_kernel - 1) / 2) + assert self.dcn_kernel * self.dcn_kernel == num_points, \ + 'The points number should be a square number.' + assert self.dcn_kernel % 2 == 1, \ + 'The points number should be an odd square number.' + dcn_base = np.arange(-self.dcn_pad, + self.dcn_pad + 1).astype(np.float64) + dcn_base_y = np.repeat(dcn_base, self.dcn_kernel) + dcn_base_x = np.tile(dcn_base, self.dcn_kernel) + dcn_base_offset = np.stack([dcn_base_y, dcn_base_x], axis=1).reshape( + (-1)) + self.dcn_base_offset = torch.tensor(dcn_base_offset).view(1, -1, 1, 1) + + super().__init__(num_classes, in_channels, loss_cls=loss_cls, **kwargs) + + self.gradient_mul = gradient_mul + self.point_base_scale = point_base_scale + self.point_strides = point_strides + self.point_generators = [PointGenerator() for _ in self.point_strides] + + self.sampling = loss_cls['type'] not in ['FocalLoss'] + if self.train_cfg: + self.init_assigner = build_assigner(self.train_cfg.init.assigner) + self.refine_assigner = build_assigner( + self.train_cfg.refine.assigner) + # use PseudoSampler when sampling is False + if self.sampling and hasattr(self.train_cfg, 'sampler'): + sampler_cfg = self.train_cfg.sampler + else: + sampler_cfg = dict(type='PseudoSampler') + self.sampler = build_sampler(sampler_cfg, context=self) + self.transform_method = transform_method + if self.transform_method == 'moment': + self.moment_transfer = nn.Parameter( + data=torch.zeros(2), requires_grad=True) + self.moment_mul = moment_mul + + self.use_sigmoid_cls = loss_cls.get('use_sigmoid', False) + if self.use_sigmoid_cls: + self.cls_out_channels = self.num_classes + else: + self.cls_out_channels = self.num_classes + 1 + self.loss_bbox_init = build_loss(loss_bbox_init) + self.loss_bbox_refine = build_loss(loss_bbox_refine) + + def _init_layers(self): + """Initialize layers of the head.""" + self.relu = nn.ReLU(inplace=True) + self.cls_convs = nn.ModuleList() + self.reg_convs = nn.ModuleList() + for i in range(self.stacked_convs): + chn = self.in_channels if i == 0 else self.feat_channels + self.cls_convs.append( + ConvModule( + chn, + self.feat_channels, + 3, + stride=1, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg)) + self.reg_convs.append( + ConvModule( + chn, + self.feat_channels, + 3, + stride=1, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg)) + pts_out_dim = 4 if self.use_grid_points else 2 * self.num_points + self.reppoints_cls_conv = DeformConv2d(self.feat_channels, + self.point_feat_channels, + self.dcn_kernel, 1, + self.dcn_pad) + self.reppoints_cls_out = nn.Conv2d(self.point_feat_channels, + self.cls_out_channels, 1, 1, 0) + self.reppoints_pts_init_conv = nn.Conv2d(self.feat_channels, + self.point_feat_channels, 3, + 1, 1) + self.reppoints_pts_init_out = nn.Conv2d(self.point_feat_channels, + pts_out_dim, 1, 1, 0) + self.reppoints_pts_refine_conv = DeformConv2d(self.feat_channels, + self.point_feat_channels, + self.dcn_kernel, 1, + self.dcn_pad) + self.reppoints_pts_refine_out = nn.Conv2d(self.point_feat_channels, + pts_out_dim, 1, 1, 0) + + def init_weights(self): + """Initialize weights of the head.""" + for m in self.cls_convs: + normal_init(m.conv, std=0.01) + for m in self.reg_convs: + normal_init(m.conv, std=0.01) + bias_cls = bias_init_with_prob(0.01) + normal_init(self.reppoints_cls_conv, std=0.01) + normal_init(self.reppoints_cls_out, std=0.01, bias=bias_cls) + normal_init(self.reppoints_pts_init_conv, std=0.01) + normal_init(self.reppoints_pts_init_out, std=0.01) + normal_init(self.reppoints_pts_refine_conv, std=0.01) + normal_init(self.reppoints_pts_refine_out, std=0.01) + + def points2bbox(self, pts, y_first=True): + """Converting the points set into bounding box. + + :param pts: the input points sets (fields), each points + set (fields) is represented as 2n scalar. + :param y_first: if y_first=True, the point set is represented as + [y1, x1, y2, x2 ... yn, xn], otherwise the point set is + represented as [x1, y1, x2, y2 ... xn, yn]. + :return: each points set is converting to a bbox [x1, y1, x2, y2]. + """ + pts_reshape = pts.view(pts.shape[0], -1, 2, *pts.shape[2:]) + pts_y = pts_reshape[:, :, 0, ...] if y_first else pts_reshape[:, :, 1, + ...] + pts_x = pts_reshape[:, :, 1, ...] if y_first else pts_reshape[:, :, 0, + ...] + if self.transform_method == 'minmax': + bbox_left = pts_x.min(dim=1, keepdim=True)[0] + bbox_right = pts_x.max(dim=1, keepdim=True)[0] + bbox_up = pts_y.min(dim=1, keepdim=True)[0] + bbox_bottom = pts_y.max(dim=1, keepdim=True)[0] + bbox = torch.cat([bbox_left, bbox_up, bbox_right, bbox_bottom], + dim=1) + elif self.transform_method == 'partial_minmax': + pts_y = pts_y[:, :4, ...] + pts_x = pts_x[:, :4, ...] + bbox_left = pts_x.min(dim=1, keepdim=True)[0] + bbox_right = pts_x.max(dim=1, keepdim=True)[0] + bbox_up = pts_y.min(dim=1, keepdim=True)[0] + bbox_bottom = pts_y.max(dim=1, keepdim=True)[0] + bbox = torch.cat([bbox_left, bbox_up, bbox_right, bbox_bottom], + dim=1) + elif self.transform_method == 'moment': + pts_y_mean = pts_y.mean(dim=1, keepdim=True) + pts_x_mean = pts_x.mean(dim=1, keepdim=True) + pts_y_std = torch.std(pts_y - pts_y_mean, dim=1, keepdim=True) + pts_x_std = torch.std(pts_x - pts_x_mean, dim=1, keepdim=True) + moment_transfer = (self.moment_transfer * self.moment_mul) + ( + self.moment_transfer.detach() * (1 - self.moment_mul)) + moment_width_transfer = moment_transfer[0] + moment_height_transfer = moment_transfer[1] + half_width = pts_x_std * torch.exp(moment_width_transfer) + half_height = pts_y_std * torch.exp(moment_height_transfer) + bbox = torch.cat([ + pts_x_mean - half_width, pts_y_mean - half_height, + pts_x_mean + half_width, pts_y_mean + half_height + ], + dim=1) + else: + raise NotImplementedError + return bbox + + def gen_grid_from_reg(self, reg, previous_boxes): + """Base on the previous bboxes and regression values, we compute the + regressed bboxes and generate the grids on the bboxes. + + :param reg: the regression value to previous bboxes. + :param previous_boxes: previous bboxes. + :return: generate grids on the regressed bboxes. + """ + b, _, h, w = reg.shape + bxy = (previous_boxes[:, :2, ...] + previous_boxes[:, 2:, ...]) / 2. + bwh = (previous_boxes[:, 2:, ...] - + previous_boxes[:, :2, ...]).clamp(min=1e-6) + grid_topleft = bxy + bwh * reg[:, :2, ...] - 0.5 * bwh * torch.exp( + reg[:, 2:, ...]) + grid_wh = bwh * torch.exp(reg[:, 2:, ...]) + grid_left = grid_topleft[:, [0], ...] + grid_top = grid_topleft[:, [1], ...] + grid_width = grid_wh[:, [0], ...] + grid_height = grid_wh[:, [1], ...] + intervel = torch.linspace(0., 1., self.dcn_kernel).view( + 1, self.dcn_kernel, 1, 1).type_as(reg) + grid_x = grid_left + grid_width * intervel + grid_x = grid_x.unsqueeze(1).repeat(1, self.dcn_kernel, 1, 1, 1) + grid_x = grid_x.view(b, -1, h, w) + grid_y = grid_top + grid_height * intervel + grid_y = grid_y.unsqueeze(2).repeat(1, 1, self.dcn_kernel, 1, 1) + grid_y = grid_y.view(b, -1, h, w) + grid_yx = torch.stack([grid_y, grid_x], dim=2) + grid_yx = grid_yx.view(b, -1, h, w) + regressed_bbox = torch.cat([ + grid_left, grid_top, grid_left + grid_width, grid_top + grid_height + ], 1) + return grid_yx, regressed_bbox + + def forward(self, feats): + return multi_apply(self.forward_single, feats) + + def forward_single(self, x): + """Forward feature map of a single FPN level.""" + dcn_base_offset = self.dcn_base_offset.type_as(x) + # If we use center_init, the initial reppoints is from center points. + # If we use bounding bbox representation, the initial reppoints is + # from regular grid placed on a pre-defined bbox. + if self.use_grid_points or not self.center_init: + scale = self.point_base_scale / 2 + points_init = dcn_base_offset / dcn_base_offset.max() * scale + bbox_init = x.new_tensor([-scale, -scale, scale, + scale]).view(1, 4, 1, 1) + else: + points_init = 0 + cls_feat = x + pts_feat = x + for cls_conv in self.cls_convs: + cls_feat = cls_conv(cls_feat) + for reg_conv in self.reg_convs: + pts_feat = reg_conv(pts_feat) + # initialize reppoints + pts_out_init = self.reppoints_pts_init_out( + self.relu(self.reppoints_pts_init_conv(pts_feat))) + if self.use_grid_points: + pts_out_init, bbox_out_init = self.gen_grid_from_reg( + pts_out_init, bbox_init.detach()) + else: + pts_out_init = pts_out_init + points_init + # refine and classify reppoints + pts_out_init_grad_mul = (1 - self.gradient_mul) * pts_out_init.detach( + ) + self.gradient_mul * pts_out_init + dcn_offset = pts_out_init_grad_mul - dcn_base_offset + cls_out = self.reppoints_cls_out( + self.relu(self.reppoints_cls_conv(cls_feat, dcn_offset))) + pts_out_refine = self.reppoints_pts_refine_out( + self.relu(self.reppoints_pts_refine_conv(pts_feat, dcn_offset))) + if self.use_grid_points: + pts_out_refine, bbox_out_refine = self.gen_grid_from_reg( + pts_out_refine, bbox_out_init.detach()) + else: + pts_out_refine = pts_out_refine + pts_out_init.detach() + return cls_out, pts_out_init, pts_out_refine + + def get_points(self, featmap_sizes, img_metas, device): + """Get points according to feature map sizes. + + Args: + featmap_sizes (list[tuple]): Multi-level feature map sizes. + img_metas (list[dict]): Image meta info. + + Returns: + tuple: points of each image, valid flags of each image + """ + num_imgs = len(img_metas) + num_levels = len(featmap_sizes) + + # since feature map sizes of all images are the same, we only compute + # points center for one time + multi_level_points = [] + for i in range(num_levels): + points = self.point_generators[i].grid_points( + featmap_sizes[i], self.point_strides[i], device) + multi_level_points.append(points) + points_list = [[point.clone() for point in multi_level_points] + for _ in range(num_imgs)] + + # for each image, we compute valid flags of multi level grids + valid_flag_list = [] + for img_id, img_meta in enumerate(img_metas): + multi_level_flags = [] + for i in range(num_levels): + point_stride = self.point_strides[i] + feat_h, feat_w = featmap_sizes[i] + h, w = img_meta['pad_shape'][:2] + valid_feat_h = min(int(np.ceil(h / point_stride)), feat_h) + valid_feat_w = min(int(np.ceil(w / point_stride)), feat_w) + flags = self.point_generators[i].valid_flags( + (feat_h, feat_w), (valid_feat_h, valid_feat_w), device) + multi_level_flags.append(flags) + valid_flag_list.append(multi_level_flags) + + return points_list, valid_flag_list + + def centers_to_bboxes(self, point_list): + """Get bboxes according to center points. + + Only used in :class:`MaxIoUAssigner`. + """ + bbox_list = [] + for i_img, point in enumerate(point_list): + bbox = [] + for i_lvl in range(len(self.point_strides)): + scale = self.point_base_scale * self.point_strides[i_lvl] * 0.5 + bbox_shift = torch.Tensor([-scale, -scale, scale, + scale]).view(1, 4).type_as(point[0]) + bbox_center = torch.cat( + [point[i_lvl][:, :2], point[i_lvl][:, :2]], dim=1) + bbox.append(bbox_center + bbox_shift) + bbox_list.append(bbox) + return bbox_list + + def offset_to_pts(self, center_list, pred_list): + """Change from point offset to point coordinate.""" + pts_list = [] + for i_lvl in range(len(self.point_strides)): + pts_lvl = [] + for i_img in range(len(center_list)): + pts_center = center_list[i_img][i_lvl][:, :2].repeat( + 1, self.num_points) + pts_shift = pred_list[i_lvl][i_img] + yx_pts_shift = pts_shift.permute(1, 2, 0).view( + -1, 2 * self.num_points) + y_pts_shift = yx_pts_shift[..., 0::2] + x_pts_shift = yx_pts_shift[..., 1::2] + xy_pts_shift = torch.stack([x_pts_shift, y_pts_shift], -1) + xy_pts_shift = xy_pts_shift.view(*yx_pts_shift.shape[:-1], -1) + pts = xy_pts_shift * self.point_strides[i_lvl] + pts_center + pts_lvl.append(pts) + pts_lvl = torch.stack(pts_lvl, 0) + pts_list.append(pts_lvl) + return pts_list + + def _point_target_single(self, + flat_proposals, + valid_flags, + gt_bboxes, + gt_bboxes_ignore, + gt_labels, + label_channels=1, + stage='init', + unmap_outputs=True): + inside_flags = valid_flags + if not inside_flags.any(): + return (None, ) * 7 + # assign gt and sample proposals + proposals = flat_proposals[inside_flags, :] + + if stage == 'init': + assigner = self.init_assigner + pos_weight = self.train_cfg.init.pos_weight + else: + assigner = self.refine_assigner + pos_weight = self.train_cfg.refine.pos_weight + assign_result = assigner.assign(proposals, gt_bboxes, gt_bboxes_ignore, + None if self.sampling else gt_labels) + sampling_result = self.sampler.sample(assign_result, proposals, + gt_bboxes) + + num_valid_proposals = proposals.shape[0] + bbox_gt = proposals.new_zeros([num_valid_proposals, 4]) + pos_proposals = torch.zeros_like(proposals) + proposals_weights = proposals.new_zeros([num_valid_proposals, 4]) + labels = proposals.new_full((num_valid_proposals, ), + self.num_classes, + dtype=torch.long) + label_weights = proposals.new_zeros( + num_valid_proposals, dtype=torch.float) + + pos_inds = sampling_result.pos_inds + neg_inds = sampling_result.neg_inds + if len(pos_inds) > 0: + pos_gt_bboxes = sampling_result.pos_gt_bboxes + bbox_gt[pos_inds, :] = pos_gt_bboxes + pos_proposals[pos_inds, :] = proposals[pos_inds, :] + proposals_weights[pos_inds, :] = 1.0 + if gt_labels is None: + # Only rpn gives gt_labels as None + # Foreground is the first class + labels[pos_inds] = 0 + else: + labels[pos_inds] = gt_labels[ + sampling_result.pos_assigned_gt_inds] + if pos_weight <= 0: + label_weights[pos_inds] = 1.0 + else: + label_weights[pos_inds] = pos_weight + if len(neg_inds) > 0: + label_weights[neg_inds] = 1.0 + + # map up to original set of proposals + if unmap_outputs: + num_total_proposals = flat_proposals.size(0) + labels = unmap(labels, num_total_proposals, inside_flags) + label_weights = unmap(label_weights, num_total_proposals, + inside_flags) + bbox_gt = unmap(bbox_gt, num_total_proposals, inside_flags) + pos_proposals = unmap(pos_proposals, num_total_proposals, + inside_flags) + proposals_weights = unmap(proposals_weights, num_total_proposals, + inside_flags) + + return (labels, label_weights, bbox_gt, pos_proposals, + proposals_weights, pos_inds, neg_inds) + + def get_targets(self, + proposals_list, + valid_flag_list, + gt_bboxes_list, + img_metas, + gt_bboxes_ignore_list=None, + gt_labels_list=None, + stage='init', + label_channels=1, + unmap_outputs=True): + """Compute corresponding GT box and classification targets for + proposals. + + Args: + proposals_list (list[list]): Multi level points/bboxes of each + image. + valid_flag_list (list[list]): Multi level valid flags of each + image. + gt_bboxes_list (list[Tensor]): Ground truth bboxes of each image. + img_metas (list[dict]): Meta info of each image. + gt_bboxes_ignore_list (list[Tensor]): Ground truth bboxes to be + ignored. + gt_bboxes_list (list[Tensor]): Ground truth labels of each box. + stage (str): `init` or `refine`. Generate target for init stage or + refine stage + label_channels (int): Channel of label. + unmap_outputs (bool): Whether to map outputs back to the original + set of anchors. + + Returns: + tuple: + - labels_list (list[Tensor]): Labels of each level. + - label_weights_list (list[Tensor]): Label weights of each level. # noqa: E501 + - bbox_gt_list (list[Tensor]): Ground truth bbox of each level. + - proposal_list (list[Tensor]): Proposals(points/bboxes) of each level. # noqa: E501 + - proposal_weights_list (list[Tensor]): Proposal weights of each level. # noqa: E501 + - num_total_pos (int): Number of positive samples in all images. # noqa: E501 + - num_total_neg (int): Number of negative samples in all images. # noqa: E501 + """ + assert stage in ['init', 'refine'] + num_imgs = len(img_metas) + assert len(proposals_list) == len(valid_flag_list) == num_imgs + + # points number of multi levels + num_level_proposals = [points.size(0) for points in proposals_list[0]] + + # concat all level points and flags to a single tensor + for i in range(num_imgs): + assert len(proposals_list[i]) == len(valid_flag_list[i]) + proposals_list[i] = torch.cat(proposals_list[i]) + valid_flag_list[i] = torch.cat(valid_flag_list[i]) + + # compute targets for each image + if gt_bboxes_ignore_list is None: + gt_bboxes_ignore_list = [None for _ in range(num_imgs)] + if gt_labels_list is None: + gt_labels_list = [None for _ in range(num_imgs)] + (all_labels, all_label_weights, all_bbox_gt, all_proposals, + all_proposal_weights, pos_inds_list, neg_inds_list) = multi_apply( + self._point_target_single, + proposals_list, + valid_flag_list, + gt_bboxes_list, + gt_bboxes_ignore_list, + gt_labels_list, + stage=stage, + label_channels=label_channels, + unmap_outputs=unmap_outputs) + # no valid points + if any([labels is None for labels in all_labels]): + return None + # sampled points of all images + num_total_pos = sum([max(inds.numel(), 1) for inds in pos_inds_list]) + num_total_neg = sum([max(inds.numel(), 1) for inds in neg_inds_list]) + labels_list = images_to_levels(all_labels, num_level_proposals) + label_weights_list = images_to_levels(all_label_weights, + num_level_proposals) + bbox_gt_list = images_to_levels(all_bbox_gt, num_level_proposals) + proposals_list = images_to_levels(all_proposals, num_level_proposals) + proposal_weights_list = images_to_levels(all_proposal_weights, + num_level_proposals) + return (labels_list, label_weights_list, bbox_gt_list, proposals_list, + proposal_weights_list, num_total_pos, num_total_neg) + + def loss_single(self, cls_score, pts_pred_init, pts_pred_refine, labels, + label_weights, bbox_gt_init, bbox_weights_init, + bbox_gt_refine, bbox_weights_refine, stride, + num_total_samples_init, num_total_samples_refine): + # classification loss + labels = labels.reshape(-1) + label_weights = label_weights.reshape(-1) + cls_score = cls_score.permute(0, 2, 3, + 1).reshape(-1, self.cls_out_channels) + cls_score = cls_score.contiguous() + loss_cls = self.loss_cls( + cls_score, + labels, + label_weights, + avg_factor=num_total_samples_refine) + + # points loss + bbox_gt_init = bbox_gt_init.reshape(-1, 4) + bbox_weights_init = bbox_weights_init.reshape(-1, 4) + bbox_pred_init = self.points2bbox( + pts_pred_init.reshape(-1, 2 * self.num_points), y_first=False) + bbox_gt_refine = bbox_gt_refine.reshape(-1, 4) + bbox_weights_refine = bbox_weights_refine.reshape(-1, 4) + bbox_pred_refine = self.points2bbox( + pts_pred_refine.reshape(-1, 2 * self.num_points), y_first=False) + normalize_term = self.point_base_scale * stride + loss_pts_init = self.loss_bbox_init( + bbox_pred_init / normalize_term, + bbox_gt_init / normalize_term, + bbox_weights_init, + avg_factor=num_total_samples_init) + loss_pts_refine = self.loss_bbox_refine( + bbox_pred_refine / normalize_term, + bbox_gt_refine / normalize_term, + bbox_weights_refine, + avg_factor=num_total_samples_refine) + return loss_cls, loss_pts_init, loss_pts_refine + + def loss(self, + cls_scores, + pts_preds_init, + pts_preds_refine, + gt_bboxes, + gt_labels, + img_metas, + gt_bboxes_ignore=None): + featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] + assert len(featmap_sizes) == len(self.point_generators) + device = cls_scores[0].device + label_channels = self.cls_out_channels if self.use_sigmoid_cls else 1 + + # target for initial stage + center_list, valid_flag_list = self.get_points(featmap_sizes, + img_metas, device) + pts_coordinate_preds_init = self.offset_to_pts(center_list, + pts_preds_init) + if self.train_cfg.init.assigner['type'] == 'PointAssigner': + # Assign target for center list + candidate_list = center_list + else: + # transform center list to bbox list and + # assign target for bbox list + bbox_list = self.centers_to_bboxes(center_list) + candidate_list = bbox_list + cls_reg_targets_init = self.get_targets( + candidate_list, + valid_flag_list, + gt_bboxes, + img_metas, + gt_bboxes_ignore_list=gt_bboxes_ignore, + gt_labels_list=gt_labels, + stage='init', + label_channels=label_channels) + (*_, bbox_gt_list_init, candidate_list_init, bbox_weights_list_init, + num_total_pos_init, num_total_neg_init) = cls_reg_targets_init + num_total_samples_init = ( + num_total_pos_init + + num_total_neg_init if self.sampling else num_total_pos_init) + + # target for refinement stage + center_list, valid_flag_list = self.get_points(featmap_sizes, + img_metas, device) + pts_coordinate_preds_refine = self.offset_to_pts( + center_list, pts_preds_refine) + bbox_list = [] + for i_img, center in enumerate(center_list): + bbox = [] + for i_lvl in range(len(pts_preds_refine)): + bbox_preds_init = self.points2bbox( + pts_preds_init[i_lvl].detach()) + bbox_shift = bbox_preds_init * self.point_strides[i_lvl] + bbox_center = torch.cat( + [center[i_lvl][:, :2], center[i_lvl][:, :2]], dim=1) + bbox.append(bbox_center + + bbox_shift[i_img].permute(1, 2, 0).reshape(-1, 4)) + bbox_list.append(bbox) + cls_reg_targets_refine = self.get_targets( + bbox_list, + valid_flag_list, + gt_bboxes, + img_metas, + gt_bboxes_ignore_list=gt_bboxes_ignore, + gt_labels_list=gt_labels, + stage='refine', + label_channels=label_channels) + (labels_list, label_weights_list, bbox_gt_list_refine, + candidate_list_refine, bbox_weights_list_refine, num_total_pos_refine, + num_total_neg_refine) = cls_reg_targets_refine + num_total_samples_refine = ( + num_total_pos_refine + + num_total_neg_refine if self.sampling else num_total_pos_refine) + + # compute loss + losses_cls, losses_pts_init, losses_pts_refine = multi_apply( + self.loss_single, + cls_scores, + pts_coordinate_preds_init, + pts_coordinate_preds_refine, + labels_list, + label_weights_list, + bbox_gt_list_init, + bbox_weights_list_init, + bbox_gt_list_refine, + bbox_weights_list_refine, + self.point_strides, + num_total_samples_init=num_total_samples_init, + num_total_samples_refine=num_total_samples_refine) + loss_dict_all = { + 'loss_cls': losses_cls, + 'loss_pts_init': losses_pts_init, + 'loss_pts_refine': losses_pts_refine + } + return loss_dict_all + + def get_bboxes(self, + cls_scores, + pts_preds_init, + pts_preds_refine, + img_metas, + cfg=None, + rescale=False, + with_nms=True): + assert len(cls_scores) == len(pts_preds_refine) + device = cls_scores[0].device + bbox_preds_refine = [ + self.points2bbox(pts_pred_refine) + for pts_pred_refine in pts_preds_refine + ] + num_levels = len(cls_scores) + mlvl_points = [ + self.point_generators[i].grid_points(cls_scores[i].size()[-2:], + self.point_strides[i], device) + for i in range(num_levels) + ] + result_list = [] + for img_id in range(len(img_metas)): + cls_score_list = [ + cls_scores[i][img_id].detach() for i in range(num_levels) + ] + bbox_pred_list = [ + bbox_preds_refine[i][img_id].detach() + for i in range(num_levels) + ] + img_shape = img_metas[img_id]['img_shape'] + scale_factor = img_metas[img_id]['scale_factor'] + proposals = self._get_bboxes_single(cls_score_list, bbox_pred_list, + mlvl_points, img_shape, + scale_factor, cfg, rescale, + with_nms) + result_list.append(proposals) + return result_list + + def _get_bboxes_single(self, + cls_scores, + bbox_preds, + mlvl_points, + img_shape, + scale_factor, + cfg, + rescale=False, + with_nms=True): + cfg = self.test_cfg if cfg is None else cfg + assert len(cls_scores) == len(bbox_preds) == len(mlvl_points) + mlvl_bboxes = [] + mlvl_scores = [] + for i_lvl, (cls_score, bbox_pred, points) in enumerate( + zip(cls_scores, bbox_preds, mlvl_points)): + assert cls_score.size()[-2:] == bbox_pred.size()[-2:] + cls_score = cls_score.permute(1, 2, + 0).reshape(-1, self.cls_out_channels) + if self.use_sigmoid_cls: + scores = cls_score.sigmoid() + else: + scores = cls_score.softmax(-1) + bbox_pred = bbox_pred.permute(1, 2, 0).reshape(-1, 4) + nms_pre = cfg.get('nms_pre', -1) + if nms_pre > 0 and scores.shape[0] > nms_pre: + if self.use_sigmoid_cls: + max_scores, _ = scores.max(dim=1) + else: + # remind that we set FG labels to [0, num_class-1] + # since mmdet v2.0 + # BG cat_id: num_class + max_scores, _ = scores[:, :-1].max(dim=1) + _, topk_inds = max_scores.topk(nms_pre) + points = points[topk_inds, :] + bbox_pred = bbox_pred[topk_inds, :] + scores = scores[topk_inds, :] + bbox_pos_center = torch.cat([points[:, :2], points[:, :2]], dim=1) + bboxes = bbox_pred * self.point_strides[i_lvl] + bbox_pos_center + x1 = bboxes[:, 0].clamp(min=0, max=img_shape[1]) + y1 = bboxes[:, 1].clamp(min=0, max=img_shape[0]) + x2 = bboxes[:, 2].clamp(min=0, max=img_shape[1]) + y2 = bboxes[:, 3].clamp(min=0, max=img_shape[0]) + bboxes = torch.stack([x1, y1, x2, y2], dim=-1) + mlvl_bboxes.append(bboxes) + mlvl_scores.append(scores) + mlvl_bboxes = torch.cat(mlvl_bboxes) + if rescale: + mlvl_bboxes /= mlvl_bboxes.new_tensor(scale_factor) + mlvl_scores = torch.cat(mlvl_scores) + if self.use_sigmoid_cls: + # Add a dummy background class to the backend when using sigmoid + # remind that we set FG labels to [0, num_class-1] since mmdet v2.0 + # BG cat_id: num_class + padding = mlvl_scores.new_zeros(mlvl_scores.shape[0], 1) + mlvl_scores = torch.cat([mlvl_scores, padding], dim=1) + if with_nms: + det_bboxes, det_labels = multiclass_nms(mlvl_bboxes, mlvl_scores, + cfg.score_thr, cfg.nms, + cfg.max_per_img) + return det_bboxes, det_labels + else: + return mlvl_bboxes, mlvl_scores diff --git a/detection/mmdet/models/dense_heads/retina_head.py b/detection/mmdet/models/dense_heads/retina_head.py new file mode 100644 index 0000000..b12416f --- /dev/null +++ b/detection/mmdet/models/dense_heads/retina_head.py @@ -0,0 +1,114 @@ +import torch.nn as nn +from mmcv.cnn import ConvModule, bias_init_with_prob, normal_init + +from ..builder import HEADS +from .anchor_head import AnchorHead + + +@HEADS.register_module() +class RetinaHead(AnchorHead): + r"""An anchor-based head used in `RetinaNet + `_. + + The head contains two subnetworks. The first classifies anchor boxes and + the second regresses deltas for the anchors. + + Example: + >>> import torch + >>> self = RetinaHead(11, 7) + >>> x = torch.rand(1, 7, 32, 32) + >>> cls_score, bbox_pred = self.forward_single(x) + >>> # Each anchor predicts a score for each class except background + >>> cls_per_anchor = cls_score.shape[1] / self.num_anchors + >>> box_per_anchor = bbox_pred.shape[1] / self.num_anchors + >>> assert cls_per_anchor == (self.num_classes) + >>> assert box_per_anchor == 4 + """ + + def __init__(self, + num_classes, + in_channels, + stacked_convs=4, + conv_cfg=None, + norm_cfg=None, + anchor_generator=dict( + type='AnchorGenerator', + octave_base_scale=4, + scales_per_octave=3, + ratios=[0.5, 1.0, 2.0], + strides=[8, 16, 32, 64, 128]), + **kwargs): + self.stacked_convs = stacked_convs + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + super(RetinaHead, self).__init__( + num_classes, + in_channels, + anchor_generator=anchor_generator, + **kwargs) + + def _init_layers(self): + """Initialize layers of the head.""" + self.relu = nn.ReLU(inplace=True) + self.cls_convs = nn.ModuleList() + self.reg_convs = nn.ModuleList() + for i in range(self.stacked_convs): + chn = self.in_channels if i == 0 else self.feat_channels + self.cls_convs.append( + ConvModule( + chn, + self.feat_channels, + 3, + stride=1, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg)) + self.reg_convs.append( + ConvModule( + chn, + self.feat_channels, + 3, + stride=1, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg)) + self.retina_cls = nn.Conv2d( + self.feat_channels, + self.num_anchors * self.cls_out_channels, + 3, + padding=1) + self.retina_reg = nn.Conv2d( + self.feat_channels, self.num_anchors * 4, 3, padding=1) + + def init_weights(self): + """Initialize weights of the head.""" + for m in self.cls_convs: + normal_init(m.conv, std=0.01) + for m in self.reg_convs: + normal_init(m.conv, std=0.01) + bias_cls = bias_init_with_prob(0.01) + normal_init(self.retina_cls, std=0.01, bias=bias_cls) + normal_init(self.retina_reg, std=0.01) + + def forward_single(self, x): + """Forward feature of a single scale level. + + Args: + x (Tensor): Features of a single scale level. + + Returns: + tuple: + cls_score (Tensor): Cls scores for a single scale level + the channels number is num_anchors * num_classes. + bbox_pred (Tensor): Box energies / deltas for a single scale + level, the channels number is num_anchors * 4. + """ + cls_feat = x + reg_feat = x + for cls_conv in self.cls_convs: + cls_feat = cls_conv(cls_feat) + for reg_conv in self.reg_convs: + reg_feat = reg_conv(reg_feat) + cls_score = self.retina_cls(cls_feat) + bbox_pred = self.retina_reg(reg_feat) + return cls_score, bbox_pred diff --git a/detection/mmdet/models/dense_heads/retina_sepbn_head.py b/detection/mmdet/models/dense_heads/retina_sepbn_head.py new file mode 100644 index 0000000..6b8ce7f --- /dev/null +++ b/detection/mmdet/models/dense_heads/retina_sepbn_head.py @@ -0,0 +1,113 @@ +import torch.nn as nn +from mmcv.cnn import ConvModule, bias_init_with_prob, normal_init + +from ..builder import HEADS +from .anchor_head import AnchorHead + + +@HEADS.register_module() +class RetinaSepBNHead(AnchorHead): + """"RetinaHead with separate BN. + + In RetinaHead, conv/norm layers are shared across different FPN levels, + while in RetinaSepBNHead, conv layers are shared across different FPN + levels, but BN layers are separated. + """ + + def __init__(self, + num_classes, + num_ins, + in_channels, + stacked_convs=4, + conv_cfg=None, + norm_cfg=None, + **kwargs): + self.stacked_convs = stacked_convs + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.num_ins = num_ins + super(RetinaSepBNHead, self).__init__(num_classes, in_channels, + **kwargs) + + def _init_layers(self): + """Initialize layers of the head.""" + self.relu = nn.ReLU(inplace=True) + self.cls_convs = nn.ModuleList() + self.reg_convs = nn.ModuleList() + for i in range(self.num_ins): + cls_convs = nn.ModuleList() + reg_convs = nn.ModuleList() + for i in range(self.stacked_convs): + chn = self.in_channels if i == 0 else self.feat_channels + cls_convs.append( + ConvModule( + chn, + self.feat_channels, + 3, + stride=1, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg)) + reg_convs.append( + ConvModule( + chn, + self.feat_channels, + 3, + stride=1, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg)) + self.cls_convs.append(cls_convs) + self.reg_convs.append(reg_convs) + for i in range(self.stacked_convs): + for j in range(1, self.num_ins): + self.cls_convs[j][i].conv = self.cls_convs[0][i].conv + self.reg_convs[j][i].conv = self.reg_convs[0][i].conv + self.retina_cls = nn.Conv2d( + self.feat_channels, + self.num_anchors * self.cls_out_channels, + 3, + padding=1) + self.retina_reg = nn.Conv2d( + self.feat_channels, self.num_anchors * 4, 3, padding=1) + + def init_weights(self): + """Initialize weights of the head.""" + for m in self.cls_convs[0]: + normal_init(m.conv, std=0.01) + for m in self.reg_convs[0]: + normal_init(m.conv, std=0.01) + bias_cls = bias_init_with_prob(0.01) + normal_init(self.retina_cls, std=0.01, bias=bias_cls) + normal_init(self.retina_reg, std=0.01) + + def forward(self, feats): + """Forward features from the upstream network. + + Args: + feats (tuple[Tensor]): Features from the upstream network, each is + a 4D-tensor. + + Returns: + tuple: Usually a tuple of classification scores and bbox prediction + cls_scores (list[Tensor]): Classification scores for all scale + levels, each is a 4D-tensor, the channels number is + num_anchors * num_classes. + bbox_preds (list[Tensor]): Box energies / deltas for all scale + levels, each is a 4D-tensor, the channels number is + num_anchors * 4. + """ + cls_scores = [] + bbox_preds = [] + for i, x in enumerate(feats): + cls_feat = feats[i] + reg_feat = feats[i] + for cls_conv in self.cls_convs[i]: + cls_feat = cls_conv(cls_feat) + for reg_conv in self.reg_convs[i]: + reg_feat = reg_conv(reg_feat) + cls_score = self.retina_cls(cls_feat) + bbox_pred = self.retina_reg(reg_feat) + cls_scores.append(cls_score) + bbox_preds.append(bbox_pred) + return cls_scores, bbox_preds diff --git a/detection/mmdet/models/dense_heads/rpn_head.py b/detection/mmdet/models/dense_heads/rpn_head.py new file mode 100644 index 0000000..a888cb8 --- /dev/null +++ b/detection/mmdet/models/dense_heads/rpn_head.py @@ -0,0 +1,236 @@ +import copy +import warnings + +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv import ConfigDict +from mmcv.cnn import normal_init +from mmcv.ops import batched_nms + +from ..builder import HEADS +from .anchor_head import AnchorHead +from .rpn_test_mixin import RPNTestMixin + + +@HEADS.register_module() +class RPNHead(RPNTestMixin, AnchorHead): + """RPN head. + + Args: + in_channels (int): Number of channels in the input feature map. + """ # noqa: W605 + + def __init__(self, in_channels, **kwargs): + super(RPNHead, self).__init__(1, in_channels, **kwargs) + + def _init_layers(self): + """Initialize layers of the head.""" + self.rpn_conv = nn.Conv2d( + self.in_channels, self.feat_channels, 3, padding=1) + self.rpn_cls = nn.Conv2d(self.feat_channels, + self.num_anchors * self.cls_out_channels, 1) + self.rpn_reg = nn.Conv2d(self.feat_channels, self.num_anchors * 4, 1) + + def init_weights(self): + """Initialize weights of the head.""" + normal_init(self.rpn_conv, std=0.01) + normal_init(self.rpn_cls, std=0.01) + normal_init(self.rpn_reg, std=0.01) + + def forward_single(self, x): + """Forward feature map of a single scale level.""" + x = self.rpn_conv(x) + x = F.relu(x, inplace=True) + rpn_cls_score = self.rpn_cls(x) + rpn_bbox_pred = self.rpn_reg(x) + return rpn_cls_score, rpn_bbox_pred + + def loss(self, + cls_scores, + bbox_preds, + gt_bboxes, + img_metas, + gt_bboxes_ignore=None): + """Compute losses of the head. + + Args: + cls_scores (list[Tensor]): Box scores for each scale level + Has shape (N, num_anchors * num_classes, H, W) + bbox_preds (list[Tensor]): Box energies / deltas for each scale + level with shape (N, num_anchors * 4, H, W) + gt_bboxes (list[Tensor]): Ground truth bboxes for each image with + shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + gt_bboxes_ignore (None | list[Tensor]): specify which bounding + boxes can be ignored when computing the loss. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + losses = super(RPNHead, self).loss( + cls_scores, + bbox_preds, + gt_bboxes, + None, + img_metas, + gt_bboxes_ignore=gt_bboxes_ignore) + return dict( + loss_rpn_cls=losses['loss_cls'], loss_rpn_bbox=losses['loss_bbox']) + + def _get_bboxes(self, + cls_scores, + bbox_preds, + mlvl_anchors, + img_shapes, + scale_factors, + cfg, + rescale=False): + """Transform outputs for a single batch item into bbox predictions. + + Args: + cls_scores (list[Tensor]): Box scores for each scale level + Has shape (N, num_anchors * num_classes, H, W). + bbox_preds (list[Tensor]): Box energies / deltas for each scale + level with shape (N, num_anchors * 4, H, W). + mlvl_anchors (list[Tensor]): Box reference for each scale level + with shape (num_total_anchors, 4). + img_shapes (list[tuple[int]]): Shape of the input image, + (height, width, 3). + scale_factors (list[ndarray]): Scale factor of the image arange as + (w_scale, h_scale, w_scale, h_scale). + cfg (mmcv.Config): Test / postprocessing configuration, + if None, test_cfg would be used. + rescale (bool): If True, return boxes in original image space. + + Returns: + list[tuple[Tensor, Tensor]]: Each item in result_list is 2-tuple. + The first item is an (n, 5) tensor, where the first 4 columns + are bounding box positions (tl_x, tl_y, br_x, br_y) and the + 5-th column is a score between 0 and 1. The second item is a + (n,) tensor where each item is the predicted class labelof the + corresponding box. + """ + cfg = self.test_cfg if cfg is None else cfg + cfg = copy.deepcopy(cfg) + # bboxes from different level should be independent during NMS, + # level_ids are used as labels for batched NMS to separate them + level_ids = [] + mlvl_scores = [] + mlvl_bbox_preds = [] + mlvl_valid_anchors = [] + batch_size = cls_scores[0].shape[0] + nms_pre_tensor = torch.tensor( + cfg.nms_pre, device=cls_scores[0].device, dtype=torch.long) + for idx in range(len(cls_scores)): + rpn_cls_score = cls_scores[idx] + rpn_bbox_pred = bbox_preds[idx] + assert rpn_cls_score.size()[-2:] == rpn_bbox_pred.size()[-2:] + rpn_cls_score = rpn_cls_score.permute(0, 2, 3, 1) + if self.use_sigmoid_cls: + rpn_cls_score = rpn_cls_score.reshape(batch_size, -1) + scores = rpn_cls_score.sigmoid() + else: + rpn_cls_score = rpn_cls_score.reshape(batch_size, -1, 2) + # We set FG labels to [0, num_class-1] and BG label to + # num_class in RPN head since mmdet v2.5, which is unified to + # be consistent with other head since mmdet v2.0. In mmdet v2.0 + # to v2.4 we keep BG label as 0 and FG label as 1 in rpn head. + scores = rpn_cls_score.softmax(-1)[..., 0] + rpn_bbox_pred = rpn_bbox_pred.permute(0, 2, 3, 1).reshape( + batch_size, -1, 4) + anchors = mlvl_anchors[idx] + anchors = anchors.expand_as(rpn_bbox_pred) + if nms_pre_tensor > 0: + # sort is faster than topk + # _, topk_inds = scores.topk(cfg.nms_pre) + # keep topk op for dynamic k in onnx model + if torch.onnx.is_in_onnx_export(): + # sort op will be converted to TopK in onnx + # and k<=3480 in TensorRT + scores_shape = torch._shape_as_tensor(scores) + nms_pre = torch.where(scores_shape[1] < nms_pre_tensor, + scores_shape[1], nms_pre_tensor) + _, topk_inds = scores.topk(nms_pre) + batch_inds = torch.arange(batch_size).view( + -1, 1).expand_as(topk_inds) + scores = scores[batch_inds, topk_inds] + rpn_bbox_pred = rpn_bbox_pred[batch_inds, topk_inds, :] + anchors = anchors[batch_inds, topk_inds, :] + + elif scores.shape[-1] > cfg.nms_pre: + ranked_scores, rank_inds = scores.sort(descending=True) + topk_inds = rank_inds[:, :cfg.nms_pre] + scores = ranked_scores[:, :cfg.nms_pre] + batch_inds = torch.arange(batch_size).view( + -1, 1).expand_as(topk_inds) + rpn_bbox_pred = rpn_bbox_pred[batch_inds, topk_inds, :] + anchors = anchors[batch_inds, topk_inds, :] + + mlvl_scores.append(scores) + mlvl_bbox_preds.append(rpn_bbox_pred) + mlvl_valid_anchors.append(anchors) + level_ids.append( + scores.new_full(( + batch_size, + scores.size(1), + ), + idx, + dtype=torch.long)) + + batch_mlvl_scores = torch.cat(mlvl_scores, dim=1) + batch_mlvl_anchors = torch.cat(mlvl_valid_anchors, dim=1) + batch_mlvl_rpn_bbox_pred = torch.cat(mlvl_bbox_preds, dim=1) + batch_mlvl_proposals = self.bbox_coder.decode( + batch_mlvl_anchors, batch_mlvl_rpn_bbox_pred, max_shape=img_shapes) + batch_mlvl_ids = torch.cat(level_ids, dim=1) + + # deprecate arguments warning + if 'nms' not in cfg or 'max_num' in cfg or 'nms_thr' in cfg: + warnings.warn( + 'In rpn_proposal or test_cfg, ' + 'nms_thr has been moved to a dict named nms as ' + 'iou_threshold, max_num has been renamed as max_per_img, ' + 'name of original arguments and the way to specify ' + 'iou_threshold of NMS will be deprecated.') + if 'nms' not in cfg: + cfg.nms = ConfigDict(dict(type='nms', iou_threshold=cfg.nms_thr)) + if 'max_num' in cfg: + if 'max_per_img' in cfg: + assert cfg.max_num == cfg.max_per_img, f'You ' \ + f'set max_num and ' \ + f'max_per_img at the same time, but get {cfg.max_num} ' \ + f'and {cfg.max_per_img} respectively' \ + 'Please delete max_num which will be deprecated.' + else: + cfg.max_per_img = cfg.max_num + if 'nms_thr' in cfg: + assert cfg.nms.iou_threshold == cfg.nms_thr, f'You set' \ + f' iou_threshold in nms and ' \ + f'nms_thr at the same time, but get' \ + f' {cfg.nms.iou_threshold} and {cfg.nms_thr}' \ + f' respectively. Please delete the nms_thr ' \ + f'which will be deprecated.' + + result_list = [] + for (mlvl_proposals, mlvl_scores, + mlvl_ids) in zip(batch_mlvl_proposals, batch_mlvl_scores, + batch_mlvl_ids): + # Skip nonzero op while exporting to ONNX + if cfg.min_bbox_size > 0 and (not torch.onnx.is_in_onnx_export()): + w = mlvl_proposals[:, 2] - mlvl_proposals[:, 0] + h = mlvl_proposals[:, 3] - mlvl_proposals[:, 1] + valid_ind = torch.nonzero( + (w >= cfg.min_bbox_size) + & (h >= cfg.min_bbox_size), + as_tuple=False).squeeze() + if valid_ind.sum().item() != len(mlvl_proposals): + mlvl_proposals = mlvl_proposals[valid_ind, :] + mlvl_scores = mlvl_scores[valid_ind] + mlvl_ids = mlvl_ids[valid_ind] + + dets, keep = batched_nms(mlvl_proposals, mlvl_scores, mlvl_ids, + cfg.nms) + result_list.append(dets[:cfg.max_per_img]) + return result_list diff --git a/detection/mmdet/models/dense_heads/rpn_test_mixin.py b/detection/mmdet/models/dense_heads/rpn_test_mixin.py new file mode 100644 index 0000000..4ce5c66 --- /dev/null +++ b/detection/mmdet/models/dense_heads/rpn_test_mixin.py @@ -0,0 +1,59 @@ +import sys + +from mmdet.core import merge_aug_proposals + +if sys.version_info >= (3, 7): + from mmdet.utils.contextmanagers import completed + + +class RPNTestMixin(object): + """Test methods of RPN.""" + + if sys.version_info >= (3, 7): + + async def async_simple_test_rpn(self, x, img_metas): + sleep_interval = self.test_cfg.pop('async_sleep_interval', 0.025) + async with completed( + __name__, 'rpn_head_forward', + sleep_interval=sleep_interval): + rpn_outs = self(x) + + proposal_list = self.get_bboxes(*rpn_outs, img_metas) + return proposal_list + + def simple_test_rpn(self, x, img_metas): + """Test without augmentation. + + Args: + x (tuple[Tensor]): Features from the upstream network, each is + a 4D-tensor. + img_metas (list[dict]): Meta info of each image. + + Returns: + list[Tensor]: Proposals of each image. + """ + rpn_outs = self(x) + proposal_list = self.get_bboxes(*rpn_outs, img_metas) + return proposal_list + + def aug_test_rpn(self, feats, img_metas): + samples_per_gpu = len(img_metas[0]) + aug_proposals = [[] for _ in range(samples_per_gpu)] + for x, img_meta in zip(feats, img_metas): + proposal_list = self.simple_test_rpn(x, img_meta) + for i, proposals in enumerate(proposal_list): + aug_proposals[i].append(proposals) + # reorganize the order of 'img_metas' to match the dimensions + # of 'aug_proposals' + aug_img_metas = [] + for i in range(samples_per_gpu): + aug_img_meta = [] + for j in range(len(img_metas)): + aug_img_meta.append(img_metas[j][i]) + aug_img_metas.append(aug_img_meta) + # after merging, proposals will be rescaled to the original image size + merged_proposals = [ + merge_aug_proposals(proposals, aug_img_meta, self.test_cfg) + for proposals, aug_img_meta in zip(aug_proposals, aug_img_metas) + ] + return merged_proposals diff --git a/detection/mmdet/models/dense_heads/sabl_retina_head.py b/detection/mmdet/models/dense_heads/sabl_retina_head.py new file mode 100644 index 0000000..4211622 --- /dev/null +++ b/detection/mmdet/models/dense_heads/sabl_retina_head.py @@ -0,0 +1,621 @@ +import numpy as np +import torch +import torch.nn as nn +from mmcv.cnn import ConvModule, bias_init_with_prob, normal_init +from mmcv.runner import force_fp32 + +from mmdet.core import (build_anchor_generator, build_assigner, + build_bbox_coder, build_sampler, images_to_levels, + multi_apply, multiclass_nms, unmap) +from ..builder import HEADS, build_loss +from .base_dense_head import BaseDenseHead +from .guided_anchor_head import GuidedAnchorHead + + +@HEADS.register_module() +class SABLRetinaHead(BaseDenseHead): + """Side-Aware Boundary Localization (SABL) for RetinaNet. + + The anchor generation, assigning and sampling in SABLRetinaHead + are the same as GuidedAnchorHead for guided anchoring. + + Please refer to https://arxiv.org/abs/1912.04260 for more details. + + Args: + num_classes (int): Number of classes. + in_channels (int): Number of channels in the input feature map. + stacked_convs (int): Number of Convs for classification \ + and regression branches. Defaults to 4. + feat_channels (int): Number of hidden channels. \ + Defaults to 256. + approx_anchor_generator (dict): Config dict for approx generator. + square_anchor_generator (dict): Config dict for square generator. + conv_cfg (dict): Config dict for ConvModule. Defaults to None. + norm_cfg (dict): Config dict for Norm Layer. Defaults to None. + bbox_coder (dict): Config dict for bbox coder. + reg_decoded_bbox (bool): If true, the regression loss would be + applied directly on decoded bounding boxes, converting both + the predicted boxes and regression targets to absolute + coordinates format. Default False. It should be `True` when + using `IoULoss`, `GIoULoss`, or `DIoULoss` in the bbox head. + train_cfg (dict): Training config of SABLRetinaHead. + test_cfg (dict): Testing config of SABLRetinaHead. + loss_cls (dict): Config of classification loss. + loss_bbox_cls (dict): Config of classification loss for bbox branch. + loss_bbox_reg (dict): Config of regression loss for bbox branch. + """ + + def __init__(self, + num_classes, + in_channels, + stacked_convs=4, + feat_channels=256, + approx_anchor_generator=dict( + type='AnchorGenerator', + octave_base_scale=4, + scales_per_octave=3, + ratios=[0.5, 1.0, 2.0], + strides=[8, 16, 32, 64, 128]), + square_anchor_generator=dict( + type='AnchorGenerator', + ratios=[1.0], + scales=[4], + strides=[8, 16, 32, 64, 128]), + conv_cfg=None, + norm_cfg=None, + bbox_coder=dict( + type='BucketingBBoxCoder', + num_buckets=14, + scale_factor=3.0), + reg_decoded_bbox=False, + train_cfg=None, + test_cfg=None, + loss_cls=dict( + type='FocalLoss', + use_sigmoid=True, + gamma=2.0, + alpha=0.25, + loss_weight=1.0), + loss_bbox_cls=dict( + type='CrossEntropyLoss', + use_sigmoid=True, + loss_weight=1.5), + loss_bbox_reg=dict( + type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.5)): + super(SABLRetinaHead, self).__init__() + self.in_channels = in_channels + self.num_classes = num_classes + self.feat_channels = feat_channels + self.num_buckets = bbox_coder['num_buckets'] + self.side_num = int(np.ceil(self.num_buckets / 2)) + + assert (approx_anchor_generator['octave_base_scale'] == + square_anchor_generator['scales'][0]) + assert (approx_anchor_generator['strides'] == + square_anchor_generator['strides']) + + self.approx_anchor_generator = build_anchor_generator( + approx_anchor_generator) + self.square_anchor_generator = build_anchor_generator( + square_anchor_generator) + self.approxs_per_octave = ( + self.approx_anchor_generator.num_base_anchors[0]) + + # one anchor per location + self.num_anchors = 1 + self.stacked_convs = stacked_convs + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + + self.reg_decoded_bbox = reg_decoded_bbox + + self.use_sigmoid_cls = loss_cls.get('use_sigmoid', False) + self.sampling = loss_cls['type'] not in [ + 'FocalLoss', 'GHMC', 'QualityFocalLoss' + ] + if self.use_sigmoid_cls: + self.cls_out_channels = num_classes + else: + self.cls_out_channels = num_classes + 1 + + self.bbox_coder = build_bbox_coder(bbox_coder) + self.loss_cls = build_loss(loss_cls) + self.loss_bbox_cls = build_loss(loss_bbox_cls) + self.loss_bbox_reg = build_loss(loss_bbox_reg) + + self.train_cfg = train_cfg + self.test_cfg = test_cfg + + if self.train_cfg: + self.assigner = build_assigner(self.train_cfg.assigner) + # use PseudoSampler when sampling is False + if self.sampling and hasattr(self.train_cfg, 'sampler'): + sampler_cfg = self.train_cfg.sampler + else: + sampler_cfg = dict(type='PseudoSampler') + self.sampler = build_sampler(sampler_cfg, context=self) + + self.fp16_enabled = False + self._init_layers() + + def _init_layers(self): + self.relu = nn.ReLU(inplace=True) + self.cls_convs = nn.ModuleList() + self.reg_convs = nn.ModuleList() + for i in range(self.stacked_convs): + chn = self.in_channels if i == 0 else self.feat_channels + self.cls_convs.append( + ConvModule( + chn, + self.feat_channels, + 3, + stride=1, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg)) + self.reg_convs.append( + ConvModule( + chn, + self.feat_channels, + 3, + stride=1, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg)) + self.retina_cls = nn.Conv2d( + self.feat_channels, self.cls_out_channels, 3, padding=1) + self.retina_bbox_reg = nn.Conv2d( + self.feat_channels, self.side_num * 4, 3, padding=1) + self.retina_bbox_cls = nn.Conv2d( + self.feat_channels, self.side_num * 4, 3, padding=1) + + def init_weights(self): + for m in self.cls_convs: + normal_init(m.conv, std=0.01) + for m in self.reg_convs: + normal_init(m.conv, std=0.01) + bias_cls = bias_init_with_prob(0.01) + normal_init(self.retina_cls, std=0.01, bias=bias_cls) + normal_init(self.retina_bbox_reg, std=0.01) + normal_init(self.retina_bbox_cls, std=0.01) + + def forward_single(self, x): + cls_feat = x + reg_feat = x + for cls_conv in self.cls_convs: + cls_feat = cls_conv(cls_feat) + for reg_conv in self.reg_convs: + reg_feat = reg_conv(reg_feat) + cls_score = self.retina_cls(cls_feat) + bbox_cls_pred = self.retina_bbox_cls(reg_feat) + bbox_reg_pred = self.retina_bbox_reg(reg_feat) + bbox_pred = (bbox_cls_pred, bbox_reg_pred) + return cls_score, bbox_pred + + def forward(self, feats): + return multi_apply(self.forward_single, feats) + + def get_anchors(self, featmap_sizes, img_metas, device='cuda'): + """Get squares according to feature map sizes and guided anchors. + + Args: + featmap_sizes (list[tuple]): Multi-level feature map sizes. + img_metas (list[dict]): Image meta info. + device (torch.device | str): device for returned tensors + + Returns: + tuple: square approxs of each image + """ + num_imgs = len(img_metas) + + # since feature map sizes of all images are the same, we only compute + # squares for one time + multi_level_squares = self.square_anchor_generator.grid_anchors( + featmap_sizes, device=device) + squares_list = [multi_level_squares for _ in range(num_imgs)] + + return squares_list + + def get_target(self, + approx_list, + inside_flag_list, + square_list, + gt_bboxes_list, + img_metas, + gt_bboxes_ignore_list=None, + gt_labels_list=None, + label_channels=None, + sampling=True, + unmap_outputs=True): + """Compute bucketing targets. + Args: + approx_list (list[list]): Multi level approxs of each image. + inside_flag_list (list[list]): Multi level inside flags of each + image. + square_list (list[list]): Multi level squares of each image. + gt_bboxes_list (list[Tensor]): Ground truth bboxes of each image. + img_metas (list[dict]): Meta info of each image. + gt_bboxes_ignore_list (list[Tensor]): ignore list of gt bboxes. + gt_bboxes_list (list[Tensor]): Gt bboxes of each image. + label_channels (int): Channel of label. + sampling (bool): Sample Anchors or not. + unmap_outputs (bool): unmap outputs or not. + + Returns: + tuple: Returns a tuple containing learning targets. + + - labels_list (list[Tensor]): Labels of each level. + - label_weights_list (list[Tensor]): Label weights of each \ + level. + - bbox_cls_targets_list (list[Tensor]): BBox cls targets of \ + each level. + - bbox_cls_weights_list (list[Tensor]): BBox cls weights of \ + each level. + - bbox_reg_targets_list (list[Tensor]): BBox reg targets of \ + each level. + - bbox_reg_weights_list (list[Tensor]): BBox reg weights of \ + each level. + - num_total_pos (int): Number of positive samples in all \ + images. + - num_total_neg (int): Number of negative samples in all \ + images. + """ + num_imgs = len(img_metas) + assert len(approx_list) == len(inside_flag_list) == len( + square_list) == num_imgs + # anchor number of multi levels + num_level_squares = [squares.size(0) for squares in square_list[0]] + # concat all level anchors and flags to a single tensor + inside_flag_flat_list = [] + approx_flat_list = [] + square_flat_list = [] + for i in range(num_imgs): + assert len(square_list[i]) == len(inside_flag_list[i]) + inside_flag_flat_list.append(torch.cat(inside_flag_list[i])) + approx_flat_list.append(torch.cat(approx_list[i])) + square_flat_list.append(torch.cat(square_list[i])) + + # compute targets for each image + if gt_bboxes_ignore_list is None: + gt_bboxes_ignore_list = [None for _ in range(num_imgs)] + if gt_labels_list is None: + gt_labels_list = [None for _ in range(num_imgs)] + (all_labels, all_label_weights, all_bbox_cls_targets, + all_bbox_cls_weights, all_bbox_reg_targets, all_bbox_reg_weights, + pos_inds_list, neg_inds_list) = multi_apply( + self._get_target_single, + approx_flat_list, + inside_flag_flat_list, + square_flat_list, + gt_bboxes_list, + gt_bboxes_ignore_list, + gt_labels_list, + img_metas, + label_channels=label_channels, + sampling=sampling, + unmap_outputs=unmap_outputs) + # no valid anchors + if any([labels is None for labels in all_labels]): + return None + # sampled anchors of all images + num_total_pos = sum([max(inds.numel(), 1) for inds in pos_inds_list]) + num_total_neg = sum([max(inds.numel(), 1) for inds in neg_inds_list]) + # split targets to a list w.r.t. multiple levels + labels_list = images_to_levels(all_labels, num_level_squares) + label_weights_list = images_to_levels(all_label_weights, + num_level_squares) + bbox_cls_targets_list = images_to_levels(all_bbox_cls_targets, + num_level_squares) + bbox_cls_weights_list = images_to_levels(all_bbox_cls_weights, + num_level_squares) + bbox_reg_targets_list = images_to_levels(all_bbox_reg_targets, + num_level_squares) + bbox_reg_weights_list = images_to_levels(all_bbox_reg_weights, + num_level_squares) + return (labels_list, label_weights_list, bbox_cls_targets_list, + bbox_cls_weights_list, bbox_reg_targets_list, + bbox_reg_weights_list, num_total_pos, num_total_neg) + + def _get_target_single(self, + flat_approxs, + inside_flags, + flat_squares, + gt_bboxes, + gt_bboxes_ignore, + gt_labels, + img_meta, + label_channels=None, + sampling=True, + unmap_outputs=True): + """Compute regression and classification targets for anchors in a + single image. + + Args: + flat_approxs (Tensor): flat approxs of a single image, + shape (n, 4) + inside_flags (Tensor): inside flags of a single image, + shape (n, ). + flat_squares (Tensor): flat squares of a single image, + shape (approxs_per_octave * n, 4) + gt_bboxes (Tensor): Ground truth bboxes of a single image, \ + shape (num_gts, 4). + gt_bboxes_ignore (Tensor): Ground truth bboxes to be + ignored, shape (num_ignored_gts, 4). + gt_labels (Tensor): Ground truth labels of each box, + shape (num_gts,). + img_meta (dict): Meta info of the image. + label_channels (int): Channel of label. + sampling (bool): Sample Anchors or not. + unmap_outputs (bool): unmap outputs or not. + + Returns: + tuple: + + - labels_list (Tensor): Labels in a single image + - label_weights (Tensor): Label weights in a single image + - bbox_cls_targets (Tensor): BBox cls targets in a single image + - bbox_cls_weights (Tensor): BBox cls weights in a single image + - bbox_reg_targets (Tensor): BBox reg targets in a single image + - bbox_reg_weights (Tensor): BBox reg weights in a single image + - num_total_pos (int): Number of positive samples \ + in a single image + - num_total_neg (int): Number of negative samples \ + in a single image + """ + if not inside_flags.any(): + return (None, ) * 8 + # assign gt and sample anchors + expand_inside_flags = inside_flags[:, None].expand( + -1, self.approxs_per_octave).reshape(-1) + approxs = flat_approxs[expand_inside_flags, :] + squares = flat_squares[inside_flags, :] + + assign_result = self.assigner.assign(approxs, squares, + self.approxs_per_octave, + gt_bboxes, gt_bboxes_ignore) + sampling_result = self.sampler.sample(assign_result, squares, + gt_bboxes) + + num_valid_squares = squares.shape[0] + bbox_cls_targets = squares.new_zeros( + (num_valid_squares, self.side_num * 4)) + bbox_cls_weights = squares.new_zeros( + (num_valid_squares, self.side_num * 4)) + bbox_reg_targets = squares.new_zeros( + (num_valid_squares, self.side_num * 4)) + bbox_reg_weights = squares.new_zeros( + (num_valid_squares, self.side_num * 4)) + labels = squares.new_full((num_valid_squares, ), + self.num_classes, + dtype=torch.long) + label_weights = squares.new_zeros(num_valid_squares, dtype=torch.float) + + pos_inds = sampling_result.pos_inds + neg_inds = sampling_result.neg_inds + if len(pos_inds) > 0: + (pos_bbox_reg_targets, pos_bbox_reg_weights, pos_bbox_cls_targets, + pos_bbox_cls_weights) = self.bbox_coder.encode( + sampling_result.pos_bboxes, sampling_result.pos_gt_bboxes) + + bbox_cls_targets[pos_inds, :] = pos_bbox_cls_targets + bbox_reg_targets[pos_inds, :] = pos_bbox_reg_targets + bbox_cls_weights[pos_inds, :] = pos_bbox_cls_weights + bbox_reg_weights[pos_inds, :] = pos_bbox_reg_weights + if gt_labels is None: + # Only rpn gives gt_labels as None + # Foreground is the first class + labels[pos_inds] = 0 + else: + labels[pos_inds] = gt_labels[ + sampling_result.pos_assigned_gt_inds] + if self.train_cfg.pos_weight <= 0: + label_weights[pos_inds] = 1.0 + else: + label_weights[pos_inds] = self.train_cfg.pos_weight + if len(neg_inds) > 0: + label_weights[neg_inds] = 1.0 + + # map up to original set of anchors + if unmap_outputs: + num_total_anchors = flat_squares.size(0) + labels = unmap( + labels, num_total_anchors, inside_flags, fill=self.num_classes) + label_weights = unmap(label_weights, num_total_anchors, + inside_flags) + bbox_cls_targets = unmap(bbox_cls_targets, num_total_anchors, + inside_flags) + bbox_cls_weights = unmap(bbox_cls_weights, num_total_anchors, + inside_flags) + bbox_reg_targets = unmap(bbox_reg_targets, num_total_anchors, + inside_flags) + bbox_reg_weights = unmap(bbox_reg_weights, num_total_anchors, + inside_flags) + return (labels, label_weights, bbox_cls_targets, bbox_cls_weights, + bbox_reg_targets, bbox_reg_weights, pos_inds, neg_inds) + + def loss_single(self, cls_score, bbox_pred, labels, label_weights, + bbox_cls_targets, bbox_cls_weights, bbox_reg_targets, + bbox_reg_weights, num_total_samples): + # classification loss + labels = labels.reshape(-1) + label_weights = label_weights.reshape(-1) + cls_score = cls_score.permute(0, 2, 3, + 1).reshape(-1, self.cls_out_channels) + loss_cls = self.loss_cls( + cls_score, labels, label_weights, avg_factor=num_total_samples) + # regression loss + bbox_cls_targets = bbox_cls_targets.reshape(-1, self.side_num * 4) + bbox_cls_weights = bbox_cls_weights.reshape(-1, self.side_num * 4) + bbox_reg_targets = bbox_reg_targets.reshape(-1, self.side_num * 4) + bbox_reg_weights = bbox_reg_weights.reshape(-1, self.side_num * 4) + (bbox_cls_pred, bbox_reg_pred) = bbox_pred + bbox_cls_pred = bbox_cls_pred.permute(0, 2, 3, 1).reshape( + -1, self.side_num * 4) + bbox_reg_pred = bbox_reg_pred.permute(0, 2, 3, 1).reshape( + -1, self.side_num * 4) + loss_bbox_cls = self.loss_bbox_cls( + bbox_cls_pred, + bbox_cls_targets.long(), + bbox_cls_weights, + avg_factor=num_total_samples * 4 * self.side_num) + loss_bbox_reg = self.loss_bbox_reg( + bbox_reg_pred, + bbox_reg_targets, + bbox_reg_weights, + avg_factor=num_total_samples * 4 * self.bbox_coder.offset_topk) + return loss_cls, loss_bbox_cls, loss_bbox_reg + + @force_fp32(apply_to=('cls_scores', 'bbox_preds')) + def loss(self, + cls_scores, + bbox_preds, + gt_bboxes, + gt_labels, + img_metas, + gt_bboxes_ignore=None): + featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] + assert len(featmap_sizes) == self.approx_anchor_generator.num_levels + + device = cls_scores[0].device + + # get sampled approxes + approxs_list, inside_flag_list = GuidedAnchorHead.get_sampled_approxs( + self, featmap_sizes, img_metas, device=device) + + square_list = self.get_anchors(featmap_sizes, img_metas, device=device) + + label_channels = self.cls_out_channels if self.use_sigmoid_cls else 1 + + cls_reg_targets = self.get_target( + approxs_list, + inside_flag_list, + square_list, + gt_bboxes, + img_metas, + gt_bboxes_ignore_list=gt_bboxes_ignore, + gt_labels_list=gt_labels, + label_channels=label_channels, + sampling=self.sampling) + if cls_reg_targets is None: + return None + (labels_list, label_weights_list, bbox_cls_targets_list, + bbox_cls_weights_list, bbox_reg_targets_list, bbox_reg_weights_list, + num_total_pos, num_total_neg) = cls_reg_targets + num_total_samples = ( + num_total_pos + num_total_neg if self.sampling else num_total_pos) + losses_cls, losses_bbox_cls, losses_bbox_reg = multi_apply( + self.loss_single, + cls_scores, + bbox_preds, + labels_list, + label_weights_list, + bbox_cls_targets_list, + bbox_cls_weights_list, + bbox_reg_targets_list, + bbox_reg_weights_list, + num_total_samples=num_total_samples) + return dict( + loss_cls=losses_cls, + loss_bbox_cls=losses_bbox_cls, + loss_bbox_reg=losses_bbox_reg) + + @force_fp32(apply_to=('cls_scores', 'bbox_preds')) + def get_bboxes(self, + cls_scores, + bbox_preds, + img_metas, + cfg=None, + rescale=False): + assert len(cls_scores) == len(bbox_preds) + num_levels = len(cls_scores) + featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] + + device = cls_scores[0].device + mlvl_anchors = self.get_anchors( + featmap_sizes, img_metas, device=device) + result_list = [] + for img_id in range(len(img_metas)): + cls_score_list = [ + cls_scores[i][img_id].detach() for i in range(num_levels) + ] + bbox_cls_pred_list = [ + bbox_preds[i][0][img_id].detach() for i in range(num_levels) + ] + bbox_reg_pred_list = [ + bbox_preds[i][1][img_id].detach() for i in range(num_levels) + ] + img_shape = img_metas[img_id]['img_shape'] + scale_factor = img_metas[img_id]['scale_factor'] + proposals = self.get_bboxes_single(cls_score_list, + bbox_cls_pred_list, + bbox_reg_pred_list, + mlvl_anchors[img_id], img_shape, + scale_factor, cfg, rescale) + result_list.append(proposals) + return result_list + + def get_bboxes_single(self, + cls_scores, + bbox_cls_preds, + bbox_reg_preds, + mlvl_anchors, + img_shape, + scale_factor, + cfg, + rescale=False): + cfg = self.test_cfg if cfg is None else cfg + mlvl_bboxes = [] + mlvl_scores = [] + mlvl_confids = [] + assert len(cls_scores) == len(bbox_cls_preds) == len( + bbox_reg_preds) == len(mlvl_anchors) + for cls_score, bbox_cls_pred, bbox_reg_pred, anchors in zip( + cls_scores, bbox_cls_preds, bbox_reg_preds, mlvl_anchors): + assert cls_score.size()[-2:] == bbox_cls_pred.size( + )[-2:] == bbox_reg_pred.size()[-2::] + cls_score = cls_score.permute(1, 2, + 0).reshape(-1, self.cls_out_channels) + if self.use_sigmoid_cls: + scores = cls_score.sigmoid() + else: + scores = cls_score.softmax(-1) + bbox_cls_pred = bbox_cls_pred.permute(1, 2, 0).reshape( + -1, self.side_num * 4) + bbox_reg_pred = bbox_reg_pred.permute(1, 2, 0).reshape( + -1, self.side_num * 4) + nms_pre = cfg.get('nms_pre', -1) + if nms_pre > 0 and scores.shape[0] > nms_pre: + if self.use_sigmoid_cls: + max_scores, _ = scores.max(dim=1) + else: + max_scores, _ = scores[:, :-1].max(dim=1) + _, topk_inds = max_scores.topk(nms_pre) + anchors = anchors[topk_inds, :] + bbox_cls_pred = bbox_cls_pred[topk_inds, :] + bbox_reg_pred = bbox_reg_pred[topk_inds, :] + scores = scores[topk_inds, :] + bbox_preds = [ + bbox_cls_pred.contiguous(), + bbox_reg_pred.contiguous() + ] + bboxes, confids = self.bbox_coder.decode( + anchors.contiguous(), bbox_preds, max_shape=img_shape) + mlvl_bboxes.append(bboxes) + mlvl_scores.append(scores) + mlvl_confids.append(confids) + mlvl_bboxes = torch.cat(mlvl_bboxes) + if rescale: + mlvl_bboxes /= mlvl_bboxes.new_tensor(scale_factor) + mlvl_scores = torch.cat(mlvl_scores) + mlvl_confids = torch.cat(mlvl_confids) + if self.use_sigmoid_cls: + padding = mlvl_scores.new_zeros(mlvl_scores.shape[0], 1) + mlvl_scores = torch.cat([mlvl_scores, padding], dim=1) + det_bboxes, det_labels = multiclass_nms( + mlvl_bboxes, + mlvl_scores, + cfg.score_thr, + cfg.nms, + cfg.max_per_img, + score_factors=mlvl_confids) + return det_bboxes, det_labels diff --git a/detection/mmdet/models/dense_heads/ssd_head.py b/detection/mmdet/models/dense_heads/ssd_head.py new file mode 100644 index 0000000..145622b --- /dev/null +++ b/detection/mmdet/models/dense_heads/ssd_head.py @@ -0,0 +1,265 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import xavier_init +from mmcv.runner import force_fp32 + +from mmdet.core import (build_anchor_generator, build_assigner, + build_bbox_coder, build_sampler, multi_apply) +from ..builder import HEADS +from ..losses import smooth_l1_loss +from .anchor_head import AnchorHead + + +# TODO: add loss evaluator for SSD +@HEADS.register_module() +class SSDHead(AnchorHead): + """SSD head used in https://arxiv.org/abs/1512.02325. + + Args: + num_classes (int): Number of categories excluding the background + category. + in_channels (int): Number of channels in the input feature map. + anchor_generator (dict): Config dict for anchor generator + bbox_coder (dict): Config of bounding box coder. + reg_decoded_bbox (bool): If true, the regression loss would be + applied directly on decoded bounding boxes, converting both + the predicted boxes and regression targets to absolute + coordinates format. Default False. It should be `True` when + using `IoULoss`, `GIoULoss`, or `DIoULoss` in the bbox head. + train_cfg (dict): Training config of anchor head. + test_cfg (dict): Testing config of anchor head. + """ # noqa: W605 + + def __init__(self, + num_classes=80, + in_channels=(512, 1024, 512, 256, 256, 256), + anchor_generator=dict( + type='SSDAnchorGenerator', + scale_major=False, + input_size=300, + strides=[8, 16, 32, 64, 100, 300], + ratios=([2], [2, 3], [2, 3], [2, 3], [2], [2]), + basesize_ratio_range=(0.1, 0.9)), + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + clip_border=True, + target_means=[.0, .0, .0, .0], + target_stds=[1.0, 1.0, 1.0, 1.0], + ), + reg_decoded_bbox=False, + train_cfg=None, + test_cfg=None): + super(AnchorHead, self).__init__() + self.num_classes = num_classes + self.in_channels = in_channels + self.cls_out_channels = num_classes + 1 # add background class + self.anchor_generator = build_anchor_generator(anchor_generator) + num_anchors = self.anchor_generator.num_base_anchors + + reg_convs = [] + cls_convs = [] + for i in range(len(in_channels)): + reg_convs.append( + nn.Conv2d( + in_channels[i], + num_anchors[i] * 4, + kernel_size=3, + padding=1)) + cls_convs.append( + nn.Conv2d( + in_channels[i], + num_anchors[i] * (num_classes + 1), + kernel_size=3, + padding=1)) + self.reg_convs = nn.ModuleList(reg_convs) + self.cls_convs = nn.ModuleList(cls_convs) + + self.bbox_coder = build_bbox_coder(bbox_coder) + self.reg_decoded_bbox = reg_decoded_bbox + self.use_sigmoid_cls = False + self.cls_focal_loss = False + self.train_cfg = train_cfg + self.test_cfg = test_cfg + # set sampling=False for archor_target + self.sampling = False + if self.train_cfg: + self.assigner = build_assigner(self.train_cfg.assigner) + # SSD sampling=False so use PseudoSampler + sampler_cfg = dict(type='PseudoSampler') + self.sampler = build_sampler(sampler_cfg, context=self) + self.fp16_enabled = False + + def init_weights(self): + """Initialize weights of the head.""" + for m in self.modules(): + if isinstance(m, nn.Conv2d): + xavier_init(m, distribution='uniform', bias=0) + + def forward(self, feats): + """Forward features from the upstream network. + + Args: + feats (tuple[Tensor]): Features from the upstream network, each is + a 4D-tensor. + + Returns: + tuple: + cls_scores (list[Tensor]): Classification scores for all scale + levels, each is a 4D-tensor, the channels number is + num_anchors * num_classes. + bbox_preds (list[Tensor]): Box energies / deltas for all scale + levels, each is a 4D-tensor, the channels number is + num_anchors * 4. + """ + cls_scores = [] + bbox_preds = [] + for feat, reg_conv, cls_conv in zip(feats, self.reg_convs, + self.cls_convs): + cls_scores.append(cls_conv(feat)) + bbox_preds.append(reg_conv(feat)) + return cls_scores, bbox_preds + + def loss_single(self, cls_score, bbox_pred, anchor, labels, label_weights, + bbox_targets, bbox_weights, num_total_samples): + """Compute loss of a single image. + + Args: + cls_score (Tensor): Box scores for eachimage + Has shape (num_total_anchors, num_classes). + bbox_pred (Tensor): Box energies / deltas for each image + level with shape (num_total_anchors, 4). + anchors (Tensor): Box reference for each scale level with shape + (num_total_anchors, 4). + labels (Tensor): Labels of each anchors with shape + (num_total_anchors,). + label_weights (Tensor): Label weights of each anchor with shape + (num_total_anchors,) + bbox_targets (Tensor): BBox regression targets of each anchor wight + shape (num_total_anchors, 4). + bbox_weights (Tensor): BBox regression loss weights of each anchor + with shape (num_total_anchors, 4). + num_total_samples (int): If sampling, num total samples equal to + the number of total anchors; Otherwise, it is the number of + positive anchors. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + + loss_cls_all = F.cross_entropy( + cls_score, labels, reduction='none') * label_weights + # FG cat_id: [0, num_classes -1], BG cat_id: num_classes + pos_inds = ((labels >= 0) & + (labels < self.num_classes)).nonzero().reshape(-1) + neg_inds = (labels == self.num_classes).nonzero().view(-1) + + num_pos_samples = pos_inds.size(0) + num_neg_samples = self.train_cfg.neg_pos_ratio * num_pos_samples + if num_neg_samples > neg_inds.size(0): + num_neg_samples = neg_inds.size(0) + topk_loss_cls_neg, _ = loss_cls_all[neg_inds].topk(num_neg_samples) + loss_cls_pos = loss_cls_all[pos_inds].sum() + loss_cls_neg = topk_loss_cls_neg.sum() + loss_cls = (loss_cls_pos + loss_cls_neg) / num_total_samples + + if self.reg_decoded_bbox: + # When the regression loss (e.g. `IouLoss`, `GIouLoss`) + # is applied directly on the decoded bounding boxes, it + # decodes the already encoded coordinates to absolute format. + bbox_pred = self.bbox_coder.decode(anchor, bbox_pred) + + loss_bbox = smooth_l1_loss( + bbox_pred, + bbox_targets, + bbox_weights, + beta=self.train_cfg.smoothl1_beta, + avg_factor=num_total_samples) + return loss_cls[None], loss_bbox + + @force_fp32(apply_to=('cls_scores', 'bbox_preds')) + def loss(self, + cls_scores, + bbox_preds, + gt_bboxes, + gt_labels, + img_metas, + gt_bboxes_ignore=None): + """Compute losses of the head. + + Args: + cls_scores (list[Tensor]): Box scores for each scale level + Has shape (N, num_anchors * num_classes, H, W) + bbox_preds (list[Tensor]): Box energies / deltas for each scale + level with shape (N, num_anchors * 4, H, W) + gt_bboxes (list[Tensor]): each item are the truth boxes for each + image in [tl_x, tl_y, br_x, br_y] format. + gt_labels (list[Tensor]): class indices corresponding to each box + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + gt_bboxes_ignore (None | list[Tensor]): specify which bounding + boxes can be ignored when computing the loss. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] + assert len(featmap_sizes) == self.anchor_generator.num_levels + + device = cls_scores[0].device + + anchor_list, valid_flag_list = self.get_anchors( + featmap_sizes, img_metas, device=device) + cls_reg_targets = self.get_targets( + anchor_list, + valid_flag_list, + gt_bboxes, + img_metas, + gt_bboxes_ignore_list=gt_bboxes_ignore, + gt_labels_list=gt_labels, + label_channels=1, + unmap_outputs=False) + if cls_reg_targets is None: + return None + (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, + num_total_pos, num_total_neg) = cls_reg_targets + + num_images = len(img_metas) + all_cls_scores = torch.cat([ + s.permute(0, 2, 3, 1).reshape( + num_images, -1, self.cls_out_channels) for s in cls_scores + ], 1) + all_labels = torch.cat(labels_list, -1).view(num_images, -1) + all_label_weights = torch.cat(label_weights_list, + -1).view(num_images, -1) + all_bbox_preds = torch.cat([ + b.permute(0, 2, 3, 1).reshape(num_images, -1, 4) + for b in bbox_preds + ], -2) + all_bbox_targets = torch.cat(bbox_targets_list, + -2).view(num_images, -1, 4) + all_bbox_weights = torch.cat(bbox_weights_list, + -2).view(num_images, -1, 4) + + # concat all level anchors to a single tensor + all_anchors = [] + for i in range(num_images): + all_anchors.append(torch.cat(anchor_list[i])) + + # check NaN and Inf + assert torch.isfinite(all_cls_scores).all().item(), \ + 'classification scores become infinite or NaN!' + assert torch.isfinite(all_bbox_preds).all().item(), \ + 'bbox predications become infinite or NaN!' + + losses_cls, losses_bbox = multi_apply( + self.loss_single, + all_cls_scores, + all_bbox_preds, + all_anchors, + all_labels, + all_label_weights, + all_bbox_targets, + all_bbox_weights, + num_total_samples=num_total_pos) + return dict(loss_cls=losses_cls, loss_bbox=losses_bbox) diff --git a/detection/mmdet/models/dense_heads/transformer_head.py b/detection/mmdet/models/dense_heads/transformer_head.py new file mode 100644 index 0000000..820fd06 --- /dev/null +++ b/detection/mmdet/models/dense_heads/transformer_head.py @@ -0,0 +1,654 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import Conv2d, Linear, build_activation_layer +from mmcv.runner import force_fp32 + +from mmdet.core import (bbox_cxcywh_to_xyxy, bbox_xyxy_to_cxcywh, + build_assigner, build_sampler, multi_apply, + reduce_mean) +from mmdet.models.utils import (FFN, build_positional_encoding, + build_transformer) +from ..builder import HEADS, build_loss +from .anchor_free_head import AnchorFreeHead + + +@HEADS.register_module() +class TransformerHead(AnchorFreeHead): + """Implements the DETR transformer head. + + See `paper: End-to-End Object Detection with Transformers + `_ for details. + + Args: + num_classes (int): Number of categories excluding the background. + in_channels (int): Number of channels in the input feature map. + num_fcs (int, optional): Number of fully-connected layers used in + `FFN`, which is then used for the regression head. Default 2. + transformer (dict, optional): Config for transformer. + positional_encoding (dict, optional): Config for position encoding. + loss_cls (dict, optional): Config of the classification loss. + Default `CrossEntropyLoss`. + loss_bbox (dict, optional): Config of the regression loss. + Default `L1Loss`. + loss_iou (dict, optional): Config of the regression iou loss. + Default `GIoULoss`. + tran_cfg (dict, optional): Training config of transformer head. + test_cfg (dict, optional): Testing config of transformer head. + + Example: + >>> import torch + >>> self = TransformerHead(80, 2048) + >>> x = torch.rand(1, 2048, 32, 32) + >>> mask = torch.ones(1, 32, 32).to(x.dtype) + >>> mask[:, :16, :15] = 0 + >>> all_cls_scores, all_bbox_preds = self(x, mask) + """ + + def __init__(self, + num_classes, + in_channels, + num_fcs=2, + transformer=dict( + type='Transformer', + embed_dims=256, + num_heads=8, + num_encoder_layers=6, + num_decoder_layers=6, + feedforward_channels=2048, + dropout=0.1, + act_cfg=dict(type='ReLU', inplace=True), + norm_cfg=dict(type='LN'), + num_fcs=2, + pre_norm=False, + return_intermediate_dec=True), + positional_encoding=dict( + type='SinePositionalEncoding', + num_feats=128, + normalize=True), + loss_cls=dict( + type='CrossEntropyLoss', + bg_cls_weight=0.1, + use_sigmoid=False, + loss_weight=1.0, + class_weight=1.0), + loss_bbox=dict(type='L1Loss', loss_weight=5.0), + loss_iou=dict(type='GIoULoss', loss_weight=2.0), + train_cfg=dict( + assigner=dict( + type='HungarianAssigner', + cls_cost=dict(type='ClassificationCost', weight=1.), + reg_cost=dict(type='BBoxL1Cost', weight=5.0), + iou_cost=dict( + type='IoUCost', iou_mode='giou', weight=2.0))), + test_cfg=dict(max_per_img=100), + **kwargs): + # NOTE here use `AnchorFreeHead` instead of `TransformerHead`, + # since it brings inconvenience when the initialization of + # `AnchorFreeHead` is called. + super(AnchorFreeHead, self).__init__() + use_sigmoid_cls = loss_cls.get('use_sigmoid', False) + assert not use_sigmoid_cls, 'setting use_sigmoid_cls as True is ' \ + 'not supported in DETR, since background is needed for the ' \ + 'matching process.' + assert 'embed_dims' in transformer \ + and 'num_feats' in positional_encoding + num_feats = positional_encoding['num_feats'] + embed_dims = transformer['embed_dims'] + assert num_feats * 2 == embed_dims, 'embed_dims should' \ + f' be exactly 2 times of num_feats. Found {embed_dims}' \ + f' and {num_feats}.' + assert test_cfg is not None and 'max_per_img' in test_cfg + + class_weight = loss_cls.get('class_weight', None) + if class_weight is not None: + assert isinstance(class_weight, float), 'Expected ' \ + 'class_weight to have type float. Found ' \ + f'{type(class_weight)}.' + # NOTE following the official DETR rep0, bg_cls_weight means + # relative classification weight of the no-object class. + bg_cls_weight = loss_cls.get('bg_cls_weight', class_weight) + assert isinstance(bg_cls_weight, float), 'Expected ' \ + 'bg_cls_weight to have type float. Found ' \ + f'{type(bg_cls_weight)}.' + class_weight = torch.ones(num_classes + 1) * class_weight + # set background class as the last indice + class_weight[num_classes] = bg_cls_weight + loss_cls.update({'class_weight': class_weight}) + if 'bg_cls_weight' in loss_cls: + loss_cls.pop('bg_cls_weight') + self.bg_cls_weight = bg_cls_weight + + if train_cfg: + assert 'assigner' in train_cfg, 'assigner should be provided '\ + 'when train_cfg is set.' + assigner = train_cfg['assigner'] + assert loss_cls['loss_weight'] == assigner['cls_cost']['weight'], \ + 'The classification weight for loss and matcher should be' \ + 'exactly the same.' + assert loss_bbox['loss_weight'] == assigner['reg_cost'][ + 'weight'], 'The regression L1 weight for loss and matcher ' \ + 'should be exactly the same.' + assert loss_iou['loss_weight'] == assigner['iou_cost']['weight'], \ + 'The regression iou weight for loss and matcher should be' \ + 'exactly the same.' + self.assigner = build_assigner(assigner) + # DETR sampling=False, so use PseudoSampler + sampler_cfg = dict(type='PseudoSampler') + self.sampler = build_sampler(sampler_cfg, context=self) + self.num_classes = num_classes + self.cls_out_channels = num_classes + 1 + self.in_channels = in_channels + self.num_fcs = num_fcs + self.train_cfg = train_cfg + self.test_cfg = test_cfg + self.use_sigmoid_cls = use_sigmoid_cls + self.embed_dims = embed_dims + self.num_query = test_cfg['max_per_img'] + self.fp16_enabled = False + self.loss_cls = build_loss(loss_cls) + self.loss_bbox = build_loss(loss_bbox) + self.loss_iou = build_loss(loss_iou) + self.act_cfg = transformer.get('act_cfg', + dict(type='ReLU', inplace=True)) + self.activate = build_activation_layer(self.act_cfg) + self.positional_encoding = build_positional_encoding( + positional_encoding) + self.transformer = build_transformer(transformer) + self._init_layers() + + def _init_layers(self): + """Initialize layers of the transformer head.""" + self.input_proj = Conv2d( + self.in_channels, self.embed_dims, kernel_size=1) + self.fc_cls = Linear(self.embed_dims, self.cls_out_channels) + self.reg_ffn = FFN( + self.embed_dims, + self.embed_dims, + self.num_fcs, + self.act_cfg, + dropout=0.0, + add_residual=False) + self.fc_reg = Linear(self.embed_dims, 4) + self.query_embedding = nn.Embedding(self.num_query, self.embed_dims) + + def init_weights(self, distribution='uniform'): + """Initialize weights of the transformer head.""" + # The initialization for transformer is important + self.transformer.init_weights() + + def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, + missing_keys, unexpected_keys, error_msgs): + """load checkpoints.""" + # NOTE here use `AnchorFreeHead` instead of `TransformerHead`, + # since `AnchorFreeHead._load_from_state_dict` should not be + # called here. Invoking the default `Module._load_from_state_dict` + # is enough. + super(AnchorFreeHead, + self)._load_from_state_dict(state_dict, prefix, local_metadata, + strict, missing_keys, + unexpected_keys, error_msgs) + + def forward(self, feats, img_metas): + """Forward function. + + Args: + feats (tuple[Tensor]): Features from the upstream network, each is + a 4D-tensor. + img_metas (list[dict]): List of image information. + + Returns: + tuple[list[Tensor], list[Tensor]]: Outputs for all scale levels. + + - all_cls_scores_list (list[Tensor]): Classification scores \ + for each scale level. Each is a 4D-tensor with shape \ + [nb_dec, bs, num_query, cls_out_channels]. Note \ + `cls_out_channels` should includes background. + - all_bbox_preds_list (list[Tensor]): Sigmoid regression \ + outputs for each scale level. Each is a 4D-tensor with \ + normalized coordinate format (cx, cy, w, h) and shape \ + [nb_dec, bs, num_query, 4]. + """ + num_levels = len(feats) + img_metas_list = [img_metas for _ in range(num_levels)] + return multi_apply(self.forward_single, feats, img_metas_list) + + def forward_single(self, x, img_metas): + """"Forward function for a single feature level. + + Args: + x (Tensor): Input feature from backbone's single stage, shape + [bs, c, h, w]. + img_metas (list[dict]): List of image information. + + Returns: + all_cls_scores (Tensor): Outputs from the classification head, + shape [nb_dec, bs, num_query, cls_out_channels]. Note + cls_out_channels should includes background. + all_bbox_preds (Tensor): Sigmoid outputs from the regression + head with normalized coordinate format (cx, cy, w, h). + Shape [nb_dec, bs, num_query, 4]. + """ + # construct binary masks which used for the transformer. + # NOTE following the official DETR repo, non-zero values representing + # ignored positions, while zero values means valid positions. + batch_size = x.size(0) + input_img_h, input_img_w = img_metas[0]['batch_input_shape'] + masks = x.new_ones((batch_size, input_img_h, input_img_w)) + for img_id in range(batch_size): + img_h, img_w, _ = img_metas[img_id]['img_shape'] + masks[img_id, :img_h, :img_w] = 0 + + x = self.input_proj(x) + # interpolate masks to have the same spatial shape with x + masks = F.interpolate( + masks.unsqueeze(1), size=x.shape[-2:]).to(torch.bool).squeeze(1) + # position encoding + pos_embed = self.positional_encoding(masks) # [bs, embed_dim, h, w] + # outs_dec: [nb_dec, bs, num_query, embed_dim] + outs_dec, _ = self.transformer(x, masks, self.query_embedding.weight, + pos_embed) + + all_cls_scores = self.fc_cls(outs_dec) + all_bbox_preds = self.fc_reg(self.activate( + self.reg_ffn(outs_dec))).sigmoid() + return all_cls_scores, all_bbox_preds + + @force_fp32(apply_to=('all_cls_scores_list', 'all_bbox_preds_list')) + def loss(self, + all_cls_scores_list, + all_bbox_preds_list, + gt_bboxes_list, + gt_labels_list, + img_metas, + gt_bboxes_ignore=None): + """"Loss function. + + Only outputs from the last feature level are used for computing + losses by default. + + Args: + all_cls_scores_list (list[Tensor]): Classification outputs + for each feature level. Each is a 4D-tensor with shape + [nb_dec, bs, num_query, cls_out_channels]. + all_bbox_preds_list (list[Tensor]): Sigmoid regression + outputs for each feature level. Each is a 4D-tensor with + normalized coordinate format (cx, cy, w, h) and shape + [nb_dec, bs, num_query, 4]. + gt_bboxes_list (list[Tensor]): Ground truth bboxes for each image + with shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + gt_labels_list (list[Tensor]): Ground truth class indices for each + image with shape (num_gts, ). + img_metas (list[dict]): List of image meta information. + gt_bboxes_ignore (list[Tensor], optional): Bounding boxes + which can be ignored for each image. Default None. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + # NOTE defaultly only the outputs from the last feature scale is used. + all_cls_scores = all_cls_scores_list[-1] + all_bbox_preds = all_bbox_preds_list[-1] + assert gt_bboxes_ignore is None, \ + 'Only supports for gt_bboxes_ignore setting to None.' + + num_dec_layers = len(all_cls_scores) + all_gt_bboxes_list = [gt_bboxes_list for _ in range(num_dec_layers)] + all_gt_labels_list = [gt_labels_list for _ in range(num_dec_layers)] + all_gt_bboxes_ignore_list = [ + gt_bboxes_ignore for _ in range(num_dec_layers) + ] + img_metas_list = [img_metas for _ in range(num_dec_layers)] + + losses_cls, losses_bbox, losses_iou = multi_apply( + self.loss_single, all_cls_scores, all_bbox_preds, + all_gt_bboxes_list, all_gt_labels_list, img_metas_list, + all_gt_bboxes_ignore_list) + + loss_dict = dict() + # loss from the last decoder layer + loss_dict['loss_cls'] = losses_cls[-1] + loss_dict['loss_bbox'] = losses_bbox[-1] + loss_dict['loss_iou'] = losses_iou[-1] + # loss from other decoder layers + num_dec_layer = 0 + for loss_cls_i, loss_bbox_i, loss_iou_i in zip(losses_cls[:-1], + losses_bbox[:-1], + losses_iou[:-1]): + loss_dict[f'd{num_dec_layer}.loss_cls'] = loss_cls_i + loss_dict[f'd{num_dec_layer}.loss_bbox'] = loss_bbox_i + loss_dict[f'd{num_dec_layer}.loss_iou'] = loss_iou_i + num_dec_layer += 1 + return loss_dict + + def loss_single(self, + cls_scores, + bbox_preds, + gt_bboxes_list, + gt_labels_list, + img_metas, + gt_bboxes_ignore_list=None): + """"Loss function for outputs from a single decoder layer of a single + feature level. + + Args: + cls_scores (Tensor): Box score logits from a single decoder layer + for all images. Shape [bs, num_query, cls_out_channels]. + bbox_preds (Tensor): Sigmoid outputs from a single decoder layer + for all images, with normalized coordinate (cx, cy, w, h) and + shape [bs, num_query, 4]. + gt_bboxes_list (list[Tensor]): Ground truth bboxes for each image + with shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + gt_labels_list (list[Tensor]): Ground truth class indices for each + image with shape (num_gts, ). + img_metas (list[dict]): List of image meta information. + gt_bboxes_ignore_list (list[Tensor], optional): Bounding + boxes which can be ignored for each image. Default None. + + Returns: + dict[str, Tensor]: A dictionary of loss components for outputs from + a single decoder layer. + """ + num_imgs = cls_scores.size(0) + cls_scores_list = [cls_scores[i] for i in range(num_imgs)] + bbox_preds_list = [bbox_preds[i] for i in range(num_imgs)] + cls_reg_targets = self.get_targets(cls_scores_list, bbox_preds_list, + gt_bboxes_list, gt_labels_list, + img_metas, gt_bboxes_ignore_list) + (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, + num_total_pos, num_total_neg) = cls_reg_targets + labels = torch.cat(labels_list, 0) + label_weights = torch.cat(label_weights_list, 0) + bbox_targets = torch.cat(bbox_targets_list, 0) + bbox_weights = torch.cat(bbox_weights_list, 0) + + # classification loss + cls_scores = cls_scores.reshape(-1, self.cls_out_channels) + # construct weighted avg_factor to match with the official DETR repo + cls_avg_factor = num_total_pos * 1.0 + \ + num_total_neg * self.bg_cls_weight + loss_cls = self.loss_cls( + cls_scores, labels, label_weights, avg_factor=cls_avg_factor) + + # Compute the average number of gt boxes accross all gpus, for + # normalization purposes + num_total_pos = loss_cls.new_tensor([num_total_pos]) + num_total_pos = torch.clamp(reduce_mean(num_total_pos), min=1).item() + + # construct factors used for rescale bboxes + factors = [] + for img_meta, bbox_pred in zip(img_metas, bbox_preds): + img_h, img_w, _ = img_meta['img_shape'] + factor = bbox_pred.new_tensor([img_w, img_h, img_w, + img_h]).unsqueeze(0).repeat( + bbox_pred.size(0), 1) + factors.append(factor) + factors = torch.cat(factors, 0) + + # DETR regress the relative position of boxes (cxcywh) in the image, + # thus the learning target is normalized by the image size. So here + # we need to re-scale them for calculating IoU loss + bbox_preds = bbox_preds.reshape(-1, 4) + bboxes = bbox_cxcywh_to_xyxy(bbox_preds) * factors + bboxes_gt = bbox_cxcywh_to_xyxy(bbox_targets) * factors + + # regression IoU loss, defaultly GIoU loss + loss_iou = self.loss_iou( + bboxes, bboxes_gt, bbox_weights, avg_factor=num_total_pos) + + # regression L1 loss + loss_bbox = self.loss_bbox( + bbox_preds, bbox_targets, bbox_weights, avg_factor=num_total_pos) + return loss_cls, loss_bbox, loss_iou + + def get_targets(self, + cls_scores_list, + bbox_preds_list, + gt_bboxes_list, + gt_labels_list, + img_metas, + gt_bboxes_ignore_list=None): + """"Compute regression and classification targets for a batch image. + + Outputs from a single decoder layer of a single feature level are used. + + Args: + cls_scores_list (list[Tensor]): Box score logits from a single + decoder layer for each image with shape [num_query, + cls_out_channels]. + bbox_preds_list (list[Tensor]): Sigmoid outputs from a single + decoder layer for each image, with normalized coordinate + (cx, cy, w, h) and shape [num_query, 4]. + gt_bboxes_list (list[Tensor]): Ground truth bboxes for each image + with shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + gt_labels_list (list[Tensor]): Ground truth class indices for each + image with shape (num_gts, ). + img_metas (list[dict]): List of image meta information. + gt_bboxes_ignore_list (list[Tensor], optional): Bounding + boxes which can be ignored for each image. Default None. + + Returns: + tuple: a tuple containing the following targets. + + - labels_list (list[Tensor]): Labels for all images. + - label_weights_list (list[Tensor]): Label weights for all \ + images. + - bbox_targets_list (list[Tensor]): BBox targets for all \ + images. + - bbox_weights_list (list[Tensor]): BBox weights for all \ + images. + - num_total_pos (int): Number of positive samples in all \ + images. + - num_total_neg (int): Number of negative samples in all \ + images. + """ + assert gt_bboxes_ignore_list is None, \ + 'Only supports for gt_bboxes_ignore setting to None.' + num_imgs = len(cls_scores_list) + gt_bboxes_ignore_list = [ + gt_bboxes_ignore_list for _ in range(num_imgs) + ] + + (labels_list, label_weights_list, bbox_targets_list, + bbox_weights_list, pos_inds_list, neg_inds_list) = multi_apply( + self._get_target_single, cls_scores_list, bbox_preds_list, + gt_bboxes_list, gt_labels_list, img_metas, gt_bboxes_ignore_list) + num_total_pos = sum((inds.numel() for inds in pos_inds_list)) + num_total_neg = sum((inds.numel() for inds in neg_inds_list)) + return (labels_list, label_weights_list, bbox_targets_list, + bbox_weights_list, num_total_pos, num_total_neg) + + def _get_target_single(self, + cls_score, + bbox_pred, + gt_bboxes, + gt_labels, + img_meta, + gt_bboxes_ignore=None): + """"Compute regression and classification targets for one image. + + Outputs from a single decoder layer of a single feature level are used. + + Args: + cls_score (Tensor): Box score logits from a single decoder layer + for one image. Shape [num_query, cls_out_channels]. + bbox_pred (Tensor): Sigmoid outputs from a single decoder layer + for one image, with normalized coordinate (cx, cy, w, h) and + shape [num_query, 4]. + gt_bboxes (Tensor): Ground truth bboxes for one image with + shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + gt_labels (Tensor): Ground truth class indices for one image + with shape (num_gts, ). + img_meta (dict): Meta information for one image. + gt_bboxes_ignore (Tensor, optional): Bounding boxes + which can be ignored. Default None. + + Returns: + tuple[Tensor]: a tuple containing the following for one image. + + - labels (Tensor): Labels of each image. + - label_weights (Tensor]): Label weights of each image. + - bbox_targets (Tensor): BBox targets of each image. + - bbox_weights (Tensor): BBox weights of each image. + - pos_inds (Tensor): Sampled positive indices for each image. + - neg_inds (Tensor): Sampled negative indices for each image. + """ + + num_bboxes = bbox_pred.size(0) + # assigner and sampler + assign_result = self.assigner.assign(bbox_pred, cls_score, gt_bboxes, + gt_labels, img_meta, + gt_bboxes_ignore) + sampling_result = self.sampler.sample(assign_result, bbox_pred, + gt_bboxes) + pos_inds = sampling_result.pos_inds + neg_inds = sampling_result.neg_inds + + # label targets + labels = gt_bboxes.new_full((num_bboxes, ), + self.num_classes, + dtype=torch.long) + labels[pos_inds] = gt_labels[sampling_result.pos_assigned_gt_inds] + label_weights = gt_bboxes.new_ones(num_bboxes) + + # bbox targets + bbox_targets = torch.zeros_like(bbox_pred) + bbox_weights = torch.zeros_like(bbox_pred) + bbox_weights[pos_inds] = 1.0 + img_h, img_w, _ = img_meta['img_shape'] + + # DETR regress the relative position of boxes (cxcywh) in the image. + # Thus the learning target should be normalized by the image size, also + # the box format should be converted from defaultly x1y1x2y2 to cxcywh. + factor = bbox_pred.new_tensor([img_w, img_h, img_w, + img_h]).unsqueeze(0) + pos_gt_bboxes_normalized = sampling_result.pos_gt_bboxes / factor + pos_gt_bboxes_targets = bbox_xyxy_to_cxcywh(pos_gt_bboxes_normalized) + bbox_targets[pos_inds] = pos_gt_bboxes_targets + return (labels, label_weights, bbox_targets, bbox_weights, pos_inds, + neg_inds) + + # over-write because img_metas are needed as inputs for bbox_head. + def forward_train(self, + x, + img_metas, + gt_bboxes, + gt_labels=None, + gt_bboxes_ignore=None, + proposal_cfg=None, + **kwargs): + """Forward function for training mode. + + Args: + x (list[Tensor]): Features from backbone. + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + gt_bboxes (Tensor): Ground truth bboxes of the image, + shape (num_gts, 4). + gt_labels (Tensor): Ground truth labels of each box, + shape (num_gts,). + gt_bboxes_ignore (Tensor): Ground truth bboxes to be + ignored, shape (num_ignored_gts, 4). + proposal_cfg (mmcv.Config): Test / postprocessing configuration, + if None, test_cfg would be used. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + assert proposal_cfg is None, '"proposal_cfg" must be None' + outs = self(x, img_metas) + if gt_labels is None: + loss_inputs = outs + (gt_bboxes, img_metas) + else: + loss_inputs = outs + (gt_bboxes, gt_labels, img_metas) + losses = self.loss(*loss_inputs, gt_bboxes_ignore=gt_bboxes_ignore) + return losses + + @force_fp32(apply_to=('all_cls_scores_list', 'all_bbox_preds_list')) + def get_bboxes(self, + all_cls_scores_list, + all_bbox_preds_list, + img_metas, + rescale=False): + """Transform network outputs for a batch into bbox predictions. + + Args: + all_cls_scores_list (list[Tensor]): Classification outputs + for each feature level. Each is a 4D-tensor with shape + [nb_dec, bs, num_query, cls_out_channels]. + all_bbox_preds_list (list[Tensor]): Sigmoid regression + outputs for each feature level. Each is a 4D-tensor with + normalized coordinate format (cx, cy, w, h) and shape + [nb_dec, bs, num_query, 4]. + img_metas (list[dict]): Meta information of each image. + rescale (bool, optional): If True, return boxes in original + image space. Default False. + + Returns: + list[list[Tensor, Tensor]]: Each item in result_list is 2-tuple. \ + The first item is an (n, 5) tensor, where the first 4 columns \ + are bounding box positions (tl_x, tl_y, br_x, br_y) and the \ + 5-th column is a score between 0 and 1. The second item is a \ + (n,) tensor where each item is the predicted class label of \ + the corresponding box. + """ + # NOTE defaultly only using outputs from the last feature level, + # and only the outputs from the last decoder layer is used. + cls_scores = all_cls_scores_list[-1][-1] + bbox_preds = all_bbox_preds_list[-1][-1] + + result_list = [] + for img_id in range(len(img_metas)): + cls_score = cls_scores[img_id] + bbox_pred = bbox_preds[img_id] + img_shape = img_metas[img_id]['img_shape'] + scale_factor = img_metas[img_id]['scale_factor'] + proposals = self._get_bboxes_single(cls_score, bbox_pred, + img_shape, scale_factor, + rescale) + result_list.append(proposals) + return result_list + + def _get_bboxes_single(self, + cls_score, + bbox_pred, + img_shape, + scale_factor, + rescale=False): + """Transform outputs from the last decoder layer into bbox predictions + for each image. + + Args: + cls_score (Tensor): Box score logits from the last decoder layer + for each image. Shape [num_query, cls_out_channels]. + bbox_pred (Tensor): Sigmoid outputs from the last decoder layer + for each image, with coordinate format (cx, cy, w, h) and + shape [num_query, 4]. + img_shape (tuple[int]): Shape of input image, (height, width, 3). + scale_factor (ndarray, optional): Scale factor of the image arange + as (w_scale, h_scale, w_scale, h_scale). + rescale (bool, optional): If True, return boxes in original image + space. Default False. + + Returns: + tuple[Tensor]: Results of detected bboxes and labels. + + - det_bboxes: Predicted bboxes with shape [num_query, 5], \ + where the first 4 columns are bounding box positions \ + (tl_x, tl_y, br_x, br_y) and the 5-th column are scores \ + between 0 and 1. + - det_labels: Predicted labels of the corresponding box with \ + shape [num_query]. + """ + assert len(cls_score) == len(bbox_pred) + # exclude background + scores, det_labels = F.softmax(cls_score, dim=-1)[..., :-1].max(-1) + det_bboxes = bbox_cxcywh_to_xyxy(bbox_pred) + det_bboxes[:, 0::2] = det_bboxes[:, 0::2] * img_shape[1] + det_bboxes[:, 1::2] = det_bboxes[:, 1::2] * img_shape[0] + det_bboxes[:, 0::2].clamp_(min=0, max=img_shape[1]) + det_bboxes[:, 1::2].clamp_(min=0, max=img_shape[0]) + if rescale: + det_bboxes /= det_bboxes.new_tensor(scale_factor) + det_bboxes = torch.cat((det_bboxes, scores.unsqueeze(1)), -1) + return det_bboxes, det_labels diff --git a/detection/mmdet/models/dense_heads/vfnet_head.py b/detection/mmdet/models/dense_heads/vfnet_head.py new file mode 100644 index 0000000..7243bb6 --- /dev/null +++ b/detection/mmdet/models/dense_heads/vfnet_head.py @@ -0,0 +1,794 @@ +import numpy as np +import torch +import torch.nn as nn +from mmcv.cnn import ConvModule, Scale, bias_init_with_prob, normal_init +from mmcv.ops import DeformConv2d +from mmcv.runner import force_fp32 + +from mmdet.core import (bbox2distance, bbox_overlaps, build_anchor_generator, + build_assigner, build_sampler, distance2bbox, + multi_apply, multiclass_nms, reduce_mean) +from ..builder import HEADS, build_loss +from .atss_head import ATSSHead +from .fcos_head import FCOSHead + +INF = 1e8 + + +@HEADS.register_module() +class VFNetHead(ATSSHead, FCOSHead): + """Head of `VarifocalNet (VFNet): An IoU-aware Dense Object + Detector.`_. + + The VFNet predicts IoU-aware classification scores which mix the + object presence confidence and object localization accuracy as the + detection score. It is built on the FCOS architecture and uses ATSS + for defining positive/negative training examples. The VFNet is trained + with Varifocal Loss and empolys star-shaped deformable convolution to + extract features for a bbox. + + Args: + num_classes (int): Number of categories excluding the background + category. + in_channels (int): Number of channels in the input feature map. + regress_ranges (tuple[tuple[int, int]]): Regress range of multiple + level points. + center_sampling (bool): If true, use center sampling. Default: False. + center_sample_radius (float): Radius of center sampling. Default: 1.5. + sync_num_pos (bool): If true, synchronize the number of positive + examples across GPUs. Default: True + gradient_mul (float): The multiplier to gradients from bbox refinement + and recognition. Default: 0.1. + bbox_norm_type (str): The bbox normalization type, 'reg_denom' or + 'stride'. Default: reg_denom + loss_cls_fl (dict): Config of focal loss. + use_vfl (bool): If true, use varifocal loss for training. + Default: True. + loss_cls (dict): Config of varifocal loss. + loss_bbox (dict): Config of localization loss, GIoU Loss. + loss_bbox (dict): Config of localization refinement loss, GIoU Loss. + norm_cfg (dict): dictionary to construct and config norm layer. + Default: norm_cfg=dict(type='GN', num_groups=32, + requires_grad=True). + use_atss (bool): If true, use ATSS to define positive/negative + examples. Default: True. + anchor_generator (dict): Config of anchor generator for ATSS. + + Example: + >>> self = VFNetHead(11, 7) + >>> feats = [torch.rand(1, 7, s, s) for s in [4, 8, 16, 32, 64]] + >>> cls_score, bbox_pred, bbox_pred_refine= self.forward(feats) + >>> assert len(cls_score) == len(self.scales) + """ # noqa: E501 + + def __init__(self, + num_classes, + in_channels, + regress_ranges=((-1, 64), (64, 128), (128, 256), (256, 512), + (512, INF)), + center_sampling=False, + center_sample_radius=1.5, + sync_num_pos=True, + gradient_mul=0.1, + bbox_norm_type='reg_denom', + loss_cls_fl=dict( + type='FocalLoss', + use_sigmoid=True, + gamma=2.0, + alpha=0.25, + loss_weight=1.0), + use_vfl=True, + loss_cls=dict( + type='VarifocalLoss', + use_sigmoid=True, + alpha=0.75, + gamma=2.0, + iou_weighted=True, + loss_weight=1.0), + loss_bbox=dict(type='GIoULoss', loss_weight=1.5), + loss_bbox_refine=dict(type='GIoULoss', loss_weight=2.0), + norm_cfg=dict(type='GN', num_groups=32, requires_grad=True), + use_atss=True, + anchor_generator=dict( + type='AnchorGenerator', + ratios=[1.0], + octave_base_scale=8, + scales_per_octave=1, + center_offset=0.0, + strides=[8, 16, 32, 64, 128]), + **kwargs): + # dcn base offsets, adapted from reppoints_head.py + self.num_dconv_points = 9 + self.dcn_kernel = int(np.sqrt(self.num_dconv_points)) + self.dcn_pad = int((self.dcn_kernel - 1) / 2) + dcn_base = np.arange(-self.dcn_pad, + self.dcn_pad + 1).astype(np.float64) + dcn_base_y = np.repeat(dcn_base, self.dcn_kernel) + dcn_base_x = np.tile(dcn_base, self.dcn_kernel) + dcn_base_offset = np.stack([dcn_base_y, dcn_base_x], axis=1).reshape( + (-1)) + self.dcn_base_offset = torch.tensor(dcn_base_offset).view(1, -1, 1, 1) + + super(FCOSHead, self).__init__( + num_classes, in_channels, norm_cfg=norm_cfg, **kwargs) + self.regress_ranges = regress_ranges + self.reg_denoms = [ + regress_range[-1] for regress_range in regress_ranges + ] + self.reg_denoms[-1] = self.reg_denoms[-2] * 2 + self.center_sampling = center_sampling + self.center_sample_radius = center_sample_radius + self.sync_num_pos = sync_num_pos + self.bbox_norm_type = bbox_norm_type + self.gradient_mul = gradient_mul + self.use_vfl = use_vfl + if self.use_vfl: + self.loss_cls = build_loss(loss_cls) + else: + self.loss_cls = build_loss(loss_cls_fl) + self.loss_bbox = build_loss(loss_bbox) + self.loss_bbox_refine = build_loss(loss_bbox_refine) + + # for getting ATSS targets + self.use_atss = use_atss + self.use_sigmoid_cls = loss_cls.get('use_sigmoid', False) + self.anchor_generator = build_anchor_generator(anchor_generator) + self.anchor_center_offset = anchor_generator['center_offset'] + self.num_anchors = self.anchor_generator.num_base_anchors[0] + self.sampling = False + if self.train_cfg: + self.assigner = build_assigner(self.train_cfg.assigner) + sampler_cfg = dict(type='PseudoSampler') + self.sampler = build_sampler(sampler_cfg, context=self) + + def _init_layers(self): + """Initialize layers of the head.""" + super(FCOSHead, self)._init_cls_convs() + super(FCOSHead, self)._init_reg_convs() + self.relu = nn.ReLU(inplace=True) + self.vfnet_reg_conv = ConvModule( + self.feat_channels, + self.feat_channels, + 3, + stride=1, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + bias=self.conv_bias) + self.vfnet_reg = nn.Conv2d(self.feat_channels, 4, 3, padding=1) + self.scales = nn.ModuleList([Scale(1.0) for _ in self.strides]) + + self.vfnet_reg_refine_dconv = DeformConv2d( + self.feat_channels, + self.feat_channels, + self.dcn_kernel, + 1, + padding=self.dcn_pad) + self.vfnet_reg_refine = nn.Conv2d(self.feat_channels, 4, 3, padding=1) + self.scales_refine = nn.ModuleList([Scale(1.0) for _ in self.strides]) + + self.vfnet_cls_dconv = DeformConv2d( + self.feat_channels, + self.feat_channels, + self.dcn_kernel, + 1, + padding=self.dcn_pad) + self.vfnet_cls = nn.Conv2d( + self.feat_channels, self.cls_out_channels, 3, padding=1) + + def init_weights(self): + """Initialize weights of the head.""" + for m in self.cls_convs: + if isinstance(m.conv, nn.Conv2d): + normal_init(m.conv, std=0.01) + for m in self.reg_convs: + if isinstance(m.conv, nn.Conv2d): + normal_init(m.conv, std=0.01) + normal_init(self.vfnet_reg_conv.conv, std=0.01) + normal_init(self.vfnet_reg, std=0.01) + normal_init(self.vfnet_reg_refine_dconv, std=0.01) + normal_init(self.vfnet_reg_refine, std=0.01) + normal_init(self.vfnet_cls_dconv, std=0.01) + bias_cls = bias_init_with_prob(0.01) + normal_init(self.vfnet_cls, std=0.01, bias=bias_cls) + + def forward(self, feats): + """Forward features from the upstream network. + + Args: + feats (tuple[Tensor]): Features from the upstream network, each is + a 4D-tensor. + + Returns: + tuple: + cls_scores (list[Tensor]): Box iou-aware scores for each scale + level, each is a 4D-tensor, the channel number is + num_points * num_classes. + bbox_preds (list[Tensor]): Box offsets for each + scale level, each is a 4D-tensor, the channel number is + num_points * 4. + bbox_preds_refine (list[Tensor]): Refined Box offsets for + each scale level, each is a 4D-tensor, the channel + number is num_points * 4. + """ + return multi_apply(self.forward_single, feats, self.scales, + self.scales_refine, self.strides, self.reg_denoms) + + def forward_single(self, x, scale, scale_refine, stride, reg_denom): + """Forward features of a single scale level. + + Args: + x (Tensor): FPN feature maps of the specified stride. + scale (:obj: `mmcv.cnn.Scale`): Learnable scale module to resize + the bbox prediction. + scale_refine (:obj: `mmcv.cnn.Scale`): Learnable scale module to + resize the refined bbox prediction. + stride (int): The corresponding stride for feature maps, + used to normalize the bbox prediction when + bbox_norm_type = 'stride'. + reg_denom (int): The corresponding regression range for feature + maps, only used to normalize the bbox prediction when + bbox_norm_type = 'reg_denom'. + + Returns: + tuple: iou-aware cls scores for each box, bbox predictions and + refined bbox predictions of input feature maps. + """ + cls_feat = x + reg_feat = x + + for cls_layer in self.cls_convs: + cls_feat = cls_layer(cls_feat) + + for reg_layer in self.reg_convs: + reg_feat = reg_layer(reg_feat) + + # predict the bbox_pred of different level + reg_feat_init = self.vfnet_reg_conv(reg_feat) + if self.bbox_norm_type == 'reg_denom': + bbox_pred = scale( + self.vfnet_reg(reg_feat_init)).float().exp() * reg_denom + elif self.bbox_norm_type == 'stride': + bbox_pred = scale( + self.vfnet_reg(reg_feat_init)).float().exp() * stride + else: + raise NotImplementedError + + # compute star deformable convolution offsets + # converting dcn_offset to reg_feat.dtype thus VFNet can be + # trained with FP16 + dcn_offset = self.star_dcn_offset(bbox_pred, self.gradient_mul, + stride).to(reg_feat.dtype) + + # refine the bbox_pred + reg_feat = self.relu(self.vfnet_reg_refine_dconv(reg_feat, dcn_offset)) + bbox_pred_refine = scale_refine( + self.vfnet_reg_refine(reg_feat)).float().exp() + bbox_pred_refine = bbox_pred_refine * bbox_pred.detach() + + # predict the iou-aware cls score + cls_feat = self.relu(self.vfnet_cls_dconv(cls_feat, dcn_offset)) + cls_score = self.vfnet_cls(cls_feat) + + return cls_score, bbox_pred, bbox_pred_refine + + def star_dcn_offset(self, bbox_pred, gradient_mul, stride): + """Compute the star deformable conv offsets. + + Args: + bbox_pred (Tensor): Predicted bbox distance offsets (l, r, t, b). + gradient_mul (float): Gradient multiplier. + stride (int): The corresponding stride for feature maps, + used to project the bbox onto the feature map. + + Returns: + dcn_offsets (Tensor): The offsets for deformable convolution. + """ + dcn_base_offset = self.dcn_base_offset.type_as(bbox_pred) + bbox_pred_grad_mul = (1 - gradient_mul) * bbox_pred.detach() + \ + gradient_mul * bbox_pred + # map to the feature map scale + bbox_pred_grad_mul = bbox_pred_grad_mul / stride + N, C, H, W = bbox_pred.size() + + x1 = bbox_pred_grad_mul[:, 0, :, :] + y1 = bbox_pred_grad_mul[:, 1, :, :] + x2 = bbox_pred_grad_mul[:, 2, :, :] + y2 = bbox_pred_grad_mul[:, 3, :, :] + bbox_pred_grad_mul_offset = bbox_pred.new_zeros( + N, 2 * self.num_dconv_points, H, W) + bbox_pred_grad_mul_offset[:, 0, :, :] = -1.0 * y1 # -y1 + bbox_pred_grad_mul_offset[:, 1, :, :] = -1.0 * x1 # -x1 + bbox_pred_grad_mul_offset[:, 2, :, :] = -1.0 * y1 # -y1 + bbox_pred_grad_mul_offset[:, 4, :, :] = -1.0 * y1 # -y1 + bbox_pred_grad_mul_offset[:, 5, :, :] = x2 # x2 + bbox_pred_grad_mul_offset[:, 7, :, :] = -1.0 * x1 # -x1 + bbox_pred_grad_mul_offset[:, 11, :, :] = x2 # x2 + bbox_pred_grad_mul_offset[:, 12, :, :] = y2 # y2 + bbox_pred_grad_mul_offset[:, 13, :, :] = -1.0 * x1 # -x1 + bbox_pred_grad_mul_offset[:, 14, :, :] = y2 # y2 + bbox_pred_grad_mul_offset[:, 16, :, :] = y2 # y2 + bbox_pred_grad_mul_offset[:, 17, :, :] = x2 # x2 + dcn_offset = bbox_pred_grad_mul_offset - dcn_base_offset + + return dcn_offset + + @force_fp32(apply_to=('cls_scores', 'bbox_preds', 'bbox_preds_refine')) + def loss(self, + cls_scores, + bbox_preds, + bbox_preds_refine, + gt_bboxes, + gt_labels, + img_metas, + gt_bboxes_ignore=None): + """Compute loss of the head. + + Args: + cls_scores (list[Tensor]): Box iou-aware scores for each scale + level, each is a 4D-tensor, the channel number is + num_points * num_classes. + bbox_preds (list[Tensor]): Box offsets for each + scale level, each is a 4D-tensor, the channel number is + num_points * 4. + bbox_preds_refine (list[Tensor]): Refined Box offsets for + each scale level, each is a 4D-tensor, the channel + number is num_points * 4. + gt_bboxes (list[Tensor]): Ground truth bboxes for each image with + shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + gt_labels (list[Tensor]): class indices corresponding to each box + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + gt_bboxes_ignore (None | list[Tensor]): specify which bounding + boxes can be ignored when computing the loss. + Default: None. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + assert len(cls_scores) == len(bbox_preds) == len(bbox_preds_refine) + featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] + all_level_points = self.get_points(featmap_sizes, bbox_preds[0].dtype, + bbox_preds[0].device) + labels, label_weights, bbox_targets, bbox_weights = self.get_targets( + cls_scores, all_level_points, gt_bboxes, gt_labels, img_metas, + gt_bboxes_ignore) + + num_imgs = cls_scores[0].size(0) + # flatten cls_scores, bbox_preds and bbox_preds_refine + flatten_cls_scores = [ + cls_score.permute(0, 2, 3, + 1).reshape(-1, + self.cls_out_channels).contiguous() + for cls_score in cls_scores + ] + flatten_bbox_preds = [ + bbox_pred.permute(0, 2, 3, 1).reshape(-1, 4).contiguous() + for bbox_pred in bbox_preds + ] + flatten_bbox_preds_refine = [ + bbox_pred_refine.permute(0, 2, 3, 1).reshape(-1, 4).contiguous() + for bbox_pred_refine in bbox_preds_refine + ] + flatten_cls_scores = torch.cat(flatten_cls_scores) + flatten_bbox_preds = torch.cat(flatten_bbox_preds) + flatten_bbox_preds_refine = torch.cat(flatten_bbox_preds_refine) + flatten_labels = torch.cat(labels) + flatten_bbox_targets = torch.cat(bbox_targets) + # repeat points to align with bbox_preds + flatten_points = torch.cat( + [points.repeat(num_imgs, 1) for points in all_level_points]) + + # FG cat_id: [0, num_classes - 1], BG cat_id: num_classes + bg_class_ind = self.num_classes + pos_inds = torch.where( + ((flatten_labels >= 0) & (flatten_labels < bg_class_ind)) > 0)[0] + num_pos = len(pos_inds) + + pos_bbox_preds = flatten_bbox_preds[pos_inds] + pos_bbox_preds_refine = flatten_bbox_preds_refine[pos_inds] + pos_labels = flatten_labels[pos_inds] + + # sync num_pos across all gpus + if self.sync_num_pos: + num_pos_avg_per_gpu = reduce_mean( + pos_inds.new_tensor(num_pos).float()).item() + num_pos_avg_per_gpu = max(num_pos_avg_per_gpu, 1.0) + else: + num_pos_avg_per_gpu = num_pos + + if num_pos > 0: + pos_bbox_targets = flatten_bbox_targets[pos_inds] + pos_points = flatten_points[pos_inds] + + pos_decoded_bbox_preds = distance2bbox(pos_points, pos_bbox_preds) + pos_decoded_target_preds = distance2bbox(pos_points, + pos_bbox_targets) + iou_targets_ini = bbox_overlaps( + pos_decoded_bbox_preds, + pos_decoded_target_preds.detach(), + is_aligned=True).clamp(min=1e-6) + bbox_weights_ini = iou_targets_ini.clone().detach() + iou_targets_ini_avg_per_gpu = reduce_mean( + bbox_weights_ini.sum()).item() + bbox_avg_factor_ini = max(iou_targets_ini_avg_per_gpu, 1.0) + loss_bbox = self.loss_bbox( + pos_decoded_bbox_preds, + pos_decoded_target_preds.detach(), + weight=bbox_weights_ini, + avg_factor=bbox_avg_factor_ini) + + pos_decoded_bbox_preds_refine = \ + distance2bbox(pos_points, pos_bbox_preds_refine) + iou_targets_rf = bbox_overlaps( + pos_decoded_bbox_preds_refine, + pos_decoded_target_preds.detach(), + is_aligned=True).clamp(min=1e-6) + bbox_weights_rf = iou_targets_rf.clone().detach() + iou_targets_rf_avg_per_gpu = reduce_mean( + bbox_weights_rf.sum()).item() + bbox_avg_factor_rf = max(iou_targets_rf_avg_per_gpu, 1.0) + loss_bbox_refine = self.loss_bbox_refine( + pos_decoded_bbox_preds_refine, + pos_decoded_target_preds.detach(), + weight=bbox_weights_rf, + avg_factor=bbox_avg_factor_rf) + + # build IoU-aware cls_score targets + if self.use_vfl: + pos_ious = iou_targets_rf.clone().detach() + cls_iou_targets = torch.zeros_like(flatten_cls_scores) + cls_iou_targets[pos_inds, pos_labels] = pos_ious + else: + loss_bbox = pos_bbox_preds.sum() * 0 + loss_bbox_refine = pos_bbox_preds_refine.sum() * 0 + if self.use_vfl: + cls_iou_targets = torch.zeros_like(flatten_cls_scores) + + if self.use_vfl: + loss_cls = self.loss_cls( + flatten_cls_scores, + cls_iou_targets, + avg_factor=num_pos_avg_per_gpu) + else: + loss_cls = self.loss_cls( + flatten_cls_scores, + flatten_labels, + weight=label_weights, + avg_factor=num_pos_avg_per_gpu) + + return dict( + loss_cls=loss_cls, + loss_bbox=loss_bbox, + loss_bbox_rf=loss_bbox_refine) + + @force_fp32(apply_to=('cls_scores', 'bbox_preds', 'bbox_preds_refine')) + def get_bboxes(self, + cls_scores, + bbox_preds, + bbox_preds_refine, + img_metas, + cfg=None, + rescale=None, + with_nms=True): + """Transform network outputs for a batch into bbox predictions. + + Args: + cls_scores (list[Tensor]): Box iou-aware scores for each scale + level with shape (N, num_points * num_classes, H, W). + bbox_preds (list[Tensor]): Box offsets for each scale + level with shape (N, num_points * 4, H, W). + bbox_preds_refine (list[Tensor]): Refined Box offsets for + each scale level with shape (N, num_points * 4, H, W). + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + cfg (mmcv.Config): Test / postprocessing configuration, + if None, test_cfg would be used. Default: None. + rescale (bool): If True, return boxes in original image space. + Default: False. + with_nms (bool): If True, do nms before returning boxes. + Default: True. + + Returns: + list[tuple[Tensor, Tensor]]: Each item in result_list is 2-tuple. + The first item is an (n, 5) tensor, where the first 4 columns + are bounding box positions (tl_x, tl_y, br_x, br_y) and the + 5-th column is a score between 0 and 1. The second item is a + (n,) tensor where each item is the predicted class label of + the corresponding box. + """ + assert len(cls_scores) == len(bbox_preds) == len(bbox_preds_refine) + num_levels = len(cls_scores) + + featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] + mlvl_points = self.get_points(featmap_sizes, bbox_preds[0].dtype, + bbox_preds[0].device) + result_list = [] + for img_id in range(len(img_metas)): + cls_score_list = [ + cls_scores[i][img_id].detach() for i in range(num_levels) + ] + bbox_pred_list = [ + bbox_preds_refine[i][img_id].detach() + for i in range(num_levels) + ] + img_shape = img_metas[img_id]['img_shape'] + scale_factor = img_metas[img_id]['scale_factor'] + det_bboxes = self._get_bboxes_single(cls_score_list, + bbox_pred_list, mlvl_points, + img_shape, scale_factor, cfg, + rescale, with_nms) + result_list.append(det_bboxes) + return result_list + + def _get_bboxes_single(self, + cls_scores, + bbox_preds, + mlvl_points, + img_shape, + scale_factor, + cfg, + rescale=False, + with_nms=True): + """Transform outputs for a single batch item into bbox predictions. + + Args: + cls_scores (list[Tensor]): Box iou-aware scores for a single scale + level with shape (num_points * num_classes, H, W). + bbox_preds (list[Tensor]): Box offsets for a single scale + level with shape (num_points * 4, H, W). + mlvl_points (list[Tensor]): Box reference for a single scale level + with shape (num_total_points, 4). + img_shape (tuple[int]): Shape of the input image, + (height, width, 3). + scale_factor (ndarray): Scale factor of the image arrange as + (w_scale, h_scale, w_scale, h_scale). + cfg (mmcv.Config | None): Test / postprocessing configuration, + if None, test_cfg would be used. + rescale (bool): If True, return boxes in original image space. + Default: False. + with_nms (bool): If True, do nms before returning boxes. + Default: True. + + Returns: + tuple(Tensor): + det_bboxes (Tensor): BBox predictions in shape (n, 5), where + the first 4 columns are bounding box positions + (tl_x, tl_y, br_x, br_y) and the 5-th column is a score + between 0 and 1. + det_labels (Tensor): A (n,) tensor where each item is the + predicted class label of the corresponding box. + """ + cfg = self.test_cfg if cfg is None else cfg + assert len(cls_scores) == len(bbox_preds) == len(mlvl_points) + mlvl_bboxes = [] + mlvl_scores = [] + for cls_score, bbox_pred, points in zip(cls_scores, bbox_preds, + mlvl_points): + assert cls_score.size()[-2:] == bbox_pred.size()[-2:] + scores = cls_score.permute(1, 2, 0).reshape( + -1, self.cls_out_channels).contiguous().sigmoid() + bbox_pred = bbox_pred.permute(1, 2, 0).reshape(-1, 4).contiguous() + + nms_pre = cfg.get('nms_pre', -1) + if 0 < nms_pre < scores.shape[0]: + max_scores, _ = scores.max(dim=1) + _, topk_inds = max_scores.topk(nms_pre) + points = points[topk_inds, :] + bbox_pred = bbox_pred[topk_inds, :] + scores = scores[topk_inds, :] + bboxes = distance2bbox(points, bbox_pred, max_shape=img_shape) + mlvl_bboxes.append(bboxes) + mlvl_scores.append(scores) + mlvl_bboxes = torch.cat(mlvl_bboxes) + if rescale: + mlvl_bboxes /= mlvl_bboxes.new_tensor(scale_factor) + mlvl_scores = torch.cat(mlvl_scores) + padding = mlvl_scores.new_zeros(mlvl_scores.shape[0], 1) + # remind that we set FG labels to [0, num_class-1] since mmdet v2.0 + # BG cat_id: num_class + mlvl_scores = torch.cat([mlvl_scores, padding], dim=1) + if with_nms: + det_bboxes, det_labels = multiclass_nms(mlvl_bboxes, mlvl_scores, + cfg.score_thr, cfg.nms, + cfg.max_per_img) + return det_bboxes, det_labels + else: + return mlvl_bboxes, mlvl_scores + + def _get_points_single(self, + featmap_size, + stride, + dtype, + device, + flatten=False): + """Get points according to feature map sizes.""" + h, w = featmap_size + x_range = torch.arange( + 0, w * stride, stride, dtype=dtype, device=device) + y_range = torch.arange( + 0, h * stride, stride, dtype=dtype, device=device) + y, x = torch.meshgrid(y_range, x_range) + # to be compatible with anchor points in ATSS + if self.use_atss: + points = torch.stack( + (x.reshape(-1), y.reshape(-1)), dim=-1) + \ + stride * self.anchor_center_offset + else: + points = torch.stack( + (x.reshape(-1), y.reshape(-1)), dim=-1) + stride // 2 + return points + + def get_targets(self, cls_scores, mlvl_points, gt_bboxes, gt_labels, + img_metas, gt_bboxes_ignore): + """A wrapper for computing ATSS and FCOS targets for points in multiple + images. + + Args: + cls_scores (list[Tensor]): Box iou-aware scores for each scale + level with shape (N, num_points * num_classes, H, W). + mlvl_points (list[Tensor]): Points of each fpn level, each has + shape (num_points, 2). + gt_bboxes (list[Tensor]): Ground truth bboxes of each image, + each has shape (num_gt, 4). + gt_labels (list[Tensor]): Ground truth labels of each box, + each has shape (num_gt,). + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + gt_bboxes_ignore (None | Tensor): Ground truth bboxes to be + ignored, shape (num_ignored_gts, 4). + + Returns: + tuple: + labels_list (list[Tensor]): Labels of each level. + label_weights (Tensor/None): Label weights of all levels. + bbox_targets_list (list[Tensor]): Regression targets of each + level, (l, t, r, b). + bbox_weights (Tensor/None): Bbox weights of all levels. + """ + if self.use_atss: + return self.get_atss_targets(cls_scores, mlvl_points, gt_bboxes, + gt_labels, img_metas, + gt_bboxes_ignore) + else: + self.norm_on_bbox = False + return self.get_fcos_targets(mlvl_points, gt_bboxes, gt_labels) + + def _get_target_single(self, *args, **kwargs): + """Avoid ambiguity in multiple inheritance.""" + if self.use_atss: + return ATSSHead._get_target_single(self, *args, **kwargs) + else: + return FCOSHead._get_target_single(self, *args, **kwargs) + + def get_fcos_targets(self, points, gt_bboxes_list, gt_labels_list): + """Compute FCOS regression and classification targets for points in + multiple images. + + Args: + points (list[Tensor]): Points of each fpn level, each has shape + (num_points, 2). + gt_bboxes_list (list[Tensor]): Ground truth bboxes of each image, + each has shape (num_gt, 4). + gt_labels_list (list[Tensor]): Ground truth labels of each box, + each has shape (num_gt,). + + Returns: + tuple: + labels (list[Tensor]): Labels of each level. + label_weights: None, to be compatible with ATSS targets. + bbox_targets (list[Tensor]): BBox targets of each level. + bbox_weights: None, to be compatible with ATSS targets. + """ + labels, bbox_targets = FCOSHead.get_targets(self, points, + gt_bboxes_list, + gt_labels_list) + label_weights = None + bbox_weights = None + return labels, label_weights, bbox_targets, bbox_weights + + def get_atss_targets(self, + cls_scores, + mlvl_points, + gt_bboxes, + gt_labels, + img_metas, + gt_bboxes_ignore=None): + """A wrapper for computing ATSS targets for points in multiple images. + + Args: + cls_scores (list[Tensor]): Box iou-aware scores for each scale + level with shape (N, num_points * num_classes, H, W). + mlvl_points (list[Tensor]): Points of each fpn level, each has + shape (num_points, 2). + gt_bboxes (list[Tensor]): Ground truth bboxes of each image, + each has shape (num_gt, 4). + gt_labels (list[Tensor]): Ground truth labels of each box, + each has shape (num_gt,). + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + gt_bboxes_ignore (None | Tensor): Ground truth bboxes to be + ignored, shape (num_ignored_gts, 4). Default: None. + + Returns: + tuple: + labels_list (list[Tensor]): Labels of each level. + label_weights (Tensor): Label weights of all levels. + bbox_targets_list (list[Tensor]): Regression targets of each + level, (l, t, r, b). + bbox_weights (Tensor): Bbox weights of all levels. + """ + featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] + assert len(featmap_sizes) == self.anchor_generator.num_levels + + device = cls_scores[0].device + anchor_list, valid_flag_list = self.get_anchors( + featmap_sizes, img_metas, device=device) + label_channels = self.cls_out_channels if self.use_sigmoid_cls else 1 + + cls_reg_targets = ATSSHead.get_targets( + self, + anchor_list, + valid_flag_list, + gt_bboxes, + img_metas, + gt_bboxes_ignore_list=gt_bboxes_ignore, + gt_labels_list=gt_labels, + label_channels=label_channels, + unmap_outputs=True) + if cls_reg_targets is None: + return None + + (anchor_list, labels_list, label_weights_list, bbox_targets_list, + bbox_weights_list, num_total_pos, num_total_neg) = cls_reg_targets + + bbox_targets_list = [ + bbox_targets.reshape(-1, 4) for bbox_targets in bbox_targets_list + ] + + num_imgs = len(img_metas) + # transform bbox_targets (x1, y1, x2, y2) into (l, t, r, b) format + bbox_targets_list = self.transform_bbox_targets( + bbox_targets_list, mlvl_points, num_imgs) + + labels_list = [labels.reshape(-1) for labels in labels_list] + label_weights_list = [ + label_weights.reshape(-1) for label_weights in label_weights_list + ] + bbox_weights_list = [ + bbox_weights.reshape(-1) for bbox_weights in bbox_weights_list + ] + label_weights = torch.cat(label_weights_list) + bbox_weights = torch.cat(bbox_weights_list) + return labels_list, label_weights, bbox_targets_list, bbox_weights + + def transform_bbox_targets(self, decoded_bboxes, mlvl_points, num_imgs): + """Transform bbox_targets (x1, y1, x2, y2) into (l, t, r, b) format. + + Args: + decoded_bboxes (list[Tensor]): Regression targets of each level, + in the form of (x1, y1, x2, y2). + mlvl_points (list[Tensor]): Points of each fpn level, each has + shape (num_points, 2). + num_imgs (int): the number of images in a batch. + + Returns: + bbox_targets (list[Tensor]): Regression targets of each level in + the form of (l, t, r, b). + """ + # TODO: Re-implemented in Class PointCoder + assert len(decoded_bboxes) == len(mlvl_points) + num_levels = len(decoded_bboxes) + mlvl_points = [points.repeat(num_imgs, 1) for points in mlvl_points] + bbox_targets = [] + for i in range(num_levels): + bbox_target = bbox2distance(mlvl_points[i], decoded_bboxes[i]) + bbox_targets.append(bbox_target) + + return bbox_targets + + def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, + missing_keys, unexpected_keys, error_msgs): + """Override the method in the parent class to avoid changing para's + name.""" + pass diff --git a/detection/mmdet/models/dense_heads/yolact_head.py b/detection/mmdet/models/dense_heads/yolact_head.py new file mode 100644 index 0000000..10d311f --- /dev/null +++ b/detection/mmdet/models/dense_heads/yolact_head.py @@ -0,0 +1,943 @@ +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import ConvModule, xavier_init +from mmcv.runner import force_fp32 + +from mmdet.core import build_sampler, fast_nms, images_to_levels, multi_apply +from ..builder import HEADS, build_loss +from .anchor_head import AnchorHead + + +@HEADS.register_module() +class YOLACTHead(AnchorHead): + """YOLACT box head used in https://arxiv.org/abs/1904.02689. + + Note that YOLACT head is a light version of RetinaNet head. + Four differences are described as follows: + + 1. YOLACT box head has three-times fewer anchors. + 2. YOLACT box head shares the convs for box and cls branches. + 3. YOLACT box head uses OHEM instead of Focal loss. + 4. YOLACT box head predicts a set of mask coefficients for each box. + + Args: + num_classes (int): Number of categories excluding the background + category. + in_channels (int): Number of channels in the input feature map. + anchor_generator (dict): Config dict for anchor generator + loss_cls (dict): Config of classification loss. + loss_bbox (dict): Config of localization loss. + num_head_convs (int): Number of the conv layers shared by + box and cls branches. + num_protos (int): Number of the mask coefficients. + use_ohem (bool): If true, ``loss_single_OHEM`` will be used for + cls loss calculation. If false, ``loss_single`` will be used. + conv_cfg (dict): Dictionary to construct and config conv layer. + norm_cfg (dict): Dictionary to construct and config norm layer. + """ + + def __init__(self, + num_classes, + in_channels, + anchor_generator=dict( + type='AnchorGenerator', + octave_base_scale=3, + scales_per_octave=1, + ratios=[0.5, 1.0, 2.0], + strides=[8, 16, 32, 64, 128]), + loss_cls=dict( + type='CrossEntropyLoss', + use_sigmoid=False, + reduction='none', + loss_weight=1.0), + loss_bbox=dict( + type='SmoothL1Loss', beta=1.0, loss_weight=1.5), + num_head_convs=1, + num_protos=32, + use_ohem=True, + conv_cfg=None, + norm_cfg=None, + **kwargs): + self.num_head_convs = num_head_convs + self.num_protos = num_protos + self.use_ohem = use_ohem + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + super(YOLACTHead, self).__init__( + num_classes, + in_channels, + loss_cls=loss_cls, + loss_bbox=loss_bbox, + anchor_generator=anchor_generator, + **kwargs) + if self.use_ohem: + sampler_cfg = dict(type='PseudoSampler') + self.sampler = build_sampler(sampler_cfg, context=self) + self.sampling = False + + def _init_layers(self): + """Initialize layers of the head.""" + self.relu = nn.ReLU(inplace=True) + self.head_convs = nn.ModuleList() + for i in range(self.num_head_convs): + chn = self.in_channels if i == 0 else self.feat_channels + self.head_convs.append( + ConvModule( + chn, + self.feat_channels, + 3, + stride=1, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg)) + self.conv_cls = nn.Conv2d( + self.feat_channels, + self.num_anchors * self.cls_out_channels, + 3, + padding=1) + self.conv_reg = nn.Conv2d( + self.feat_channels, self.num_anchors * 4, 3, padding=1) + self.conv_coeff = nn.Conv2d( + self.feat_channels, + self.num_anchors * self.num_protos, + 3, + padding=1) + + def init_weights(self): + """Initialize weights of the head.""" + for m in self.head_convs: + xavier_init(m.conv, distribution='uniform', bias=0) + xavier_init(self.conv_cls, distribution='uniform', bias=0) + xavier_init(self.conv_reg, distribution='uniform', bias=0) + xavier_init(self.conv_coeff, distribution='uniform', bias=0) + + def forward_single(self, x): + """Forward feature of a single scale level. + + Args: + x (Tensor): Features of a single scale level. + + Returns: + tuple: + cls_score (Tensor): Cls scores for a single scale level \ + the channels number is num_anchors * num_classes. + bbox_pred (Tensor): Box energies / deltas for a single scale \ + level, the channels number is num_anchors * 4. + coeff_pred (Tensor): Mask coefficients for a single scale \ + level, the channels number is num_anchors * num_protos. + """ + for head_conv in self.head_convs: + x = head_conv(x) + cls_score = self.conv_cls(x) + bbox_pred = self.conv_reg(x) + coeff_pred = self.conv_coeff(x).tanh() + return cls_score, bbox_pred, coeff_pred + + @force_fp32(apply_to=('cls_scores', 'bbox_preds')) + def loss(self, + cls_scores, + bbox_preds, + gt_bboxes, + gt_labels, + img_metas, + gt_bboxes_ignore=None): + """A combination of the func:``AnchorHead.loss`` and + func:``SSDHead.loss``. + + When ``self.use_ohem == True``, it functions like ``SSDHead.loss``, + otherwise, it follows ``AnchorHead.loss``. Besides, it additionally + returns ``sampling_results``. + + Args: + cls_scores (list[Tensor]): Box scores for each scale level + Has shape (N, num_anchors * num_classes, H, W) + bbox_preds (list[Tensor]): Box energies / deltas for each scale + level with shape (N, num_anchors * 4, H, W) + gt_bboxes (list[Tensor]): Ground truth bboxes for each image with + shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + gt_labels (list[Tensor]): Class indices corresponding to each box + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + gt_bboxes_ignore (None | list[Tensor]): Specify which bounding + boxes can be ignored when computing the loss. Default: None + + Returns: + tuple: + dict[str, Tensor]: A dictionary of loss components. + List[:obj:``SamplingResult``]: Sampler results for each image. + """ + featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] + assert len(featmap_sizes) == self.anchor_generator.num_levels + + device = cls_scores[0].device + + anchor_list, valid_flag_list = self.get_anchors( + featmap_sizes, img_metas, device=device) + label_channels = self.cls_out_channels if self.use_sigmoid_cls else 1 + cls_reg_targets = self.get_targets( + anchor_list, + valid_flag_list, + gt_bboxes, + img_metas, + gt_bboxes_ignore_list=gt_bboxes_ignore, + gt_labels_list=gt_labels, + label_channels=label_channels, + unmap_outputs=not self.use_ohem, + return_sampling_results=True) + if cls_reg_targets is None: + return None + (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, + num_total_pos, num_total_neg, sampling_results) = cls_reg_targets + + if self.use_ohem: + num_images = len(img_metas) + all_cls_scores = torch.cat([ + s.permute(0, 2, 3, 1).reshape( + num_images, -1, self.cls_out_channels) for s in cls_scores + ], 1) + all_labels = torch.cat(labels_list, -1).view(num_images, -1) + all_label_weights = torch.cat(label_weights_list, + -1).view(num_images, -1) + all_bbox_preds = torch.cat([ + b.permute(0, 2, 3, 1).reshape(num_images, -1, 4) + for b in bbox_preds + ], -2) + all_bbox_targets = torch.cat(bbox_targets_list, + -2).view(num_images, -1, 4) + all_bbox_weights = torch.cat(bbox_weights_list, + -2).view(num_images, -1, 4) + + # concat all level anchors to a single tensor + all_anchors = [] + for i in range(num_images): + all_anchors.append(torch.cat(anchor_list[i])) + + # check NaN and Inf + assert torch.isfinite(all_cls_scores).all().item(), \ + 'classification scores become infinite or NaN!' + assert torch.isfinite(all_bbox_preds).all().item(), \ + 'bbox predications become infinite or NaN!' + + losses_cls, losses_bbox = multi_apply( + self.loss_single_OHEM, + all_cls_scores, + all_bbox_preds, + all_anchors, + all_labels, + all_label_weights, + all_bbox_targets, + all_bbox_weights, + num_total_samples=num_total_pos) + else: + num_total_samples = ( + num_total_pos + + num_total_neg if self.sampling else num_total_pos) + + # anchor number of multi levels + num_level_anchors = [anchors.size(0) for anchors in anchor_list[0]] + # concat all level anchors and flags to a single tensor + concat_anchor_list = [] + for i in range(len(anchor_list)): + concat_anchor_list.append(torch.cat(anchor_list[i])) + all_anchor_list = images_to_levels(concat_anchor_list, + num_level_anchors) + losses_cls, losses_bbox = multi_apply( + self.loss_single, + cls_scores, + bbox_preds, + all_anchor_list, + labels_list, + label_weights_list, + bbox_targets_list, + bbox_weights_list, + num_total_samples=num_total_samples) + + return dict( + loss_cls=losses_cls, loss_bbox=losses_bbox), sampling_results + + def loss_single_OHEM(self, cls_score, bbox_pred, anchors, labels, + label_weights, bbox_targets, bbox_weights, + num_total_samples): + """"See func:``SSDHead.loss``.""" + loss_cls_all = self.loss_cls(cls_score, labels, label_weights) + + # FG cat_id: [0, num_classes -1], BG cat_id: num_classes + pos_inds = ((labels >= 0) & (labels < self.num_classes)).nonzero( + as_tuple=False).reshape(-1) + neg_inds = (labels == self.num_classes).nonzero( + as_tuple=False).view(-1) + + num_pos_samples = pos_inds.size(0) + if num_pos_samples == 0: + num_neg_samples = neg_inds.size(0) + else: + num_neg_samples = self.train_cfg.neg_pos_ratio * num_pos_samples + if num_neg_samples > neg_inds.size(0): + num_neg_samples = neg_inds.size(0) + topk_loss_cls_neg, _ = loss_cls_all[neg_inds].topk(num_neg_samples) + loss_cls_pos = loss_cls_all[pos_inds].sum() + loss_cls_neg = topk_loss_cls_neg.sum() + loss_cls = (loss_cls_pos + loss_cls_neg) / num_total_samples + if self.reg_decoded_bbox: + # When the regression loss (e.g. `IouLoss`, `GIouLoss`) + # is applied directly on the decoded bounding boxes, it + # decodes the already encoded coordinates to absolute format. + bbox_pred = self.bbox_coder.decode(anchors, bbox_pred) + loss_bbox = self.loss_bbox( + bbox_pred, + bbox_targets, + bbox_weights, + avg_factor=num_total_samples) + return loss_cls[None], loss_bbox + + @force_fp32(apply_to=('cls_scores', 'bbox_preds', 'coeff_preds')) + def get_bboxes(self, + cls_scores, + bbox_preds, + coeff_preds, + img_metas, + cfg=None, + rescale=False): + """"Similiar to func:``AnchorHead.get_bboxes``, but additionally + processes coeff_preds. + + Args: + cls_scores (list[Tensor]): Box scores for each scale level + with shape (N, num_anchors * num_classes, H, W) + bbox_preds (list[Tensor]): Box energies / deltas for each scale + level with shape (N, num_anchors * 4, H, W) + coeff_preds (list[Tensor]): Mask coefficients for each scale + level with shape (N, num_anchors * num_protos, H, W) + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + cfg (mmcv.Config | None): Test / postprocessing configuration, + if None, test_cfg would be used + rescale (bool): If True, return boxes in original image space. + Default: False. + + Returns: + list[tuple[Tensor, Tensor, Tensor]]: Each item in result_list is + a 3-tuple. The first item is an (n, 5) tensor, where the + first 4 columns are bounding box positions + (tl_x, tl_y, br_x, br_y) and the 5-th column is a score + between 0 and 1. The second item is an (n,) tensor where each + item is the predicted class label of the corresponding box. + The third item is an (n, num_protos) tensor where each item + is the predicted mask coefficients of instance inside the + corresponding box. + """ + assert len(cls_scores) == len(bbox_preds) + num_levels = len(cls_scores) + + device = cls_scores[0].device + featmap_sizes = [cls_scores[i].shape[-2:] for i in range(num_levels)] + mlvl_anchors = self.anchor_generator.grid_anchors( + featmap_sizes, device=device) + + det_bboxes = [] + det_labels = [] + det_coeffs = [] + for img_id in range(len(img_metas)): + cls_score_list = [ + cls_scores[i][img_id].detach() for i in range(num_levels) + ] + bbox_pred_list = [ + bbox_preds[i][img_id].detach() for i in range(num_levels) + ] + coeff_pred_list = [ + coeff_preds[i][img_id].detach() for i in range(num_levels) + ] + img_shape = img_metas[img_id]['img_shape'] + scale_factor = img_metas[img_id]['scale_factor'] + bbox_res = self._get_bboxes_single(cls_score_list, bbox_pred_list, + coeff_pred_list, mlvl_anchors, + img_shape, scale_factor, cfg, + rescale) + det_bboxes.append(bbox_res[0]) + det_labels.append(bbox_res[1]) + det_coeffs.append(bbox_res[2]) + return det_bboxes, det_labels, det_coeffs + + def _get_bboxes_single(self, + cls_score_list, + bbox_pred_list, + coeff_preds_list, + mlvl_anchors, + img_shape, + scale_factor, + cfg, + rescale=False): + """"Similiar to func:``AnchorHead._get_bboxes_single``, but + additionally processes coeff_preds_list and uses fast NMS instead of + traditional NMS. + + Args: + cls_score_list (list[Tensor]): Box scores for a single scale level + Has shape (num_anchors * num_classes, H, W). + bbox_pred_list (list[Tensor]): Box energies / deltas for a single + scale level with shape (num_anchors * 4, H, W). + coeff_preds_list (list[Tensor]): Mask coefficients for a single + scale level with shape (num_anchors * num_protos, H, W). + mlvl_anchors (list[Tensor]): Box reference for a single scale level + with shape (num_total_anchors, 4). + img_shape (tuple[int]): Shape of the input image, + (height, width, 3). + scale_factor (ndarray): Scale factor of the image arange as + (w_scale, h_scale, w_scale, h_scale). + cfg (mmcv.Config): Test / postprocessing configuration, + if None, test_cfg would be used. + rescale (bool): If True, return boxes in original image space. + + Returns: + tuple[Tensor, Tensor, Tensor]: The first item is an (n, 5) tensor, + where the first 4 columns are bounding box positions + (tl_x, tl_y, br_x, br_y) and the 5-th column is a score between + 0 and 1. The second item is an (n,) tensor where each item is + the predicted class label of the corresponding box. The third + item is an (n, num_protos) tensor where each item is the + predicted mask coefficients of instance inside the + corresponding box. + """ + cfg = self.test_cfg if cfg is None else cfg + assert len(cls_score_list) == len(bbox_pred_list) == len(mlvl_anchors) + mlvl_bboxes = [] + mlvl_scores = [] + mlvl_coeffs = [] + for cls_score, bbox_pred, coeff_pred, anchors in \ + zip(cls_score_list, bbox_pred_list, + coeff_preds_list, mlvl_anchors): + assert cls_score.size()[-2:] == bbox_pred.size()[-2:] + cls_score = cls_score.permute(1, 2, + 0).reshape(-1, self.cls_out_channels) + if self.use_sigmoid_cls: + scores = cls_score.sigmoid() + else: + scores = cls_score.softmax(-1) + bbox_pred = bbox_pred.permute(1, 2, 0).reshape(-1, 4) + coeff_pred = coeff_pred.permute(1, 2, + 0).reshape(-1, self.num_protos) + nms_pre = cfg.get('nms_pre', -1) + if nms_pre > 0 and scores.shape[0] > nms_pre: + # Get maximum scores for foreground classes. + if self.use_sigmoid_cls: + max_scores, _ = scores.max(dim=1) + else: + # remind that we set FG labels to [0, num_class-1] + # since mmdet v2.0 + # BG cat_id: num_class + max_scores, _ = scores[:, :-1].max(dim=1) + _, topk_inds = max_scores.topk(nms_pre) + anchors = anchors[topk_inds, :] + bbox_pred = bbox_pred[topk_inds, :] + scores = scores[topk_inds, :] + coeff_pred = coeff_pred[topk_inds, :] + bboxes = self.bbox_coder.decode( + anchors, bbox_pred, max_shape=img_shape) + mlvl_bboxes.append(bboxes) + mlvl_scores.append(scores) + mlvl_coeffs.append(coeff_pred) + mlvl_bboxes = torch.cat(mlvl_bboxes) + if rescale: + mlvl_bboxes /= mlvl_bboxes.new_tensor(scale_factor) + mlvl_scores = torch.cat(mlvl_scores) + mlvl_coeffs = torch.cat(mlvl_coeffs) + if self.use_sigmoid_cls: + # Add a dummy background class to the backend when using sigmoid + # remind that we set FG labels to [0, num_class-1] since mmdet v2.0 + # BG cat_id: num_class + padding = mlvl_scores.new_zeros(mlvl_scores.shape[0], 1) + mlvl_scores = torch.cat([mlvl_scores, padding], dim=1) + det_bboxes, det_labels, det_coeffs = fast_nms(mlvl_bboxes, mlvl_scores, + mlvl_coeffs, + cfg.score_thr, + cfg.iou_thr, cfg.top_k, + cfg.max_per_img) + return det_bboxes, det_labels, det_coeffs + + +@HEADS.register_module() +class YOLACTSegmHead(nn.Module): + """YOLACT segmentation head used in https://arxiv.org/abs/1904.02689. + + Apply a semantic segmentation loss on feature space using layers that are + only evaluated during training to increase performance with no speed + penalty. + + Args: + in_channels (int): Number of channels in the input feature map. + num_classes (int): Number of categories excluding the background + category. + loss_segm (dict): Config of semantic segmentation loss. + """ + + def __init__(self, + num_classes, + in_channels=256, + loss_segm=dict( + type='CrossEntropyLoss', + use_sigmoid=True, + loss_weight=1.0)): + super(YOLACTSegmHead, self).__init__() + self.in_channels = in_channels + self.num_classes = num_classes + self.loss_segm = build_loss(loss_segm) + self._init_layers() + self.fp16_enabled = False + + def _init_layers(self): + """Initialize layers of the head.""" + self.segm_conv = nn.Conv2d( + self.in_channels, self.num_classes, kernel_size=1) + + def init_weights(self): + """Initialize weights of the head.""" + xavier_init(self.segm_conv, distribution='uniform') + + def forward(self, x): + """Forward feature from the upstream network. + + Args: + x (Tensor): Feature from the upstream network, which is + a 4D-tensor. + + Returns: + Tensor: Predicted semantic segmentation map with shape + (N, num_classes, H, W). + """ + return self.segm_conv(x) + + @force_fp32(apply_to=('segm_pred', )) + def loss(self, segm_pred, gt_masks, gt_labels): + """Compute loss of the head. + + Args: + segm_pred (list[Tensor]): Predicted semantic segmentation map + with shape (N, num_classes, H, W). + gt_masks (list[Tensor]): Ground truth masks for each image with + the same shape of the input image. + gt_labels (list[Tensor]): Class indices corresponding to each box. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + loss_segm = [] + num_imgs, num_classes, mask_h, mask_w = segm_pred.size() + for idx in range(num_imgs): + cur_segm_pred = segm_pred[idx] + cur_gt_masks = gt_masks[idx].float() + cur_gt_labels = gt_labels[idx] + segm_targets = self.get_targets(cur_segm_pred, cur_gt_masks, + cur_gt_labels) + if segm_targets is None: + loss = self.loss_segm(cur_segm_pred, + torch.zeros_like(cur_segm_pred), + torch.zeros_like(cur_segm_pred)) + else: + loss = self.loss_segm( + cur_segm_pred, + segm_targets, + avg_factor=num_imgs * mask_h * mask_w) + loss_segm.append(loss) + return dict(loss_segm=loss_segm) + + def get_targets(self, segm_pred, gt_masks, gt_labels): + """Compute semantic segmentation targets for each image. + + Args: + segm_pred (Tensor): Predicted semantic segmentation map + with shape (num_classes, H, W). + gt_masks (Tensor): Ground truth masks for each image with + the same shape of the input image. + gt_labels (Tensor): Class indices corresponding to each box. + + Returns: + Tensor: Semantic segmentation targets with shape + (num_classes, H, W). + """ + if gt_masks.size(0) == 0: + return None + num_classes, mask_h, mask_w = segm_pred.size() + with torch.no_grad(): + downsampled_masks = F.interpolate( + gt_masks.unsqueeze(0), (mask_h, mask_w), + mode='bilinear', + align_corners=False).squeeze(0) + downsampled_masks = downsampled_masks.gt(0.5).float() + segm_targets = torch.zeros_like(segm_pred, requires_grad=False) + for obj_idx in range(downsampled_masks.size(0)): + segm_targets[gt_labels[obj_idx] - 1] = torch.max( + segm_targets[gt_labels[obj_idx] - 1], + downsampled_masks[obj_idx]) + return segm_targets + + +@HEADS.register_module() +class YOLACTProtonet(nn.Module): + """YOLACT mask head used in https://arxiv.org/abs/1904.02689. + + This head outputs the mask prototypes for YOLACT. + + Args: + in_channels (int): Number of channels in the input feature map. + proto_channels (tuple[int]): Output channels of protonet convs. + proto_kernel_sizes (tuple[int]): Kernel sizes of protonet convs. + include_last_relu (Bool): If keep the last relu of protonet. + num_protos (int): Number of prototypes. + num_classes (int): Number of categories excluding the background + category. + loss_mask_weight (float): Reweight the mask loss by this factor. + max_masks_to_train (int): Maximum number of masks to train for + each image. + """ + + def __init__(self, + num_classes, + in_channels=256, + proto_channels=(256, 256, 256, None, 256, 32), + proto_kernel_sizes=(3, 3, 3, -2, 3, 1), + include_last_relu=True, + num_protos=32, + loss_mask_weight=1.0, + max_masks_to_train=100): + super(YOLACTProtonet, self).__init__() + self.in_channels = in_channels + self.proto_channels = proto_channels + self.proto_kernel_sizes = proto_kernel_sizes + self.include_last_relu = include_last_relu + self.protonet = self._init_layers() + + self.loss_mask_weight = loss_mask_weight + self.num_protos = num_protos + self.num_classes = num_classes + self.max_masks_to_train = max_masks_to_train + self.fp16_enabled = False + + def _init_layers(self): + """A helper function to take a config setting and turn it into a + network.""" + # Possible patterns: + # ( 256, 3) -> conv + # ( 256,-2) -> deconv + # (None,-2) -> bilinear interpolate + in_channels = self.in_channels + protonets = nn.ModuleList() + for num_channels, kernel_size in zip(self.proto_channels, + self.proto_kernel_sizes): + if kernel_size > 0: + layer = nn.Conv2d( + in_channels, + num_channels, + kernel_size, + padding=kernel_size // 2) + else: + if num_channels is None: + layer = InterpolateModule( + scale_factor=-kernel_size, + mode='bilinear', + align_corners=False) + else: + layer = nn.ConvTranspose2d( + in_channels, + num_channels, + -kernel_size, + padding=kernel_size // 2) + protonets.append(layer) + protonets.append(nn.ReLU(inplace=True)) + in_channels = num_channels if num_channels is not None \ + else in_channels + if not self.include_last_relu: + protonets = protonets[:-1] + return nn.Sequential(*protonets) + + def init_weights(self): + """Initialize weights of the head.""" + for m in self.protonet: + if isinstance(m, nn.Conv2d): + xavier_init(m, distribution='uniform') + + def forward(self, x, coeff_pred, bboxes, img_meta, sampling_results=None): + """Forward feature from the upstream network to get prototypes and + linearly combine the prototypes, using masks coefficients, into + instance masks. Finally, crop the instance masks with given bboxes. + + Args: + x (Tensor): Feature from the upstream network, which is + a 4D-tensor. + coeff_pred (list[Tensor]): Mask coefficients for each scale + level with shape (N, num_anchors * num_protos, H, W). + bboxes (list[Tensor]): Box used for cropping with shape + (N, num_anchors * 4, H, W). During training, they are + ground truth boxes. During testing, they are predicted + boxes. + img_meta (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + sampling_results (List[:obj:``SamplingResult``]): Sampler results + for each image. + + Returns: + list[Tensor]: Predicted instance segmentation masks. + """ + prototypes = self.protonet(x) + prototypes = prototypes.permute(0, 2, 3, 1).contiguous() + + num_imgs = x.size(0) + # Training state + if self.training: + coeff_pred_list = [] + for coeff_pred_per_level in coeff_pred: + coeff_pred_per_level = \ + coeff_pred_per_level.permute(0, 2, 3, 1)\ + .reshape(num_imgs, -1, self.num_protos) + coeff_pred_list.append(coeff_pred_per_level) + coeff_pred = torch.cat(coeff_pred_list, dim=1) + + mask_pred_list = [] + for idx in range(num_imgs): + cur_prototypes = prototypes[idx] + cur_coeff_pred = coeff_pred[idx] + cur_bboxes = bboxes[idx] + cur_img_meta = img_meta[idx] + + # Testing state + if not self.training: + bboxes_for_cropping = cur_bboxes + else: + cur_sampling_results = sampling_results[idx] + pos_assigned_gt_inds = \ + cur_sampling_results.pos_assigned_gt_inds + bboxes_for_cropping = cur_bboxes[pos_assigned_gt_inds].clone() + pos_inds = cur_sampling_results.pos_inds + cur_coeff_pred = cur_coeff_pred[pos_inds] + + # Linearly combine the prototypes with the mask coefficients + mask_pred = cur_prototypes @ cur_coeff_pred.t() + mask_pred = torch.sigmoid(mask_pred) + + h, w = cur_img_meta['img_shape'][:2] + bboxes_for_cropping[:, 0] /= w + bboxes_for_cropping[:, 1] /= h + bboxes_for_cropping[:, 2] /= w + bboxes_for_cropping[:, 3] /= h + + mask_pred = self.crop(mask_pred, bboxes_for_cropping) + mask_pred = mask_pred.permute(2, 0, 1).contiguous() + mask_pred_list.append(mask_pred) + return mask_pred_list + + @force_fp32(apply_to=('mask_pred', )) + def loss(self, mask_pred, gt_masks, gt_bboxes, img_meta, sampling_results): + """Compute loss of the head. + + Args: + mask_pred (list[Tensor]): Predicted prototypes with shape + (num_classes, H, W). + gt_masks (list[Tensor]): Ground truth masks for each image with + the same shape of the input image. + gt_bboxes (list[Tensor]): Ground truth bboxes for each image with + shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + img_meta (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + sampling_results (List[:obj:``SamplingResult``]): Sampler results + for each image. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + loss_mask = [] + num_imgs = len(mask_pred) + total_pos = 0 + for idx in range(num_imgs): + cur_mask_pred = mask_pred[idx] + cur_gt_masks = gt_masks[idx].float() + cur_gt_bboxes = gt_bboxes[idx] + cur_img_meta = img_meta[idx] + cur_sampling_results = sampling_results[idx] + + pos_assigned_gt_inds = cur_sampling_results.pos_assigned_gt_inds + num_pos = pos_assigned_gt_inds.size(0) + # Since we're producing (near) full image masks, + # it'd take too much vram to backprop on every single mask. + # Thus we select only a subset. + if num_pos > self.max_masks_to_train: + perm = torch.randperm(num_pos) + select = perm[:self.max_masks_to_train] + cur_mask_pred = cur_mask_pred[select] + pos_assigned_gt_inds = pos_assigned_gt_inds[select] + num_pos = self.max_masks_to_train + total_pos += num_pos + + gt_bboxes_for_reweight = cur_gt_bboxes[pos_assigned_gt_inds] + + mask_targets = self.get_targets(cur_mask_pred, cur_gt_masks, + pos_assigned_gt_inds) + if num_pos == 0: + loss = cur_mask_pred.sum() * 0. + elif mask_targets is None: + loss = F.binary_cross_entropy(cur_mask_pred, + torch.zeros_like(cur_mask_pred), + torch.zeros_like(cur_mask_pred)) + else: + cur_mask_pred = torch.clamp(cur_mask_pred, 0, 1) + loss = F.binary_cross_entropy( + cur_mask_pred, mask_targets, + reduction='none') * self.loss_mask_weight + + h, w = cur_img_meta['img_shape'][:2] + gt_bboxes_width = (gt_bboxes_for_reweight[:, 2] - + gt_bboxes_for_reweight[:, 0]) / w + gt_bboxes_height = (gt_bboxes_for_reweight[:, 3] - + gt_bboxes_for_reweight[:, 1]) / h + loss = loss.mean(dim=(1, + 2)) / gt_bboxes_width / gt_bboxes_height + loss = torch.sum(loss) + loss_mask.append(loss) + + if total_pos == 0: + total_pos += 1 # avoid nan + loss_mask = [x / total_pos for x in loss_mask] + + return dict(loss_mask=loss_mask) + + def get_targets(self, mask_pred, gt_masks, pos_assigned_gt_inds): + """Compute instance segmentation targets for each image. + + Args: + mask_pred (Tensor): Predicted prototypes with shape + (num_classes, H, W). + gt_masks (Tensor): Ground truth masks for each image with + the same shape of the input image. + pos_assigned_gt_inds (Tensor): GT indices of the corresponding + positive samples. + Returns: + Tensor: Instance segmentation targets with shape + (num_instances, H, W). + """ + if gt_masks.size(0) == 0: + return None + mask_h, mask_w = mask_pred.shape[-2:] + gt_masks = F.interpolate( + gt_masks.unsqueeze(0), (mask_h, mask_w), + mode='bilinear', + align_corners=False).squeeze(0) + gt_masks = gt_masks.gt(0.5).float() + mask_targets = gt_masks[pos_assigned_gt_inds] + return mask_targets + + def get_seg_masks(self, mask_pred, label_pred, img_meta, rescale): + """Resize, binarize, and format the instance mask predictions. + + Args: + mask_pred (Tensor): shape (N, H, W). + label_pred (Tensor): shape (N, ). + img_meta (dict): Meta information of each image, e.g., + image size, scaling factor, etc. + rescale (bool): If rescale is False, then returned masks will + fit the scale of imgs[0]. + Returns: + list[ndarray]: Mask predictions grouped by their predicted classes. + """ + ori_shape = img_meta['ori_shape'] + scale_factor = img_meta['scale_factor'] + if rescale: + img_h, img_w = ori_shape[:2] + else: + img_h = np.round(ori_shape[0] * scale_factor[1]).astype(np.int32) + img_w = np.round(ori_shape[1] * scale_factor[0]).astype(np.int32) + + cls_segms = [[] for _ in range(self.num_classes)] + if mask_pred.size(0) == 0: + return cls_segms + + mask_pred = F.interpolate( + mask_pred.unsqueeze(0), (img_h, img_w), + mode='bilinear', + align_corners=False).squeeze(0) > 0.5 + mask_pred = mask_pred.cpu().numpy().astype(np.uint8) + + for m, l in zip(mask_pred, label_pred): + cls_segms[l].append(m) + return cls_segms + + def crop(self, masks, boxes, padding=1): + """Crop predicted masks by zeroing out everything not in the predicted + bbox. + + Args: + masks (Tensor): shape [H, W, N]. + boxes (Tensor): bbox coords in relative point form with + shape [N, 4]. + + Return: + Tensor: The cropped masks. + """ + h, w, n = masks.size() + x1, x2 = self.sanitize_coordinates( + boxes[:, 0], boxes[:, 2], w, padding, cast=False) + y1, y2 = self.sanitize_coordinates( + boxes[:, 1], boxes[:, 3], h, padding, cast=False) + + rows = torch.arange( + w, device=masks.device, dtype=x1.dtype).view(1, -1, + 1).expand(h, w, n) + cols = torch.arange( + h, device=masks.device, dtype=x1.dtype).view(-1, 1, + 1).expand(h, w, n) + + masks_left = rows >= x1.view(1, 1, -1) + masks_right = rows < x2.view(1, 1, -1) + masks_up = cols >= y1.view(1, 1, -1) + masks_down = cols < y2.view(1, 1, -1) + + crop_mask = masks_left * masks_right * masks_up * masks_down + + return masks * crop_mask.float() + + def sanitize_coordinates(self, x1, x2, img_size, padding=0, cast=True): + """Sanitizes the input coordinates so that x1 < x2, x1 != x2, x1 >= 0, + and x2 <= image_size. Also converts from relative to absolute + coordinates and casts the results to long tensors. + + Warning: this does things in-place behind the scenes so + copy if necessary. + + Args: + _x1 (Tensor): shape (N, ). + _x2 (Tensor): shape (N, ). + img_size (int): Size of the input image. + padding (int): x1 >= padding, x2 <= image_size-padding. + cast (bool): If cast is false, the result won't be cast to longs. + + Returns: + tuple: + x1 (Tensor): Sanitized _x1. + x2 (Tensor): Sanitized _x2. + """ + x1 = x1 * img_size + x2 = x2 * img_size + if cast: + x1 = x1.long() + x2 = x2.long() + x1 = torch.min(x1, x2) + x2 = torch.max(x1, x2) + x1 = torch.clamp(x1 - padding, min=0) + x2 = torch.clamp(x2 + padding, max=img_size) + return x1, x2 + + +class InterpolateModule(nn.Module): + """This is a module version of F.interpolate. + + Any arguments you give it just get passed along for the ride. + """ + + def __init__(self, *args, **kwargs): + super().__init__() + + self.args = args + self.kwargs = kwargs + + def forward(self, x): + """Forward features from the upstream network.""" + return F.interpolate(x, *self.args, **self.kwargs) diff --git a/detection/mmdet/models/dense_heads/yolo_head.py b/detection/mmdet/models/dense_heads/yolo_head.py new file mode 100644 index 0000000..25a005d --- /dev/null +++ b/detection/mmdet/models/dense_heads/yolo_head.py @@ -0,0 +1,577 @@ +# Copyright (c) 2019 Western Digital Corporation or its affiliates. + +import warnings + +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import ConvModule, normal_init +from mmcv.runner import force_fp32 + +from mmdet.core import (build_anchor_generator, build_assigner, + build_bbox_coder, build_sampler, images_to_levels, + multi_apply, multiclass_nms) +from ..builder import HEADS, build_loss +from .base_dense_head import BaseDenseHead +from .dense_test_mixins import BBoxTestMixin + + +@HEADS.register_module() +class YOLOV3Head(BaseDenseHead, BBoxTestMixin): + """YOLOV3Head Paper link: https://arxiv.org/abs/1804.02767. + + Args: + num_classes (int): The number of object classes (w/o background) + in_channels (List[int]): Number of input channels per scale. + out_channels (List[int]): The number of output channels per scale + before the final 1x1 layer. Default: (1024, 512, 256). + anchor_generator (dict): Config dict for anchor generator + bbox_coder (dict): Config of bounding box coder. + featmap_strides (List[int]): The stride of each scale. + Should be in descending order. Default: (32, 16, 8). + one_hot_smoother (float): Set a non-zero value to enable label-smooth + Default: 0. + conv_cfg (dict): Config dict for convolution layer. Default: None. + norm_cfg (dict): Dictionary to construct and config norm layer. + Default: dict(type='BN', requires_grad=True) + act_cfg (dict): Config dict for activation layer. + Default: dict(type='LeakyReLU', negative_slope=0.1). + loss_cls (dict): Config of classification loss. + loss_conf (dict): Config of confidence loss. + loss_xy (dict): Config of xy coordinate loss. + loss_wh (dict): Config of wh coordinate loss. + train_cfg (dict): Training config of YOLOV3 head. Default: None. + test_cfg (dict): Testing config of YOLOV3 head. Default: None. + """ + + def __init__(self, + num_classes, + in_channels, + out_channels=(1024, 512, 256), + anchor_generator=dict( + type='YOLOAnchorGenerator', + base_sizes=[[(116, 90), (156, 198), (373, 326)], + [(30, 61), (62, 45), (59, 119)], + [(10, 13), (16, 30), (33, 23)]], + strides=[32, 16, 8]), + bbox_coder=dict(type='YOLOBBoxCoder'), + featmap_strides=[32, 16, 8], + one_hot_smoother=0., + conv_cfg=None, + norm_cfg=dict(type='BN', requires_grad=True), + act_cfg=dict(type='LeakyReLU', negative_slope=0.1), + loss_cls=dict( + type='CrossEntropyLoss', + use_sigmoid=True, + loss_weight=1.0), + loss_conf=dict( + type='CrossEntropyLoss', + use_sigmoid=True, + loss_weight=1.0), + loss_xy=dict( + type='CrossEntropyLoss', + use_sigmoid=True, + loss_weight=1.0), + loss_wh=dict(type='MSELoss', loss_weight=1.0), + train_cfg=None, + test_cfg=None): + super(YOLOV3Head, self).__init__() + # Check params + assert (len(in_channels) == len(out_channels) == len(featmap_strides)) + + self.num_classes = num_classes + self.in_channels = in_channels + self.out_channels = out_channels + self.featmap_strides = featmap_strides + self.train_cfg = train_cfg + self.test_cfg = test_cfg + if self.train_cfg: + self.assigner = build_assigner(self.train_cfg.assigner) + if hasattr(self.train_cfg, 'sampler'): + sampler_cfg = self.train_cfg.sampler + else: + sampler_cfg = dict(type='PseudoSampler') + self.sampler = build_sampler(sampler_cfg, context=self) + + self.one_hot_smoother = one_hot_smoother + + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + + self.bbox_coder = build_bbox_coder(bbox_coder) + self.anchor_generator = build_anchor_generator(anchor_generator) + + self.loss_cls = build_loss(loss_cls) + self.loss_conf = build_loss(loss_conf) + self.loss_xy = build_loss(loss_xy) + self.loss_wh = build_loss(loss_wh) + # usually the numbers of anchors for each level are the same + # except SSD detectors + self.num_anchors = self.anchor_generator.num_base_anchors[0] + assert len( + self.anchor_generator.num_base_anchors) == len(featmap_strides) + self._init_layers() + + @property + def num_levels(self): + return len(self.featmap_strides) + + @property + def num_attrib(self): + """int: number of attributes in pred_map, bboxes (4) + + objectness (1) + num_classes""" + + return 5 + self.num_classes + + def _init_layers(self): + self.convs_bridge = nn.ModuleList() + self.convs_pred = nn.ModuleList() + for i in range(self.num_levels): + conv_bridge = ConvModule( + self.in_channels[i], + self.out_channels[i], + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + conv_pred = nn.Conv2d(self.out_channels[i], + self.num_anchors * self.num_attrib, 1) + + self.convs_bridge.append(conv_bridge) + self.convs_pred.append(conv_pred) + + def init_weights(self): + """Initialize weights of the head.""" + for m in self.convs_pred: + normal_init(m, std=0.01) + + def forward(self, feats): + """Forward features from the upstream network. + + Args: + feats (tuple[Tensor]): Features from the upstream network, each is + a 4D-tensor. + + Returns: + tuple[Tensor]: A tuple of multi-level predication map, each is a + 4D-tensor of shape (batch_size, 5+num_classes, height, width). + """ + + assert len(feats) == self.num_levels + pred_maps = [] + for i in range(self.num_levels): + x = feats[i] + x = self.convs_bridge[i](x) + pred_map = self.convs_pred[i](x) + pred_maps.append(pred_map) + + return tuple(pred_maps), + + @force_fp32(apply_to=('pred_maps', )) + def get_bboxes(self, + pred_maps, + img_metas, + cfg=None, + rescale=False, + with_nms=True): + """Transform network output for a batch into bbox predictions. + + Args: + pred_maps (list[Tensor]): Raw predictions for a batch of images. + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + cfg (mmcv.Config | None): Test / postprocessing configuration, + if None, test_cfg would be used. Default: None. + rescale (bool): If True, return boxes in original image space. + Default: False. + with_nms (bool): If True, do nms before return boxes. + Default: True. + + Returns: + list[tuple[Tensor, Tensor]]: Each item in result_list is 2-tuple. + The first item is an (n, 5) tensor, where 5 represent + (tl_x, tl_y, br_x, br_y, score) and the score between 0 and 1. + The shape of the second tensor in the tuple is (n,), and + each element represents the class label of the corresponding + box. + """ + num_levels = len(pred_maps) + pred_maps_list = [pred_maps[i].detach() for i in range(num_levels)] + scale_factors = [ + img_metas[i]['scale_factor'] + for i in range(pred_maps_list[0].shape[0]) + ] + result_list = self._get_bboxes(pred_maps_list, scale_factors, cfg, + rescale, with_nms) + return result_list + + def _get_bboxes(self, + pred_maps_list, + scale_factors, + cfg, + rescale=False, + with_nms=True): + """Transform outputs for a single batch item into bbox predictions. + + Args: + pred_maps_list (list[Tensor]): Prediction maps for different scales + of each single image in the batch. + scale_factors (list(ndarray)): Scale factor of the image arrange as + (w_scale, h_scale, w_scale, h_scale). + cfg (mmcv.Config | None): Test / postprocessing configuration, + if None, test_cfg would be used. + rescale (bool): If True, return boxes in original image space. + Default: False. + with_nms (bool): If True, do nms before return boxes. + Default: True. + + Returns: + list[tuple[Tensor, Tensor]]: Each item in result_list is 2-tuple. + The first item is an (n, 5) tensor, where 5 represent + (tl_x, tl_y, br_x, br_y, score) and the score between 0 and 1. + The shape of the second tensor in the tuple is (n,), and + each element represents the class label of the corresponding + box. + """ + cfg = self.test_cfg if cfg is None else cfg + assert len(pred_maps_list) == self.num_levels + + device = pred_maps_list[0].device + batch_size = pred_maps_list[0].shape[0] + + featmap_sizes = [ + pred_maps_list[i].shape[-2:] for i in range(self.num_levels) + ] + multi_lvl_anchors = self.anchor_generator.grid_anchors( + featmap_sizes, device) + # convert to tensor to keep tracing + nms_pre_tensor = torch.tensor( + cfg.get('nms_pre', -1), device=device, dtype=torch.long) + + multi_lvl_bboxes = [] + multi_lvl_cls_scores = [] + multi_lvl_conf_scores = [] + for i in range(self.num_levels): + # get some key info for current scale + pred_map = pred_maps_list[i] + stride = self.featmap_strides[i] + # (b,h, w, num_anchors*num_attrib) -> + # (b,h*w*num_anchors, num_attrib) + pred_map = pred_map.permute(0, 2, 3, + 1).reshape(batch_size, -1, + self.num_attrib) + # Inplace operation like + # ```pred_map[..., :2] = \torch.sigmoid(pred_map[..., :2])``` + # would create constant tensor when exporting to onnx + pred_map_conf = torch.sigmoid(pred_map[..., :2]) + pred_map_rest = pred_map[..., 2:] + pred_map = torch.cat([pred_map_conf, pred_map_rest], dim=-1) + pred_map_boxes = pred_map[..., :4] + multi_lvl_anchor = multi_lvl_anchors[i] + multi_lvl_anchor = multi_lvl_anchor.expand_as(pred_map_boxes) + bbox_pred = self.bbox_coder.decode(multi_lvl_anchor, + pred_map_boxes, stride) + # conf and cls + conf_pred = torch.sigmoid(pred_map[..., 4]) + cls_pred = torch.sigmoid(pred_map[..., 5:]).view( + batch_size, -1, self.num_classes) # Cls pred one-hot. + + # Get top-k prediction + # Always keep topk op for dynamic input in onnx + if nms_pre_tensor > 0 and (torch.onnx.is_in_onnx_export() + or conf_pred.shape[1] > nms_pre_tensor): + from torch import _shape_as_tensor + # keep shape as tensor and get k + num_anchor = _shape_as_tensor(conf_pred)[1].to(device) + nms_pre = torch.where(nms_pre_tensor < num_anchor, + nms_pre_tensor, num_anchor) + _, topk_inds = conf_pred.topk(nms_pre) + batch_inds = torch.arange(batch_size).view( + -1, 1).expand_as(topk_inds).long() + bbox_pred = bbox_pred[batch_inds, topk_inds, :] + cls_pred = cls_pred[batch_inds, topk_inds, :] + conf_pred = conf_pred[batch_inds, topk_inds] + + # Save the result of current scale + multi_lvl_bboxes.append(bbox_pred) + multi_lvl_cls_scores.append(cls_pred) + multi_lvl_conf_scores.append(conf_pred) + + # Merge the results of different scales together + batch_mlvl_bboxes = torch.cat(multi_lvl_bboxes, dim=1) + batch_mlvl_scores = torch.cat(multi_lvl_cls_scores, dim=1) + batch_mlvl_conf_scores = torch.cat(multi_lvl_conf_scores, dim=1) + + # Set max number of box to be feed into nms in deployment + deploy_nms_pre = cfg.get('deploy_nms_pre', -1) + if deploy_nms_pre > 0 and torch.onnx.is_in_onnx_export(): + _, topk_inds = batch_mlvl_conf_scores.topk(deploy_nms_pre) + batch_inds = torch.arange(batch_size).view( + -1, 1).expand_as(topk_inds).long() + batch_mlvl_bboxes = batch_mlvl_bboxes[batch_inds, topk_inds, :] + batch_mlvl_scores = batch_mlvl_scores[batch_inds, topk_inds, :] + batch_mlvl_conf_scores = batch_mlvl_conf_scores[batch_inds, + topk_inds] + + if with_nms and (batch_mlvl_conf_scores.size(0) == 0): + return torch.zeros((0, 5)), torch.zeros((0, )) + + if rescale: + batch_mlvl_bboxes /= batch_mlvl_bboxes.new_tensor( + scale_factors).unsqueeze(1) + + # In mmdet 2.x, the class_id for background is num_classes. + # i.e., the last column. + padding = batch_mlvl_scores.new_zeros(batch_size, + batch_mlvl_scores.shape[1], 1) + batch_mlvl_scores = torch.cat([batch_mlvl_scores, padding], dim=-1) + + # Support exporting to onnx without nms + if with_nms and cfg.get('nms', None) is not None: + det_results = [] + for (mlvl_bboxes, mlvl_scores, + mlvl_conf_scores) in zip(batch_mlvl_bboxes, batch_mlvl_scores, + batch_mlvl_conf_scores): + # Filtering out all predictions with conf < conf_thr + conf_thr = cfg.get('conf_thr', -1) + if conf_thr > 0 and (not torch.onnx.is_in_onnx_export()): + # TensorRT not support NonZero + # add as_tuple=False for compatibility in Pytorch 1.6 + # flatten would create a Reshape op with constant values, + # and raise RuntimeError when doing inference in ONNX + # Runtime with a different input image (#4221). + conf_inds = mlvl_conf_scores.ge(conf_thr).nonzero( + as_tuple=False).squeeze(1) + mlvl_bboxes = mlvl_bboxes[conf_inds, :] + mlvl_scores = mlvl_scores[conf_inds, :] + mlvl_conf_scores = mlvl_conf_scores[conf_inds] + + det_bboxes, det_labels = multiclass_nms( + mlvl_bboxes, + mlvl_scores, + cfg.score_thr, + cfg.nms, + cfg.max_per_img, + score_factors=mlvl_conf_scores) + det_results.append(tuple([det_bboxes, det_labels])) + + else: + det_results = [ + tuple(mlvl_bs) + for mlvl_bs in zip(batch_mlvl_bboxes, batch_mlvl_scores, + batch_mlvl_conf_scores) + ] + return det_results + + @force_fp32(apply_to=('pred_maps', )) + def loss(self, + pred_maps, + gt_bboxes, + gt_labels, + img_metas, + gt_bboxes_ignore=None): + """Compute loss of the head. + + Args: + pred_maps (list[Tensor]): Prediction map for each scale level, + shape (N, num_anchors * num_attrib, H, W) + gt_bboxes (list[Tensor]): Ground truth bboxes for each image with + shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + gt_labels (list[Tensor]): class indices corresponding to each box + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + gt_bboxes_ignore (None | list[Tensor]): specify which bounding + boxes can be ignored when computing the loss. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + num_imgs = len(img_metas) + device = pred_maps[0][0].device + + featmap_sizes = [ + pred_maps[i].shape[-2:] for i in range(self.num_levels) + ] + multi_level_anchors = self.anchor_generator.grid_anchors( + featmap_sizes, device) + anchor_list = [multi_level_anchors for _ in range(num_imgs)] + + responsible_flag_list = [] + for img_id in range(len(img_metas)): + responsible_flag_list.append( + self.anchor_generator.responsible_flags( + featmap_sizes, gt_bboxes[img_id], device)) + + target_maps_list, neg_maps_list = self.get_targets( + anchor_list, responsible_flag_list, gt_bboxes, gt_labels) + + losses_cls, losses_conf, losses_xy, losses_wh = multi_apply( + self.loss_single, pred_maps, target_maps_list, neg_maps_list) + + return dict( + loss_cls=losses_cls, + loss_conf=losses_conf, + loss_xy=losses_xy, + loss_wh=losses_wh) + + def loss_single(self, pred_map, target_map, neg_map): + """Compute loss of a single image from a batch. + + Args: + pred_map (Tensor): Raw predictions for a single level. + target_map (Tensor): The Ground-Truth target for a single level. + neg_map (Tensor): The negative masks for a single level. + + Returns: + tuple: + loss_cls (Tensor): Classification loss. + loss_conf (Tensor): Confidence loss. + loss_xy (Tensor): Regression loss of x, y coordinate. + loss_wh (Tensor): Regression loss of w, h coordinate. + """ + + num_imgs = len(pred_map) + pred_map = pred_map.permute(0, 2, 3, + 1).reshape(num_imgs, -1, self.num_attrib) + neg_mask = neg_map.float() + pos_mask = target_map[..., 4] + pos_and_neg_mask = neg_mask + pos_mask + pos_mask = pos_mask.unsqueeze(dim=-1) + if torch.max(pos_and_neg_mask) > 1.: + warnings.warn('There is overlap between pos and neg sample.') + pos_and_neg_mask = pos_and_neg_mask.clamp(min=0., max=1.) + + pred_xy = pred_map[..., :2] + pred_wh = pred_map[..., 2:4] + pred_conf = pred_map[..., 4] + pred_label = pred_map[..., 5:] + + target_xy = target_map[..., :2] + target_wh = target_map[..., 2:4] + target_conf = target_map[..., 4] + target_label = target_map[..., 5:] + + loss_cls = self.loss_cls(pred_label, target_label, weight=pos_mask) + loss_conf = self.loss_conf( + pred_conf, target_conf, weight=pos_and_neg_mask) + loss_xy = self.loss_xy(pred_xy, target_xy, weight=pos_mask) + loss_wh = self.loss_wh(pred_wh, target_wh, weight=pos_mask) + + return loss_cls, loss_conf, loss_xy, loss_wh + + def get_targets(self, anchor_list, responsible_flag_list, gt_bboxes_list, + gt_labels_list): + """Compute target maps for anchors in multiple images. + + Args: + anchor_list (list[list[Tensor]]): Multi level anchors of each + image. The outer list indicates images, and the inner list + corresponds to feature levels of the image. Each element of + the inner list is a tensor of shape (num_total_anchors, 4). + responsible_flag_list (list[list[Tensor]]): Multi level responsible + flags of each image. Each element is a tensor of shape + (num_total_anchors, ) + gt_bboxes_list (list[Tensor]): Ground truth bboxes of each image. + gt_labels_list (list[Tensor]): Ground truth labels of each box. + + Returns: + tuple: Usually returns a tuple containing learning targets. + - target_map_list (list[Tensor]): Target map of each level. + - neg_map_list (list[Tensor]): Negative map of each level. + """ + num_imgs = len(anchor_list) + + # anchor number of multi levels + num_level_anchors = [anchors.size(0) for anchors in anchor_list[0]] + + results = multi_apply(self._get_targets_single, anchor_list, + responsible_flag_list, gt_bboxes_list, + gt_labels_list) + + all_target_maps, all_neg_maps = results + assert num_imgs == len(all_target_maps) == len(all_neg_maps) + target_maps_list = images_to_levels(all_target_maps, num_level_anchors) + neg_maps_list = images_to_levels(all_neg_maps, num_level_anchors) + + return target_maps_list, neg_maps_list + + def _get_targets_single(self, anchors, responsible_flags, gt_bboxes, + gt_labels): + """Generate matching bounding box prior and converted GT. + + Args: + anchors (list[Tensor]): Multi-level anchors of the image. + responsible_flags (list[Tensor]): Multi-level responsible flags of + anchors + gt_bboxes (Tensor): Ground truth bboxes of single image. + gt_labels (Tensor): Ground truth labels of single image. + + Returns: + tuple: + target_map (Tensor): Predication target map of each + scale level, shape (num_total_anchors, + 5+num_classes) + neg_map (Tensor): Negative map of each scale level, + shape (num_total_anchors,) + """ + + anchor_strides = [] + for i in range(len(anchors)): + anchor_strides.append( + torch.tensor(self.featmap_strides[i], + device=gt_bboxes.device).repeat(len(anchors[i]))) + concat_anchors = torch.cat(anchors) + concat_responsible_flags = torch.cat(responsible_flags) + + anchor_strides = torch.cat(anchor_strides) + assert len(anchor_strides) == len(concat_anchors) == \ + len(concat_responsible_flags) + assign_result = self.assigner.assign(concat_anchors, + concat_responsible_flags, + gt_bboxes) + sampling_result = self.sampler.sample(assign_result, concat_anchors, + gt_bboxes) + + target_map = concat_anchors.new_zeros( + concat_anchors.size(0), self.num_attrib) + + target_map[sampling_result.pos_inds, :4] = self.bbox_coder.encode( + sampling_result.pos_bboxes, sampling_result.pos_gt_bboxes, + anchor_strides[sampling_result.pos_inds]) + + target_map[sampling_result.pos_inds, 4] = 1 + + gt_labels_one_hot = F.one_hot( + gt_labels, num_classes=self.num_classes).float() + if self.one_hot_smoother != 0: # label smooth + gt_labels_one_hot = gt_labels_one_hot * ( + 1 - self.one_hot_smoother + ) + self.one_hot_smoother / self.num_classes + target_map[sampling_result.pos_inds, 5:] = gt_labels_one_hot[ + sampling_result.pos_assigned_gt_inds] + + neg_map = concat_anchors.new_zeros( + concat_anchors.size(0), dtype=torch.uint8) + neg_map[sampling_result.neg_inds] = 1 + + return target_map, neg_map + + def aug_test(self, feats, img_metas, rescale=False): + """Test function with test time augmentation. + + Args: + feats (list[Tensor]): the outer list indicates test-time + augmentations and inner Tensor should have a shape NxCxHxW, + which contains features for all images in the batch. + img_metas (list[list[dict]]): the outer list indicates test-time + augs (multiscale, flip, etc.) and the inner list indicates + images in a batch. each dict has image information. + rescale (bool, optional): Whether to rescale the results. + Defaults to False. + + Returns: + list[ndarray]: bbox results of each class + """ + return self.aug_test_bboxes(feats, img_metas, rescale=rescale) diff --git a/detection/mmdet/models/detectors/__init__.py b/detection/mmdet/models/detectors/__init__.py new file mode 100644 index 0000000..0401113 --- /dev/null +++ b/detection/mmdet/models/detectors/__init__.py @@ -0,0 +1,40 @@ +from .atss import ATSS +from .base import BaseDetector +from .cascade_rcnn import CascadeRCNN +from .cornernet import CornerNet +from .detr import DETR +from .fast_rcnn import FastRCNN +from .faster_rcnn import FasterRCNN +from .fcos import FCOS +from .fovea import FOVEA +from .fsaf import FSAF +from .gfl import GFL +from .grid_rcnn import GridRCNN +from .htc import HybridTaskCascade +from .kd_one_stage import KnowledgeDistillationSingleStageDetector +from .mask_rcnn import MaskRCNN +from .mask_scoring_rcnn import MaskScoringRCNN +from .nasfcos import NASFCOS +from .paa import PAA +from .point_rend import PointRend +from .reppoints_detector import RepPointsDetector +from .retinanet import RetinaNet +from .rpn import RPN +from .scnet import SCNet +from .single_stage import SingleStageDetector +from .sparse_rcnn import SparseRCNN +from .trident_faster_rcnn import TridentFasterRCNN +from .two_stage import TwoStageDetector +from .vfnet import VFNet +from .yolact import YOLACT +from .yolo import YOLOV3 + +__all__ = [ + 'ATSS', 'BaseDetector', 'SingleStageDetector', + 'KnowledgeDistillationSingleStageDetector', 'TwoStageDetector', 'RPN', + 'FastRCNN', 'FasterRCNN', 'MaskRCNN', 'CascadeRCNN', 'HybridTaskCascade', + 'RetinaNet', 'FCOS', 'GridRCNN', 'MaskScoringRCNN', 'RepPointsDetector', + 'FOVEA', 'FSAF', 'NASFCOS', 'PointRend', 'GFL', 'CornerNet', 'PAA', + 'YOLOV3', 'YOLACT', 'VFNet', 'DETR', 'TridentFasterRCNN', 'SparseRCNN', + 'SCNet' +] diff --git a/detection/mmdet/models/detectors/atss.py b/detection/mmdet/models/detectors/atss.py new file mode 100644 index 0000000..db7139c --- /dev/null +++ b/detection/mmdet/models/detectors/atss.py @@ -0,0 +1,17 @@ +from ..builder import DETECTORS +from .single_stage import SingleStageDetector + + +@DETECTORS.register_module() +class ATSS(SingleStageDetector): + """Implementation of `ATSS `_.""" + + def __init__(self, + backbone, + neck, + bbox_head, + train_cfg=None, + test_cfg=None, + pretrained=None): + super(ATSS, self).__init__(backbone, neck, bbox_head, train_cfg, + test_cfg, pretrained) diff --git a/detection/mmdet/models/detectors/base.py b/detection/mmdet/models/detectors/base.py new file mode 100644 index 0000000..89134f3 --- /dev/null +++ b/detection/mmdet/models/detectors/base.py @@ -0,0 +1,355 @@ +from abc import ABCMeta, abstractmethod +from collections import OrderedDict + +import mmcv +import numpy as np +import torch +import torch.distributed as dist +import torch.nn as nn +from mmcv.runner import auto_fp16 +from mmcv.utils import print_log + +from mmdet.core.visualization import imshow_det_bboxes +from mmdet.utils import get_root_logger + + +class BaseDetector(nn.Module, metaclass=ABCMeta): + """Base class for detectors.""" + + def __init__(self): + super(BaseDetector, self).__init__() + self.fp16_enabled = False + + @property + def with_neck(self): + """bool: whether the detector has a neck""" + return hasattr(self, 'neck') and self.neck is not None + + # TODO: these properties need to be carefully handled + # for both single stage & two stage detectors + @property + def with_shared_head(self): + """bool: whether the detector has a shared head in the RoI Head""" + return hasattr(self, 'roi_head') and self.roi_head.with_shared_head + + @property + def with_bbox(self): + """bool: whether the detector has a bbox head""" + return ((hasattr(self, 'roi_head') and self.roi_head.with_bbox) + or (hasattr(self, 'bbox_head') and self.bbox_head is not None)) + + @property + def with_mask(self): + """bool: whether the detector has a mask head""" + return ((hasattr(self, 'roi_head') and self.roi_head.with_mask) + or (hasattr(self, 'mask_head') and self.mask_head is not None)) + + @abstractmethod + def extract_feat(self, imgs): + """Extract features from images.""" + pass + + def extract_feats(self, imgs): + """Extract features from multiple images. + + Args: + imgs (list[torch.Tensor]): A list of images. The images are + augmented from the same image but in different ways. + + Returns: + list[torch.Tensor]: Features of different images + """ + assert isinstance(imgs, list) + return [self.extract_feat(img) for img in imgs] + + def forward_train(self, imgs, img_metas, **kwargs): + """ + Args: + img (list[Tensor]): List of tensors of shape (1, C, H, W). + Typically these should be mean centered and std scaled. + img_metas (list[dict]): List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys, see + :class:`mmdet.datasets.pipelines.Collect`. + kwargs (keyword arguments): Specific to concrete implementation. + """ + # NOTE the batched image size information may be useful, e.g. + # in DETR, this is needed for the construction of masks, which is + # then used for the transformer_head. + batch_input_shape = tuple(imgs[0].size()[-2:]) + for img_meta in img_metas: + img_meta['batch_input_shape'] = batch_input_shape + + async def async_simple_test(self, img, img_metas, **kwargs): + raise NotImplementedError + + @abstractmethod + def simple_test(self, img, img_metas, **kwargs): + pass + + @abstractmethod + def aug_test(self, imgs, img_metas, **kwargs): + """Test function with test time augmentation.""" + pass + + def init_weights(self, pretrained=None): + """Initialize the weights in detector. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + if pretrained is not None: + logger = get_root_logger() + print_log(f'load model from: {pretrained}', logger=logger) + + async def aforward_test(self, *, img, img_metas, **kwargs): + for var, name in [(img, 'img'), (img_metas, 'img_metas')]: + if not isinstance(var, list): + raise TypeError(f'{name} must be a list, but got {type(var)}') + + num_augs = len(img) + if num_augs != len(img_metas): + raise ValueError(f'num of augmentations ({len(img)}) ' + f'!= num of image metas ({len(img_metas)})') + # TODO: remove the restriction of samples_per_gpu == 1 when prepared + samples_per_gpu = img[0].size(0) + assert samples_per_gpu == 1 + + if num_augs == 1: + return await self.async_simple_test(img[0], img_metas[0], **kwargs) + else: + raise NotImplementedError + + def forward_test(self, imgs, img_metas, **kwargs): + """ + Args: + imgs (List[Tensor]): the outer list indicates test-time + augmentations and inner Tensor should have a shape NxCxHxW, + which contains all images in the batch. + img_metas (List[List[dict]]): the outer list indicates test-time + augs (multiscale, flip, etc.) and the inner list indicates + images in a batch. + """ + for var, name in [(imgs, 'imgs'), (img_metas, 'img_metas')]: + if not isinstance(var, list): + raise TypeError(f'{name} must be a list, but got {type(var)}') + + num_augs = len(imgs) + if num_augs != len(img_metas): + raise ValueError(f'num of augmentations ({len(imgs)}) ' + f'!= num of image meta ({len(img_metas)})') + + # NOTE the batched image size information may be useful, e.g. + # in DETR, this is needed for the construction of masks, which is + # then used for the transformer_head. + for img, img_meta in zip(imgs, img_metas): + batch_size = len(img_meta) + for img_id in range(batch_size): + img_meta[img_id]['batch_input_shape'] = tuple(img.size()[-2:]) + + if num_augs == 1: + # proposals (List[List[Tensor]]): the outer list indicates + # test-time augs (multiscale, flip, etc.) and the inner list + # indicates images in a batch. + # The Tensor should have a shape Px4, where P is the number of + # proposals. + if 'proposals' in kwargs: + kwargs['proposals'] = kwargs['proposals'][0] + return self.simple_test(imgs[0], img_metas[0], **kwargs) + else: + assert imgs[0].size(0) == 1, 'aug test does not support ' \ + 'inference with batch size ' \ + f'{imgs[0].size(0)}' + # TODO: support test augmentation for predefined proposals + assert 'proposals' not in kwargs + return self.aug_test(imgs, img_metas, **kwargs) + + @auto_fp16(apply_to=('img', )) + def forward(self, img, img_metas, return_loss=True, **kwargs): + """Calls either :func:`forward_train` or :func:`forward_test` depending + on whether ``return_loss`` is ``True``. + + Note this setting will change the expected inputs. When + ``return_loss=True``, img and img_meta are single-nested (i.e. Tensor + and List[dict]), and when ``resturn_loss=False``, img and img_meta + should be double nested (i.e. List[Tensor], List[List[dict]]), with + the outer list indicating test time augmentations. + """ + if return_loss: + return self.forward_train(img, img_metas, **kwargs) + else: + return self.forward_test(img, img_metas, **kwargs) + + def _parse_losses(self, losses): + """Parse the raw outputs (losses) of the network. + + Args: + losses (dict): Raw output of the network, which usually contain + losses and other necessary infomation. + + Returns: + tuple[Tensor, dict]: (loss, log_vars), loss is the loss tensor \ + which may be a weighted sum of all losses, log_vars contains \ + all the variables to be sent to the logger. + """ + log_vars = OrderedDict() + for loss_name, loss_value in losses.items(): + if isinstance(loss_value, torch.Tensor): + log_vars[loss_name] = loss_value.mean() + elif isinstance(loss_value, list): + log_vars[loss_name] = sum(_loss.mean() for _loss in loss_value) + else: + raise TypeError( + f'{loss_name} is not a tensor or list of tensors') + + loss = sum(_value for _key, _value in log_vars.items() + if 'loss' in _key) + + log_vars['loss'] = loss + for loss_name, loss_value in log_vars.items(): + # reduce loss when distributed training + if dist.is_available() and dist.is_initialized(): + loss_value = loss_value.data.clone() + dist.all_reduce(loss_value.div_(dist.get_world_size())) + log_vars[loss_name] = loss_value.item() + + return loss, log_vars + + def train_step(self, data, optimizer): + """The iteration step during training. + + This method defines an iteration step during training, except for the + back propagation and optimizer updating, which are done in an optimizer + hook. Note that in some complicated cases or models, the whole process + including back propagation and optimizer updating is also defined in + this method, such as GAN. + + Args: + data (dict): The output of dataloader. + optimizer (:obj:`torch.optim.Optimizer` | dict): The optimizer of + runner is passed to ``train_step()``. This argument is unused + and reserved. + + Returns: + dict: It should contain at least 3 keys: ``loss``, ``log_vars``, \ + ``num_samples``. + + - ``loss`` is a tensor for back propagation, which can be a \ + weighted sum of multiple losses. + - ``log_vars`` contains all the variables to be sent to the + logger. + - ``num_samples`` indicates the batch size (when the model is \ + DDP, it means the batch size on each GPU), which is used for \ + averaging the logs. + """ + losses = self(**data) + loss, log_vars = self._parse_losses(losses) + + outputs = dict( + loss=loss, log_vars=log_vars, num_samples=len(data['img_metas'])) + + return outputs + + def val_step(self, data, optimizer): + """The iteration step during validation. + + This method shares the same signature as :func:`train_step`, but used + during val epochs. Note that the evaluation after training epochs is + not implemented with this method, but an evaluation hook. + """ + losses = self(**data) + loss, log_vars = self._parse_losses(losses) + + outputs = dict( + loss=loss, log_vars=log_vars, num_samples=len(data['img_metas'])) + + return outputs + + def show_result(self, + img, + result, + score_thr=0.3, + bbox_color=(72, 101, 241), + text_color=(72, 101, 241), + mask_color=None, + thickness=2, + font_size=13, + win_name='', + show=False, + wait_time=0, + out_file=None): + """Draw `result` over `img`. + + Args: + img (str or Tensor): The image to be displayed. + result (Tensor or tuple): The results to draw over `img` + bbox_result or (bbox_result, segm_result). + score_thr (float, optional): Minimum score of bboxes to be shown. + Default: 0.3. + bbox_color (str or tuple(int) or :obj:`Color`):Color of bbox lines. + The tuple of color should be in BGR order. Default: 'green' + text_color (str or tuple(int) or :obj:`Color`):Color of texts. + The tuple of color should be in BGR order. Default: 'green' + mask_color (None or str or tuple(int) or :obj:`Color`): + Color of masks. The tuple of color should be in BGR order. + Default: None + thickness (int): Thickness of lines. Default: 2 + font_size (int): Font size of texts. Default: 13 + win_name (str): The window name. Default: '' + wait_time (float): Value of waitKey param. + Default: 0. + show (bool): Whether to show the image. + Default: False. + out_file (str or None): The filename to write the image. + Default: None. + + Returns: + img (Tensor): Only if not `show` or `out_file` + """ + img = mmcv.imread(img) + img = img.copy() + if isinstance(result, tuple): + bbox_result, segm_result = result + if isinstance(segm_result, tuple): + segm_result = segm_result[0] # ms rcnn + else: + bbox_result, segm_result = result, None + bboxes = np.vstack(bbox_result) + labels = [ + np.full(bbox.shape[0], i, dtype=np.int32) + for i, bbox in enumerate(bbox_result) + ] + labels = np.concatenate(labels) + # draw segmentation masks + segms = None + if segm_result is not None and len(labels) > 0: # non empty + segms = mmcv.concat_list(segm_result) + if isinstance(segms[0], torch.Tensor): + segms = torch.stack(segms, dim=0).detach().cpu().numpy() + else: + segms = np.stack(segms, axis=0) + # if out_file specified, do not show image in window + if out_file is not None: + show = False + # draw bounding boxes + img = imshow_det_bboxes( + img, + bboxes, + labels, + segms, + class_names=self.CLASSES, + score_thr=score_thr, + bbox_color=bbox_color, + text_color=text_color, + mask_color=mask_color, + thickness=thickness, + font_size=font_size, + win_name=win_name, + show=show, + wait_time=wait_time, + out_file=out_file) + + if not (show or out_file): + return img diff --git a/detection/mmdet/models/detectors/cascade_rcnn.py b/detection/mmdet/models/detectors/cascade_rcnn.py new file mode 100644 index 0000000..d873dce --- /dev/null +++ b/detection/mmdet/models/detectors/cascade_rcnn.py @@ -0,0 +1,46 @@ +from ..builder import DETECTORS +from .two_stage import TwoStageDetector + + +@DETECTORS.register_module() +class CascadeRCNN(TwoStageDetector): + r"""Implementation of `Cascade R-CNN: Delving into High Quality Object + Detection `_""" + + def __init__(self, + backbone, + neck=None, + rpn_head=None, + roi_head=None, + train_cfg=None, + test_cfg=None, + pretrained=None): + super(CascadeRCNN, self).__init__( + backbone=backbone, + neck=neck, + rpn_head=rpn_head, + roi_head=roi_head, + train_cfg=train_cfg, + test_cfg=test_cfg, + pretrained=pretrained) + + def show_result(self, data, result, **kwargs): + """Show prediction results of the detector. + + Args: + data (str or np.ndarray): Image filename or loaded image. + result (Tensor or tuple): The results to draw over `img` + bbox_result or (bbox_result, segm_result). + + Returns: + np.ndarray: The image with bboxes drawn on it. + """ + if self.with_mask: + ms_bbox_result, ms_segm_result = result + if isinstance(ms_bbox_result, dict): + result = (ms_bbox_result['ensemble'], + ms_segm_result['ensemble']) + else: + if isinstance(result, dict): + result = result['ensemble'] + return super(CascadeRCNN, self).show_result(data, result, **kwargs) diff --git a/detection/mmdet/models/detectors/cornernet.py b/detection/mmdet/models/detectors/cornernet.py new file mode 100644 index 0000000..bb8ccc1 --- /dev/null +++ b/detection/mmdet/models/detectors/cornernet.py @@ -0,0 +1,95 @@ +import torch + +from mmdet.core import bbox2result, bbox_mapping_back +from ..builder import DETECTORS +from .single_stage import SingleStageDetector + + +@DETECTORS.register_module() +class CornerNet(SingleStageDetector): + """CornerNet. + + This detector is the implementation of the paper `CornerNet: Detecting + Objects as Paired Keypoints `_ . + """ + + def __init__(self, + backbone, + neck, + bbox_head, + train_cfg=None, + test_cfg=None, + pretrained=None): + super(CornerNet, self).__init__(backbone, neck, bbox_head, train_cfg, + test_cfg, pretrained) + + def merge_aug_results(self, aug_results, img_metas): + """Merge augmented detection bboxes and score. + + Args: + aug_results (list[list[Tensor]]): Det_bboxes and det_labels of each + image. + img_metas (list[list[dict]]): Meta information of each image, e.g., + image size, scaling factor, etc. + + Returns: + tuple: (bboxes, labels) + """ + recovered_bboxes, aug_labels = [], [] + for bboxes_labels, img_info in zip(aug_results, img_metas): + img_shape = img_info[0]['img_shape'] # using shape before padding + scale_factor = img_info[0]['scale_factor'] + flip = img_info[0]['flip'] + bboxes, labels = bboxes_labels + bboxes, scores = bboxes[:, :4], bboxes[:, -1:] + bboxes = bbox_mapping_back(bboxes, img_shape, scale_factor, flip) + recovered_bboxes.append(torch.cat([bboxes, scores], dim=-1)) + aug_labels.append(labels) + + bboxes = torch.cat(recovered_bboxes, dim=0) + labels = torch.cat(aug_labels) + + if bboxes.shape[0] > 0: + out_bboxes, out_labels = self.bbox_head._bboxes_nms( + bboxes, labels, self.bbox_head.test_cfg) + else: + out_bboxes, out_labels = bboxes, labels + + return out_bboxes, out_labels + + def aug_test(self, imgs, img_metas, rescale=False): + """Augment testing of CornerNet. + + Args: + imgs (list[Tensor]): Augmented images. + img_metas (list[list[dict]]): Meta information of each image, e.g., + image size, scaling factor, etc. + rescale (bool): If True, return boxes in original image space. + Default: False. + + Note: + ``imgs`` must including flipped image pairs. + + Returns: + list[list[np.ndarray]]: BBox results of each image and classes. + The outer list corresponds to each image. The inner list + corresponds to each class. + """ + img_inds = list(range(len(imgs))) + + assert img_metas[0][0]['flip'] + img_metas[1][0]['flip'], ( + 'aug test must have flipped image pair') + aug_results = [] + for ind, flip_ind in zip(img_inds[0::2], img_inds[1::2]): + img_pair = torch.cat([imgs[ind], imgs[flip_ind]]) + x = self.extract_feat(img_pair) + outs = self.bbox_head(x) + bbox_list = self.bbox_head.get_bboxes( + *outs, [img_metas[ind], img_metas[flip_ind]], False, False) + aug_results.append(bbox_list[0]) + aug_results.append(bbox_list[1]) + + bboxes, labels = self.merge_aug_results(aug_results, img_metas) + bbox_results = bbox2result(bboxes, labels, self.bbox_head.num_classes) + + return [bbox_results] diff --git a/detection/mmdet/models/detectors/detr.py b/detection/mmdet/models/detectors/detr.py new file mode 100644 index 0000000..5ff82a2 --- /dev/null +++ b/detection/mmdet/models/detectors/detr.py @@ -0,0 +1,46 @@ +from mmdet.core import bbox2result +from ..builder import DETECTORS +from .single_stage import SingleStageDetector + + +@DETECTORS.register_module() +class DETR(SingleStageDetector): + r"""Implementation of `DETR: End-to-End Object Detection with + Transformers `_""" + + def __init__(self, + backbone, + bbox_head, + train_cfg=None, + test_cfg=None, + pretrained=None): + super(DETR, self).__init__(backbone, None, bbox_head, train_cfg, + test_cfg, pretrained) + + def simple_test(self, img, img_metas, rescale=False): + """Test function without test time augmentation. + + Args: + imgs (list[torch.Tensor]): List of multiple images + img_metas (list[dict]): List of image information. + rescale (bool, optional): Whether to rescale the results. + Defaults to False. + + Returns: + list[list[np.ndarray]]: BBox results of each image and classes. + The outer list corresponds to each image. The inner list + corresponds to each class. + """ + batch_size = len(img_metas) + assert batch_size == 1, 'Currently only batch_size 1 for inference ' \ + f'mode is supported. Found batch_size {batch_size}.' + x = self.extract_feat(img) + outs = self.bbox_head(x, img_metas) + bbox_list = self.bbox_head.get_bboxes( + *outs, img_metas, rescale=rescale) + + bbox_results = [ + bbox2result(det_bboxes, det_labels, self.bbox_head.num_classes) + for det_bboxes, det_labels in bbox_list + ] + return bbox_results diff --git a/detection/mmdet/models/detectors/fast_rcnn.py b/detection/mmdet/models/detectors/fast_rcnn.py new file mode 100644 index 0000000..3d6e242 --- /dev/null +++ b/detection/mmdet/models/detectors/fast_rcnn.py @@ -0,0 +1,52 @@ +from ..builder import DETECTORS +from .two_stage import TwoStageDetector + + +@DETECTORS.register_module() +class FastRCNN(TwoStageDetector): + """Implementation of `Fast R-CNN `_""" + + def __init__(self, + backbone, + roi_head, + train_cfg, + test_cfg, + neck=None, + pretrained=None): + super(FastRCNN, self).__init__( + backbone=backbone, + neck=neck, + roi_head=roi_head, + train_cfg=train_cfg, + test_cfg=test_cfg, + pretrained=pretrained) + + def forward_test(self, imgs, img_metas, proposals, **kwargs): + """ + Args: + imgs (List[Tensor]): the outer list indicates test-time + augmentations and inner Tensor should have a shape NxCxHxW, + which contains all images in the batch. + img_metas (List[List[dict]]): the outer list indicates test-time + augs (multiscale, flip, etc.) and the inner list indicates + images in a batch. + proposals (List[List[Tensor]]): the outer list indicates test-time + augs (multiscale, flip, etc.) and the inner list indicates + images in a batch. The Tensor should have a shape Px4, where + P is the number of proposals. + """ + for var, name in [(imgs, 'imgs'), (img_metas, 'img_metas')]: + if not isinstance(var, list): + raise TypeError(f'{name} must be a list, but got {type(var)}') + + num_augs = len(imgs) + if num_augs != len(img_metas): + raise ValueError(f'num of augmentations ({len(imgs)}) ' + f'!= num of image meta ({len(img_metas)})') + + if num_augs == 1: + return self.simple_test(imgs[0], img_metas[0], proposals[0], + **kwargs) + else: + # TODO: support test-time augmentation + assert NotImplementedError diff --git a/detection/mmdet/models/detectors/faster_rcnn.py b/detection/mmdet/models/detectors/faster_rcnn.py new file mode 100644 index 0000000..81bad0f --- /dev/null +++ b/detection/mmdet/models/detectors/faster_rcnn.py @@ -0,0 +1,24 @@ +from ..builder import DETECTORS +from .two_stage import TwoStageDetector + + +@DETECTORS.register_module() +class FasterRCNN(TwoStageDetector): + """Implementation of `Faster R-CNN `_""" + + def __init__(self, + backbone, + rpn_head, + roi_head, + train_cfg, + test_cfg, + neck=None, + pretrained=None): + super(FasterRCNN, self).__init__( + backbone=backbone, + neck=neck, + rpn_head=rpn_head, + roi_head=roi_head, + train_cfg=train_cfg, + test_cfg=test_cfg, + pretrained=pretrained) diff --git a/detection/mmdet/models/detectors/fcos.py b/detection/mmdet/models/detectors/fcos.py new file mode 100644 index 0000000..58485c1 --- /dev/null +++ b/detection/mmdet/models/detectors/fcos.py @@ -0,0 +1,17 @@ +from ..builder import DETECTORS +from .single_stage import SingleStageDetector + + +@DETECTORS.register_module() +class FCOS(SingleStageDetector): + """Implementation of `FCOS `_""" + + def __init__(self, + backbone, + neck, + bbox_head, + train_cfg=None, + test_cfg=None, + pretrained=None): + super(FCOS, self).__init__(backbone, neck, bbox_head, train_cfg, + test_cfg, pretrained) diff --git a/detection/mmdet/models/detectors/fovea.py b/detection/mmdet/models/detectors/fovea.py new file mode 100644 index 0000000..22a578e --- /dev/null +++ b/detection/mmdet/models/detectors/fovea.py @@ -0,0 +1,17 @@ +from ..builder import DETECTORS +from .single_stage import SingleStageDetector + + +@DETECTORS.register_module() +class FOVEA(SingleStageDetector): + """Implementation of `FoveaBox `_""" + + def __init__(self, + backbone, + neck, + bbox_head, + train_cfg=None, + test_cfg=None, + pretrained=None): + super(FOVEA, self).__init__(backbone, neck, bbox_head, train_cfg, + test_cfg, pretrained) diff --git a/detection/mmdet/models/detectors/fsaf.py b/detection/mmdet/models/detectors/fsaf.py new file mode 100644 index 0000000..9f10fa1 --- /dev/null +++ b/detection/mmdet/models/detectors/fsaf.py @@ -0,0 +1,17 @@ +from ..builder import DETECTORS +from .single_stage import SingleStageDetector + + +@DETECTORS.register_module() +class FSAF(SingleStageDetector): + """Implementation of `FSAF `_""" + + def __init__(self, + backbone, + neck, + bbox_head, + train_cfg=None, + test_cfg=None, + pretrained=None): + super(FSAF, self).__init__(backbone, neck, bbox_head, train_cfg, + test_cfg, pretrained) diff --git a/detection/mmdet/models/detectors/gfl.py b/detection/mmdet/models/detectors/gfl.py new file mode 100644 index 0000000..64d65cb --- /dev/null +++ b/detection/mmdet/models/detectors/gfl.py @@ -0,0 +1,16 @@ +from ..builder import DETECTORS +from .single_stage import SingleStageDetector + + +@DETECTORS.register_module() +class GFL(SingleStageDetector): + + def __init__(self, + backbone, + neck, + bbox_head, + train_cfg=None, + test_cfg=None, + pretrained=None): + super(GFL, self).__init__(backbone, neck, bbox_head, train_cfg, + test_cfg, pretrained) diff --git a/detection/mmdet/models/detectors/grid_rcnn.py b/detection/mmdet/models/detectors/grid_rcnn.py new file mode 100644 index 0000000..b6145a1 --- /dev/null +++ b/detection/mmdet/models/detectors/grid_rcnn.py @@ -0,0 +1,29 @@ +from ..builder import DETECTORS +from .two_stage import TwoStageDetector + + +@DETECTORS.register_module() +class GridRCNN(TwoStageDetector): + """Grid R-CNN. + + This detector is the implementation of: + - Grid R-CNN (https://arxiv.org/abs/1811.12030) + - Grid R-CNN Plus: Faster and Better (https://arxiv.org/abs/1906.05688) + """ + + def __init__(self, + backbone, + rpn_head, + roi_head, + train_cfg, + test_cfg, + neck=None, + pretrained=None): + super(GridRCNN, self).__init__( + backbone=backbone, + neck=neck, + rpn_head=rpn_head, + roi_head=roi_head, + train_cfg=train_cfg, + test_cfg=test_cfg, + pretrained=pretrained) diff --git a/detection/mmdet/models/detectors/htc.py b/detection/mmdet/models/detectors/htc.py new file mode 100644 index 0000000..d9efdf4 --- /dev/null +++ b/detection/mmdet/models/detectors/htc.py @@ -0,0 +1,15 @@ +from ..builder import DETECTORS +from .cascade_rcnn import CascadeRCNN + + +@DETECTORS.register_module() +class HybridTaskCascade(CascadeRCNN): + """Implementation of `HTC `_""" + + def __init__(self, **kwargs): + super(HybridTaskCascade, self).__init__(**kwargs) + + @property + def with_semantic(self): + """bool: whether the detector has a semantic head""" + return self.roi_head.with_semantic diff --git a/detection/mmdet/models/detectors/kd_one_stage.py b/detection/mmdet/models/detectors/kd_one_stage.py new file mode 100644 index 0000000..671ec19 --- /dev/null +++ b/detection/mmdet/models/detectors/kd_one_stage.py @@ -0,0 +1,100 @@ +import mmcv +import torch +from mmcv.runner import load_checkpoint + +from .. import build_detector +from ..builder import DETECTORS +from .single_stage import SingleStageDetector + + +@DETECTORS.register_module() +class KnowledgeDistillationSingleStageDetector(SingleStageDetector): + r"""Implementation of `Distilling the Knowledge in a Neural Network. + `_. + + Args: + teacher_config (str | dict): Config file path + or the config object of teacher model. + teacher_ckpt (str, optional): Checkpoint path of teacher model. + If left as None, the model will not load any weights. + """ + + def __init__(self, + backbone, + neck, + bbox_head, + teacher_config, + teacher_ckpt=None, + eval_teacher=True, + train_cfg=None, + test_cfg=None, + pretrained=None): + super().__init__(backbone, neck, bbox_head, train_cfg, test_cfg, + pretrained) + self.eval_teacher = eval_teacher + # Build teacher model + if isinstance(teacher_config, str): + teacher_config = mmcv.Config.fromfile(teacher_config) + self.teacher_model = build_detector(teacher_config['model']) + if teacher_ckpt is not None: + load_checkpoint( + self.teacher_model, teacher_ckpt, map_location='cpu') + + def forward_train(self, + img, + img_metas, + gt_bboxes, + gt_labels, + gt_bboxes_ignore=None): + """ + Args: + img (Tensor): Input images of shape (N, C, H, W). + Typically these should be mean centered and std scaled. + img_metas (list[dict]): A List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + :class:`mmdet.datasets.pipelines.Collect`. + gt_bboxes (list[Tensor]): Each item are the truth boxes for each + image in [tl_x, tl_y, br_x, br_y] format. + gt_labels (list[Tensor]): Class indices corresponding to each box + gt_bboxes_ignore (None | list[Tensor]): Specify which bounding + boxes can be ignored when computing the loss. + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + x = self.extract_feat(img) + with torch.no_grad(): + teacher_x = self.teacher_model.extract_feat(img) + out_teacher = self.teacher_model.bbox_head(teacher_x) + losses = self.bbox_head.forward_train(x, out_teacher, img_metas, + gt_bboxes, gt_labels, + gt_bboxes_ignore) + return losses + + def cuda(self, device=None): + """Since teacher_model is registered as a plain object, it is necessary + to put the teacher model to cuda when calling cuda function.""" + self.teacher_model.cuda(device=device) + return super().cuda(device=device) + + def train(self, mode=True): + """Set the same train mode for teacher and student model.""" + if self.eval_teacher: + self.teacher_model.train(False) + else: + self.teacher_model.train(mode) + super().train(mode) + + def __setattr__(self, name, value): + """Set attribute, i.e. self.name = value + + This reloading prevent the teacher model from being registered as a + nn.Module. The teacher module is registered as a plain object, so that + the teacher parameters will not show up when calling + ``self.parameters``, ``self.modules``, ``self.children`` methods. + """ + if name == 'teacher_model': + object.__setattr__(self, name, value) + else: + super().__setattr__(name, value) diff --git a/detection/mmdet/models/detectors/mask_rcnn.py b/detection/mmdet/models/detectors/mask_rcnn.py new file mode 100644 index 0000000..c15a773 --- /dev/null +++ b/detection/mmdet/models/detectors/mask_rcnn.py @@ -0,0 +1,24 @@ +from ..builder import DETECTORS +from .two_stage import TwoStageDetector + + +@DETECTORS.register_module() +class MaskRCNN(TwoStageDetector): + """Implementation of `Mask R-CNN `_""" + + def __init__(self, + backbone, + rpn_head, + roi_head, + train_cfg, + test_cfg, + neck=None, + pretrained=None): + super(MaskRCNN, self).__init__( + backbone=backbone, + neck=neck, + rpn_head=rpn_head, + roi_head=roi_head, + train_cfg=train_cfg, + test_cfg=test_cfg, + pretrained=pretrained) diff --git a/detection/mmdet/models/detectors/mask_scoring_rcnn.py b/detection/mmdet/models/detectors/mask_scoring_rcnn.py new file mode 100644 index 0000000..b6252b6 --- /dev/null +++ b/detection/mmdet/models/detectors/mask_scoring_rcnn.py @@ -0,0 +1,27 @@ +from ..builder import DETECTORS +from .two_stage import TwoStageDetector + + +@DETECTORS.register_module() +class MaskScoringRCNN(TwoStageDetector): + """Mask Scoring RCNN. + + https://arxiv.org/abs/1903.00241 + """ + + def __init__(self, + backbone, + rpn_head, + roi_head, + train_cfg, + test_cfg, + neck=None, + pretrained=None): + super(MaskScoringRCNN, self).__init__( + backbone=backbone, + neck=neck, + rpn_head=rpn_head, + roi_head=roi_head, + train_cfg=train_cfg, + test_cfg=test_cfg, + pretrained=pretrained) diff --git a/detection/mmdet/models/detectors/nasfcos.py b/detection/mmdet/models/detectors/nasfcos.py new file mode 100644 index 0000000..fb01483 --- /dev/null +++ b/detection/mmdet/models/detectors/nasfcos.py @@ -0,0 +1,20 @@ +from ..builder import DETECTORS +from .single_stage import SingleStageDetector + + +@DETECTORS.register_module() +class NASFCOS(SingleStageDetector): + """NAS-FCOS: Fast Neural Architecture Search for Object Detection. + + https://arxiv.org/abs/1906.0442 + """ + + def __init__(self, + backbone, + neck, + bbox_head, + train_cfg=None, + test_cfg=None, + pretrained=None): + super(NASFCOS, self).__init__(backbone, neck, bbox_head, train_cfg, + test_cfg, pretrained) diff --git a/detection/mmdet/models/detectors/paa.py b/detection/mmdet/models/detectors/paa.py new file mode 100644 index 0000000..9b4bb5e --- /dev/null +++ b/detection/mmdet/models/detectors/paa.py @@ -0,0 +1,17 @@ +from ..builder import DETECTORS +from .single_stage import SingleStageDetector + + +@DETECTORS.register_module() +class PAA(SingleStageDetector): + """Implementation of `PAA `_.""" + + def __init__(self, + backbone, + neck, + bbox_head, + train_cfg=None, + test_cfg=None, + pretrained=None): + super(PAA, self).__init__(backbone, neck, bbox_head, train_cfg, + test_cfg, pretrained) diff --git a/detection/mmdet/models/detectors/point_rend.py b/detection/mmdet/models/detectors/point_rend.py new file mode 100644 index 0000000..808ef22 --- /dev/null +++ b/detection/mmdet/models/detectors/point_rend.py @@ -0,0 +1,29 @@ +from ..builder import DETECTORS +from .two_stage import TwoStageDetector + + +@DETECTORS.register_module() +class PointRend(TwoStageDetector): + """PointRend: Image Segmentation as Rendering + + This detector is the implementation of + `PointRend `_. + + """ + + def __init__(self, + backbone, + rpn_head, + roi_head, + train_cfg, + test_cfg, + neck=None, + pretrained=None): + super(PointRend, self).__init__( + backbone=backbone, + neck=neck, + rpn_head=rpn_head, + roi_head=roi_head, + train_cfg=train_cfg, + test_cfg=test_cfg, + pretrained=pretrained) diff --git a/detection/mmdet/models/detectors/reppoints_detector.py b/detection/mmdet/models/detectors/reppoints_detector.py new file mode 100644 index 0000000..a5f6be3 --- /dev/null +++ b/detection/mmdet/models/detectors/reppoints_detector.py @@ -0,0 +1,22 @@ +from ..builder import DETECTORS +from .single_stage import SingleStageDetector + + +@DETECTORS.register_module() +class RepPointsDetector(SingleStageDetector): + """RepPoints: Point Set Representation for Object Detection. + + This detector is the implementation of: + - RepPoints detector (https://arxiv.org/pdf/1904.11490) + """ + + def __init__(self, + backbone, + neck, + bbox_head, + train_cfg=None, + test_cfg=None, + pretrained=None): + super(RepPointsDetector, + self).__init__(backbone, neck, bbox_head, train_cfg, test_cfg, + pretrained) diff --git a/detection/mmdet/models/detectors/retinanet.py b/detection/mmdet/models/detectors/retinanet.py new file mode 100644 index 0000000..41378e8 --- /dev/null +++ b/detection/mmdet/models/detectors/retinanet.py @@ -0,0 +1,17 @@ +from ..builder import DETECTORS +from .single_stage import SingleStageDetector + + +@DETECTORS.register_module() +class RetinaNet(SingleStageDetector): + """Implementation of `RetinaNet `_""" + + def __init__(self, + backbone, + neck, + bbox_head, + train_cfg=None, + test_cfg=None, + pretrained=None): + super(RetinaNet, self).__init__(backbone, neck, bbox_head, train_cfg, + test_cfg, pretrained) diff --git a/detection/mmdet/models/detectors/rpn.py b/detection/mmdet/models/detectors/rpn.py new file mode 100644 index 0000000..1a77294 --- /dev/null +++ b/detection/mmdet/models/detectors/rpn.py @@ -0,0 +1,154 @@ +import mmcv +from mmcv.image import tensor2imgs + +from mmdet.core import bbox_mapping +from ..builder import DETECTORS, build_backbone, build_head, build_neck +from .base import BaseDetector + + +@DETECTORS.register_module() +class RPN(BaseDetector): + """Implementation of Region Proposal Network.""" + + def __init__(self, + backbone, + neck, + rpn_head, + train_cfg, + test_cfg, + pretrained=None): + super(RPN, self).__init__() + self.backbone = build_backbone(backbone) + self.neck = build_neck(neck) if neck is not None else None + rpn_train_cfg = train_cfg.rpn if train_cfg is not None else None + rpn_head.update(train_cfg=rpn_train_cfg) + rpn_head.update(test_cfg=test_cfg.rpn) + self.rpn_head = build_head(rpn_head) + self.train_cfg = train_cfg + self.test_cfg = test_cfg + self.init_weights(pretrained=pretrained) + + def init_weights(self, pretrained=None): + """Initialize the weights in detector. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + super(RPN, self).init_weights(pretrained) + self.backbone.init_weights(pretrained=pretrained) + if self.with_neck: + self.neck.init_weights() + self.rpn_head.init_weights() + + def extract_feat(self, img): + """Extract features. + + Args: + img (torch.Tensor): Image tensor with shape (n, c, h ,w). + + Returns: + list[torch.Tensor]: Multi-level features that may have + different resolutions. + """ + x = self.backbone(img) + if self.with_neck: + x = self.neck(x) + return x + + def forward_dummy(self, img): + """Dummy forward function.""" + x = self.extract_feat(img) + rpn_outs = self.rpn_head(x) + return rpn_outs + + def forward_train(self, + img, + img_metas, + gt_bboxes=None, + gt_bboxes_ignore=None): + """ + Args: + img (Tensor): Input images of shape (N, C, H, W). + Typically these should be mean centered and std scaled. + img_metas (list[dict]): A List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + :class:`mmdet.datasets.pipelines.Collect`. + gt_bboxes (list[Tensor]): Each item are the truth boxes for each + image in [tl_x, tl_y, br_x, br_y] format. + gt_bboxes_ignore (None | list[Tensor]): Specify which bounding + boxes can be ignored when computing the loss. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + if (isinstance(self.train_cfg.rpn, dict) + and self.train_cfg.rpn.get('debug', False)): + self.rpn_head.debug_imgs = tensor2imgs(img) + + x = self.extract_feat(img) + losses = self.rpn_head.forward_train(x, img_metas, gt_bboxes, None, + gt_bboxes_ignore) + return losses + + def simple_test(self, img, img_metas, rescale=False): + """Test function without test time augmentation. + + Args: + imgs (list[torch.Tensor]): List of multiple images + img_metas (list[dict]): List of image information. + rescale (bool, optional): Whether to rescale the results. + Defaults to False. + + Returns: + list[np.ndarray]: proposals + """ + x = self.extract_feat(img) + proposal_list = self.rpn_head.simple_test_rpn(x, img_metas) + if rescale: + for proposals, meta in zip(proposal_list, img_metas): + proposals[:, :4] /= proposals.new_tensor(meta['scale_factor']) + + return [proposal.cpu().numpy() for proposal in proposal_list] + + def aug_test(self, imgs, img_metas, rescale=False): + """Test function with test time augmentation. + + Args: + imgs (list[torch.Tensor]): List of multiple images + img_metas (list[dict]): List of image information. + rescale (bool, optional): Whether to rescale the results. + Defaults to False. + + Returns: + list[np.ndarray]: proposals + """ + proposal_list = self.rpn_head.aug_test_rpn( + self.extract_feats(imgs), img_metas) + if not rescale: + for proposals, img_meta in zip(proposal_list, img_metas[0]): + img_shape = img_meta['img_shape'] + scale_factor = img_meta['scale_factor'] + flip = img_meta['flip'] + flip_direction = img_meta['flip_direction'] + proposals[:, :4] = bbox_mapping(proposals[:, :4], img_shape, + scale_factor, flip, + flip_direction) + return [proposal.cpu().numpy() for proposal in proposal_list] + + def show_result(self, data, result, top_k=20, **kwargs): + """Show RPN proposals on the image. + + Args: + data (str or np.ndarray): Image filename or loaded image. + result (Tensor or tuple): The results to draw over `img` + bbox_result or (bbox_result, segm_result). + top_k (int): Plot the first k bboxes only + if set positive. Default: 20 + + Returns: + np.ndarray: The image with bboxes drawn on it. + """ + mmcv.imshow_bboxes(data, result, top_k=top_k) diff --git a/detection/mmdet/models/detectors/scnet.py b/detection/mmdet/models/detectors/scnet.py new file mode 100644 index 0000000..04a2347 --- /dev/null +++ b/detection/mmdet/models/detectors/scnet.py @@ -0,0 +1,10 @@ +from ..builder import DETECTORS +from .cascade_rcnn import CascadeRCNN + + +@DETECTORS.register_module() +class SCNet(CascadeRCNN): + """Implementation of `SCNet `_""" + + def __init__(self, **kwargs): + super(SCNet, self).__init__(**kwargs) diff --git a/detection/mmdet/models/detectors/single_stage.py b/detection/mmdet/models/detectors/single_stage.py new file mode 100644 index 0000000..5172bdb --- /dev/null +++ b/detection/mmdet/models/detectors/single_stage.py @@ -0,0 +1,154 @@ +import torch +import torch.nn as nn + +from mmdet.core import bbox2result +from ..builder import DETECTORS, build_backbone, build_head, build_neck +from .base import BaseDetector + + +@DETECTORS.register_module() +class SingleStageDetector(BaseDetector): + """Base class for single-stage detectors. + + Single-stage detectors directly and densely predict bounding boxes on the + output features of the backbone+neck. + """ + + def __init__(self, + backbone, + neck=None, + bbox_head=None, + train_cfg=None, + test_cfg=None, + pretrained=None): + super(SingleStageDetector, self).__init__() + self.backbone = build_backbone(backbone) + if neck is not None: + self.neck = build_neck(neck) + bbox_head.update(train_cfg=train_cfg) + bbox_head.update(test_cfg=test_cfg) + self.bbox_head = build_head(bbox_head) + self.train_cfg = train_cfg + self.test_cfg = test_cfg + self.init_weights(pretrained=pretrained) + + def init_weights(self, pretrained=None): + """Initialize the weights in detector. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + super(SingleStageDetector, self).init_weights(pretrained) + self.backbone.init_weights(pretrained=pretrained) + if self.with_neck: + if isinstance(self.neck, nn.Sequential): + for m in self.neck: + m.init_weights() + else: + self.neck.init_weights() + self.bbox_head.init_weights() + + def extract_feat(self, img): + """Directly extract features from the backbone+neck.""" + x = self.backbone(img) + if self.with_neck: + x = self.neck(x) + return x + + def forward_dummy(self, img): + """Used for computing network flops. + + See `mmdetection/tools/analysis_tools/get_flops.py` + """ + x = self.extract_feat(img) + outs = self.bbox_head(x) + return outs + + def forward_train(self, + img, + img_metas, + gt_bboxes, + gt_labels, + gt_bboxes_ignore=None): + """ + Args: + img (Tensor): Input images of shape (N, C, H, W). + Typically these should be mean centered and std scaled. + img_metas (list[dict]): A List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + :class:`mmdet.datasets.pipelines.Collect`. + gt_bboxes (list[Tensor]): Each item are the truth boxes for each + image in [tl_x, tl_y, br_x, br_y] format. + gt_labels (list[Tensor]): Class indices corresponding to each box + gt_bboxes_ignore (None | list[Tensor]): Specify which bounding + boxes can be ignored when computing the loss. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + super(SingleStageDetector, self).forward_train(img, img_metas) + x = self.extract_feat(img) + losses = self.bbox_head.forward_train(x, img_metas, gt_bboxes, + gt_labels, gt_bboxes_ignore) + return losses + + def simple_test(self, img, img_metas, rescale=False): + """Test function without test time augmentation. + + Args: + imgs (list[torch.Tensor]): List of multiple images + img_metas (list[dict]): List of image information. + rescale (bool, optional): Whether to rescale the results. + Defaults to False. + + Returns: + list[list[np.ndarray]]: BBox results of each image and classes. + The outer list corresponds to each image. The inner list + corresponds to each class. + """ + x = self.extract_feat(img) + outs = self.bbox_head(x) + # get origin input shape to support onnx dynamic shape + if torch.onnx.is_in_onnx_export(): + # get shape as tensor + img_shape = torch._shape_as_tensor(img)[2:] + img_metas[0]['img_shape_for_onnx'] = img_shape + bbox_list = self.bbox_head.get_bboxes( + *outs, img_metas, rescale=rescale) + # skip post-processing when exporting to ONNX + if torch.onnx.is_in_onnx_export(): + return bbox_list + + bbox_results = [ + bbox2result(det_bboxes, det_labels, self.bbox_head.num_classes) + for det_bboxes, det_labels in bbox_list + ] + return bbox_results + + def aug_test(self, imgs, img_metas, rescale=False): + """Test function with test time augmentation. + + Args: + imgs (list[Tensor]): the outer list indicates test-time + augmentations and inner Tensor should have a shape NxCxHxW, + which contains all images in the batch. + img_metas (list[list[dict]]): the outer list indicates test-time + augs (multiscale, flip, etc.) and the inner list indicates + images in a batch. each dict has image information. + rescale (bool, optional): Whether to rescale the results. + Defaults to False. + + Returns: + list[list[np.ndarray]]: BBox results of each image and classes. + The outer list corresponds to each image. The inner list + corresponds to each class. + """ + assert hasattr(self.bbox_head, 'aug_test'), \ + f'{self.bbox_head.__class__.__name__}' \ + ' does not support test-time augmentation' + + feats = self.extract_feats(imgs) + return [self.bbox_head.aug_test(feats, img_metas, rescale=rescale)] diff --git a/detection/mmdet/models/detectors/sparse_rcnn.py b/detection/mmdet/models/detectors/sparse_rcnn.py new file mode 100644 index 0000000..0dbd025 --- /dev/null +++ b/detection/mmdet/models/detectors/sparse_rcnn.py @@ -0,0 +1,110 @@ +from ..builder import DETECTORS +from .two_stage import TwoStageDetector + + +@DETECTORS.register_module() +class SparseRCNN(TwoStageDetector): + r"""Implementation of `Sparse R-CNN: End-to-End Object Detection with + Learnable Proposals `_""" + + def __init__(self, *args, **kwargs): + super(SparseRCNN, self).__init__(*args, **kwargs) + assert self.with_rpn, 'Sparse R-CNN do not support external proposals' + + def forward_train(self, + img, + img_metas, + gt_bboxes, + gt_labels, + gt_bboxes_ignore=None, + gt_masks=None, + proposals=None, + **kwargs): + """Forward function of SparseR-CNN in train stage. + + Args: + img (Tensor): of shape (N, C, H, W) encoding input images. + Typically these should be mean centered and std scaled. + img_metas (list[dict]): list of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + :class:`mmdet.datasets.pipelines.Collect`. + gt_bboxes (list[Tensor]): Ground truth bboxes for each image with + shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + gt_labels (list[Tensor]): class indices corresponding to each box + gt_bboxes_ignore (None | list[Tensor): specify which bounding + boxes can be ignored when computing the loss. + gt_masks (List[Tensor], optional) : Segmentation masks for + each box. But we don't support it in this architecture. + proposals (List[Tensor], optional): override rpn proposals with + custom proposals. Use when `with_rpn` is False. + + Returns: + dict[str, Tensor]: a dictionary of loss components + """ + + assert proposals is None, 'Sparse R-CNN does not support' \ + ' external proposals' + assert gt_masks is None, 'Sparse R-CNN does not instance segmentation' + + x = self.extract_feat(img) + proposal_boxes, proposal_features, imgs_whwh = \ + self.rpn_head.forward_train(x, img_metas) + roi_losses = self.roi_head.forward_train( + x, + proposal_boxes, + proposal_features, + img_metas, + gt_bboxes, + gt_labels, + gt_bboxes_ignore=gt_bboxes_ignore, + gt_masks=gt_masks, + imgs_whwh=imgs_whwh) + return roi_losses + + def simple_test(self, img, img_metas, rescale=False): + """Test function without test time augmentation. + + Args: + imgs (list[torch.Tensor]): List of multiple images + img_metas (list[dict]): List of image information. + rescale (bool): Whether to rescale the results. + Defaults to False. + + Returns: + list[list[np.ndarray]]: BBox results of each image and classes. + The outer list corresponds to each image. The inner list + corresponds to each class. + """ + x = self.extract_feat(img) + proposal_boxes, proposal_features, imgs_whwh = \ + self.rpn_head.simple_test_rpn(x, img_metas) + bbox_results = self.roi_head.simple_test( + x, + proposal_boxes, + proposal_features, + img_metas, + imgs_whwh=imgs_whwh, + rescale=rescale) + return bbox_results + + def forward_dummy(self, img): + """Used for computing network flops. + + See `mmdetection/tools/analysis_tools/get_flops.py` + """ + # backbone + x = self.extract_feat(img) + # rpn + num_imgs = len(img) + dummy_img_metas = [ + dict(img_shape=(800, 1333, 3)) for _ in range(num_imgs) + ] + proposal_boxes, proposal_features, imgs_whwh = \ + self.rpn_head.simple_test_rpn(x, dummy_img_metas) + # roi_head + roi_outs = self.roi_head.forward_dummy(x, proposal_boxes, + proposal_features, + dummy_img_metas) + return roi_outs diff --git a/detection/mmdet/models/detectors/trident_faster_rcnn.py b/detection/mmdet/models/detectors/trident_faster_rcnn.py new file mode 100644 index 0000000..f0fd80d --- /dev/null +++ b/detection/mmdet/models/detectors/trident_faster_rcnn.py @@ -0,0 +1,66 @@ +from ..builder import DETECTORS +from .faster_rcnn import FasterRCNN + + +@DETECTORS.register_module() +class TridentFasterRCNN(FasterRCNN): + """Implementation of `TridentNet `_""" + + def __init__(self, + backbone, + rpn_head, + roi_head, + train_cfg, + test_cfg, + neck=None, + pretrained=None): + + super(TridentFasterRCNN, self).__init__( + backbone=backbone, + neck=neck, + rpn_head=rpn_head, + roi_head=roi_head, + train_cfg=train_cfg, + test_cfg=test_cfg, + pretrained=pretrained) + assert self.backbone.num_branch == self.roi_head.num_branch + assert self.backbone.test_branch_idx == self.roi_head.test_branch_idx + self.num_branch = self.backbone.num_branch + self.test_branch_idx = self.backbone.test_branch_idx + + def simple_test(self, img, img_metas, proposals=None, rescale=False): + """Test without augmentation.""" + assert self.with_bbox, 'Bbox head must be implemented.' + x = self.extract_feat(img) + if proposals is None: + num_branch = (self.num_branch if self.test_branch_idx == -1 else 1) + trident_img_metas = img_metas * num_branch + proposal_list = self.rpn_head.simple_test_rpn(x, trident_img_metas) + else: + proposal_list = proposals + + return self.roi_head.simple_test( + x, proposal_list, trident_img_metas, rescale=rescale) + + def aug_test(self, imgs, img_metas, rescale=False): + """Test with augmentations. + + If rescale is False, then returned bboxes and masks will fit the scale + of imgs[0]. + """ + x = self.extract_feats(imgs) + num_branch = (self.num_branch if self.test_branch_idx == -1 else 1) + trident_img_metas = [img_metas * num_branch for img_metas in img_metas] + proposal_list = self.rpn_head.aug_test_rpn(x, trident_img_metas) + return self.roi_head.aug_test( + x, proposal_list, img_metas, rescale=rescale) + + def forward_train(self, img, img_metas, gt_bboxes, gt_labels, **kwargs): + """make copies of img and gts to fit multi-branch.""" + trident_gt_bboxes = tuple(gt_bboxes * self.num_branch) + trident_gt_labels = tuple(gt_labels * self.num_branch) + trident_img_metas = tuple(img_metas * self.num_branch) + + return super(TridentFasterRCNN, + self).forward_train(img, trident_img_metas, + trident_gt_bboxes, trident_gt_labels) diff --git a/detection/mmdet/models/detectors/two_stage.py b/detection/mmdet/models/detectors/two_stage.py new file mode 100644 index 0000000..ba5bdde --- /dev/null +++ b/detection/mmdet/models/detectors/two_stage.py @@ -0,0 +1,215 @@ +import torch +import torch.nn as nn + +# from mmdet.core import bbox2result, bbox2roi, build_assigner, build_sampler +from ..builder import DETECTORS, build_backbone, build_head, build_neck +from .base import BaseDetector + + +@DETECTORS.register_module() +class TwoStageDetector(BaseDetector): + """Base class for two-stage detectors. + + Two-stage detectors typically consisting of a region proposal network and a + task-specific regression head. + """ + + def __init__(self, + backbone, + neck=None, + rpn_head=None, + roi_head=None, + train_cfg=None, + test_cfg=None, + pretrained=None): + super(TwoStageDetector, self).__init__() + self.backbone = build_backbone(backbone) + + if neck is not None: + self.neck = build_neck(neck) + + if rpn_head is not None: + rpn_train_cfg = train_cfg.rpn if train_cfg is not None else None + rpn_head_ = rpn_head.copy() + rpn_head_.update(train_cfg=rpn_train_cfg, test_cfg=test_cfg.rpn) + self.rpn_head = build_head(rpn_head_) + + if roi_head is not None: + # update train and test cfg here for now + # TODO: refactor assigner & sampler + rcnn_train_cfg = train_cfg.rcnn if train_cfg is not None else None + roi_head.update(train_cfg=rcnn_train_cfg) + roi_head.update(test_cfg=test_cfg.rcnn) + self.roi_head = build_head(roi_head) + + self.train_cfg = train_cfg + self.test_cfg = test_cfg + + self.init_weights(pretrained=pretrained) + + @property + def with_rpn(self): + """bool: whether the detector has RPN""" + return hasattr(self, 'rpn_head') and self.rpn_head is not None + + @property + def with_roi_head(self): + """bool: whether the detector has a RoI head""" + return hasattr(self, 'roi_head') and self.roi_head is not None + + def init_weights(self, pretrained=None): + """Initialize the weights in detector. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + super(TwoStageDetector, self).init_weights(pretrained) + self.backbone.init_weights(pretrained=pretrained) + if self.with_neck: + if isinstance(self.neck, nn.Sequential): + for m in self.neck: + m.init_weights() + else: + self.neck.init_weights() + if self.with_rpn: + self.rpn_head.init_weights() + if self.with_roi_head: + self.roi_head.init_weights(pretrained) + + def extract_feat(self, img): + """Directly extract features from the backbone+neck.""" + x = self.backbone(img) + if self.with_neck: + x = self.neck(x) + return x + + def forward_dummy(self, img): + """Used for computing network flops. + + See `mmdetection/tools/analysis_tools/get_flops.py` + """ + outs = () + # backbone + x = self.extract_feat(img) + # rpn + if self.with_rpn: + rpn_outs = self.rpn_head(x) + outs = outs + (rpn_outs, ) + proposals = torch.randn(1000, 4).to(img.device) + # roi_head + roi_outs = self.roi_head.forward_dummy(x, proposals) + outs = outs + (roi_outs, ) + return outs + + def forward_train(self, + img, + img_metas, + gt_bboxes, + gt_labels, + gt_bboxes_ignore=None, + gt_masks=None, + proposals=None, + **kwargs): + """ + Args: + img (Tensor): of shape (N, C, H, W) encoding input images. + Typically these should be mean centered and std scaled. + + img_metas (list[dict]): list of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmdet/datasets/pipelines/formatting.py:Collect`. + + gt_bboxes (list[Tensor]): Ground truth bboxes for each image with + shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + + gt_labels (list[Tensor]): class indices corresponding to each box + + gt_bboxes_ignore (None | list[Tensor]): specify which bounding + boxes can be ignored when computing the loss. + + gt_masks (None | Tensor) : true segmentation masks for each box + used if the architecture supports a segmentation task. + + proposals : override rpn proposals with custom proposals. Use when + `with_rpn` is False. + + Returns: + dict[str, Tensor]: a dictionary of loss components + """ + x = self.extract_feat(img) + + losses = dict() + + # RPN forward and loss + if self.with_rpn: + proposal_cfg = self.train_cfg.get('rpn_proposal', + self.test_cfg.rpn) + rpn_losses, proposal_list = self.rpn_head.forward_train( + x, + img_metas, + gt_bboxes, + gt_labels=None, + gt_bboxes_ignore=gt_bboxes_ignore, + proposal_cfg=proposal_cfg) + losses.update(rpn_losses) + else: + proposal_list = proposals + + roi_losses = self.roi_head.forward_train(x, img_metas, proposal_list, + gt_bboxes, gt_labels, + gt_bboxes_ignore, gt_masks, + **kwargs) + losses.update(roi_losses) + + return losses + + async def async_simple_test(self, + img, + img_meta, + proposals=None, + rescale=False): + """Async test without augmentation.""" + assert self.with_bbox, 'Bbox head must be implemented.' + x = self.extract_feat(img) + + if proposals is None: + proposal_list = await self.rpn_head.async_simple_test_rpn( + x, img_meta) + else: + proposal_list = proposals + + return await self.roi_head.async_simple_test( + x, proposal_list, img_meta, rescale=rescale) + + def simple_test(self, img, img_metas, proposals=None, rescale=False): + """Test without augmentation.""" + assert self.with_bbox, 'Bbox head must be implemented.' + + x = self.extract_feat(img) + + # get origin input shape to onnx dynamic input shape + if torch.onnx.is_in_onnx_export(): + img_shape = torch._shape_as_tensor(img)[2:] + img_metas[0]['img_shape_for_onnx'] = img_shape + + if proposals is None: + proposal_list = self.rpn_head.simple_test_rpn(x, img_metas) + else: + proposal_list = proposals + + return self.roi_head.simple_test( + x, proposal_list, img_metas, rescale=rescale) + + def aug_test(self, imgs, img_metas, rescale=False): + """Test with augmentations. + + If rescale is False, then returned bboxes and masks will fit the scale + of imgs[0]. + """ + x = self.extract_feats(imgs) + proposal_list = self.rpn_head.aug_test_rpn(x, img_metas) + return self.roi_head.aug_test( + x, proposal_list, img_metas, rescale=rescale) diff --git a/detection/mmdet/models/detectors/vfnet.py b/detection/mmdet/models/detectors/vfnet.py new file mode 100644 index 0000000..e23f896 --- /dev/null +++ b/detection/mmdet/models/detectors/vfnet.py @@ -0,0 +1,18 @@ +from ..builder import DETECTORS +from .single_stage import SingleStageDetector + + +@DETECTORS.register_module() +class VFNet(SingleStageDetector): + """Implementation of `VarifocalNet + (VFNet).`_""" + + def __init__(self, + backbone, + neck, + bbox_head, + train_cfg=None, + test_cfg=None, + pretrained=None): + super(VFNet, self).__init__(backbone, neck, bbox_head, train_cfg, + test_cfg, pretrained) diff --git a/detection/mmdet/models/detectors/yolact.py b/detection/mmdet/models/detectors/yolact.py new file mode 100644 index 0000000..f32fde0 --- /dev/null +++ b/detection/mmdet/models/detectors/yolact.py @@ -0,0 +1,146 @@ +import torch + +from mmdet.core import bbox2result +from ..builder import DETECTORS, build_head +from .single_stage import SingleStageDetector + + +@DETECTORS.register_module() +class YOLACT(SingleStageDetector): + """Implementation of `YOLACT `_""" + + def __init__(self, + backbone, + neck, + bbox_head, + segm_head, + mask_head, + train_cfg=None, + test_cfg=None, + pretrained=None): + super(YOLACT, self).__init__(backbone, neck, bbox_head, train_cfg, + test_cfg, pretrained) + self.segm_head = build_head(segm_head) + self.mask_head = build_head(mask_head) + self.init_segm_mask_weights() + + def init_segm_mask_weights(self): + """Initialize weights of the YOLACT segm head and YOLACT mask head.""" + self.segm_head.init_weights() + self.mask_head.init_weights() + + def forward_dummy(self, img): + """Used for computing network flops. + + See `mmdetection/tools/analysis_tools/get_flops.py` + """ + raise NotImplementedError + + def forward_train(self, + img, + img_metas, + gt_bboxes, + gt_labels, + gt_bboxes_ignore=None, + gt_masks=None): + """ + Args: + img (Tensor): of shape (N, C, H, W) encoding input images. + Typically these should be mean centered and std scaled. + img_metas (list[dict]): list of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmdet/datasets/pipelines/formatting.py:Collect`. + gt_bboxes (list[Tensor]): Ground truth bboxes for each image with + shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + gt_labels (list[Tensor]): class indices corresponding to each box + gt_bboxes_ignore (None | list[Tensor]): specify which bounding + boxes can be ignored when computing the loss. + gt_masks (None | Tensor) : true segmentation masks for each box + used if the architecture supports a segmentation task. + + Returns: + dict[str, Tensor]: a dictionary of loss components + """ + # convert Bitmap mask or Polygon Mask to Tensor here + gt_masks = [ + gt_mask.to_tensor(dtype=torch.uint8, device=img.device) + for gt_mask in gt_masks + ] + + x = self.extract_feat(img) + + cls_score, bbox_pred, coeff_pred = self.bbox_head(x) + bbox_head_loss_inputs = (cls_score, bbox_pred) + (gt_bboxes, gt_labels, + img_metas) + losses, sampling_results = self.bbox_head.loss( + *bbox_head_loss_inputs, gt_bboxes_ignore=gt_bboxes_ignore) + + segm_head_outs = self.segm_head(x[0]) + loss_segm = self.segm_head.loss(segm_head_outs, gt_masks, gt_labels) + losses.update(loss_segm) + + mask_pred = self.mask_head(x[0], coeff_pred, gt_bboxes, img_metas, + sampling_results) + loss_mask = self.mask_head.loss(mask_pred, gt_masks, gt_bboxes, + img_metas, sampling_results) + losses.update(loss_mask) + + # check NaN and Inf + for loss_name in losses.keys(): + assert torch.isfinite(torch.stack(losses[loss_name]))\ + .all().item(), '{} becomes infinite or NaN!'\ + .format(loss_name) + + return losses + + def simple_test(self, img, img_metas, rescale=False): + """Test function without test time augmentation.""" + x = self.extract_feat(img) + + cls_score, bbox_pred, coeff_pred = self.bbox_head(x) + + bbox_inputs = (cls_score, bbox_pred, + coeff_pred) + (img_metas, self.test_cfg, rescale) + det_bboxes, det_labels, det_coeffs = self.bbox_head.get_bboxes( + *bbox_inputs) + bbox_results = [ + bbox2result(det_bbox, det_label, self.bbox_head.num_classes) + for det_bbox, det_label in zip(det_bboxes, det_labels) + ] + + num_imgs = len(img_metas) + scale_factors = tuple(meta['scale_factor'] for meta in img_metas) + if all(det_bbox.shape[0] == 0 for det_bbox in det_bboxes): + segm_results = [[[] for _ in range(self.mask_head.num_classes)] + for _ in range(num_imgs)] + else: + # if det_bboxes is rescaled to the original image size, we need to + # rescale it back to the testing scale to obtain RoIs. + if rescale and not isinstance(scale_factors[0], float): + scale_factors = [ + torch.from_numpy(scale_factor).to(det_bboxes[0].device) + for scale_factor in scale_factors + ] + _bboxes = [ + det_bboxes[i][:, :4] * + scale_factors[i] if rescale else det_bboxes[i][:, :4] + for i in range(len(det_bboxes)) + ] + mask_preds = self.mask_head(x[0], det_coeffs, _bboxes, img_metas) + # apply mask post-processing to each image individually + segm_results = [] + for i in range(num_imgs): + if det_bboxes[i].shape[0] == 0: + segm_results.append( + [[] for _ in range(self.mask_head.num_classes)]) + else: + segm_result = self.mask_head.get_seg_masks( + mask_preds[i], det_labels[i], img_metas[i], rescale) + segm_results.append(segm_result) + return list(zip(bbox_results, segm_results)) + + def aug_test(self, imgs, img_metas, rescale=False): + """Test with augmentations.""" + raise NotImplementedError diff --git a/detection/mmdet/models/detectors/yolo.py b/detection/mmdet/models/detectors/yolo.py new file mode 100644 index 0000000..240aab2 --- /dev/null +++ b/detection/mmdet/models/detectors/yolo.py @@ -0,0 +1,18 @@ +# Copyright (c) 2019 Western Digital Corporation or its affiliates. + +from ..builder import DETECTORS +from .single_stage import SingleStageDetector + + +@DETECTORS.register_module() +class YOLOV3(SingleStageDetector): + + def __init__(self, + backbone, + neck, + bbox_head, + train_cfg=None, + test_cfg=None, + pretrained=None): + super(YOLOV3, self).__init__(backbone, neck, bbox_head, train_cfg, + test_cfg, pretrained) diff --git a/detection/mmdet/models/losses/__init__.py b/detection/mmdet/models/losses/__init__.py new file mode 100644 index 0000000..297aa22 --- /dev/null +++ b/detection/mmdet/models/losses/__init__.py @@ -0,0 +1,29 @@ +from .accuracy import Accuracy, accuracy +from .ae_loss import AssociativeEmbeddingLoss +from .balanced_l1_loss import BalancedL1Loss, balanced_l1_loss +from .cross_entropy_loss import (CrossEntropyLoss, binary_cross_entropy, + cross_entropy, mask_cross_entropy) +from .focal_loss import FocalLoss, sigmoid_focal_loss +from .gaussian_focal_loss import GaussianFocalLoss +from .gfocal_loss import DistributionFocalLoss, QualityFocalLoss +from .ghm_loss import GHMC, GHMR +from .iou_loss import (BoundedIoULoss, CIoULoss, DIoULoss, GIoULoss, IoULoss, + bounded_iou_loss, iou_loss) +from .kd_loss import KnowledgeDistillationKLDivLoss +from .mse_loss import MSELoss, mse_loss +from .pisa_loss import carl_loss, isr_p +from .smooth_l1_loss import L1Loss, SmoothL1Loss, l1_loss, smooth_l1_loss +from .utils import reduce_loss, weight_reduce_loss, weighted_loss +from .varifocal_loss import VarifocalLoss + +__all__ = [ + 'accuracy', 'Accuracy', 'cross_entropy', 'binary_cross_entropy', + 'mask_cross_entropy', 'CrossEntropyLoss', 'sigmoid_focal_loss', + 'FocalLoss', 'smooth_l1_loss', 'SmoothL1Loss', 'balanced_l1_loss', + 'BalancedL1Loss', 'mse_loss', 'MSELoss', 'iou_loss', 'bounded_iou_loss', + 'IoULoss', 'BoundedIoULoss', 'GIoULoss', 'DIoULoss', 'CIoULoss', 'GHMC', + 'GHMR', 'reduce_loss', 'weight_reduce_loss', 'weighted_loss', 'L1Loss', + 'l1_loss', 'isr_p', 'carl_loss', 'AssociativeEmbeddingLoss', + 'GaussianFocalLoss', 'QualityFocalLoss', 'DistributionFocalLoss', + 'VarifocalLoss', 'KnowledgeDistillationKLDivLoss' +] diff --git a/detection/mmdet/models/losses/accuracy.py b/detection/mmdet/models/losses/accuracy.py new file mode 100644 index 0000000..789a224 --- /dev/null +++ b/detection/mmdet/models/losses/accuracy.py @@ -0,0 +1,78 @@ +import mmcv +import torch.nn as nn + + +@mmcv.jit(coderize=True) +def accuracy(pred, target, topk=1, thresh=None): + """Calculate accuracy according to the prediction and target. + + Args: + pred (torch.Tensor): The model prediction, shape (N, num_class) + target (torch.Tensor): The target of each prediction, shape (N, ) + topk (int | tuple[int], optional): If the predictions in ``topk`` + matches the target, the predictions will be regarded as + correct ones. Defaults to 1. + thresh (float, optional): If not None, predictions with scores under + this threshold are considered incorrect. Default to None. + + Returns: + float | tuple[float]: If the input ``topk`` is a single integer, + the function will return a single float as accuracy. If + ``topk`` is a tuple containing multiple integers, the + function will return a tuple containing accuracies of + each ``topk`` number. + """ + assert isinstance(topk, (int, tuple)) + if isinstance(topk, int): + topk = (topk, ) + return_single = True + else: + return_single = False + + maxk = max(topk) + if pred.size(0) == 0: + accu = [pred.new_tensor(0.) for i in range(len(topk))] + return accu[0] if return_single else accu + assert pred.ndim == 2 and target.ndim == 1 + assert pred.size(0) == target.size(0) + assert maxk <= pred.size(1), \ + f'maxk {maxk} exceeds pred dimension {pred.size(1)}' + pred_value, pred_label = pred.topk(maxk, dim=1) + pred_label = pred_label.t() # transpose to shape (maxk, N) + correct = pred_label.eq(target.view(1, -1).expand_as(pred_label)) + if thresh is not None: + # Only prediction values larger than thresh are counted as correct + correct = correct & (pred_value > thresh).t() + res = [] + for k in topk: + correct_k = correct[:k].reshape(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / pred.size(0))) + return res[0] if return_single else res + + +class Accuracy(nn.Module): + + def __init__(self, topk=(1, ), thresh=None): + """Module to calculate the accuracy. + + Args: + topk (tuple, optional): The criterion used to calculate the + accuracy. Defaults to (1,). + thresh (float, optional): If not None, predictions with scores + under this threshold are considered incorrect. Default to None. + """ + super().__init__() + self.topk = topk + self.thresh = thresh + + def forward(self, pred, target): + """Forward function to calculate accuracy. + + Args: + pred (torch.Tensor): Prediction of models. + target (torch.Tensor): Target for each prediction. + + Returns: + tuple[float]: The accuracies under different topk criterions. + """ + return accuracy(pred, target, self.topk, self.thresh) diff --git a/detection/mmdet/models/losses/ae_loss.py b/detection/mmdet/models/losses/ae_loss.py new file mode 100644 index 0000000..cff472a --- /dev/null +++ b/detection/mmdet/models/losses/ae_loss.py @@ -0,0 +1,102 @@ +import mmcv +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ..builder import LOSSES + + +@mmcv.jit(derivate=True, coderize=True) +def ae_loss_per_image(tl_preds, br_preds, match): + """Associative Embedding Loss in one image. + + Associative Embedding Loss including two parts: pull loss and push loss. + Pull loss makes embedding vectors from same object closer to each other. + Push loss distinguish embedding vector from different objects, and makes + the gap between them is large enough. + + During computing, usually there are 3 cases: + - no object in image: both pull loss and push loss will be 0. + - one object in image: push loss will be 0 and pull loss is computed + by the two corner of the only object. + - more than one objects in image: pull loss is computed by corner pairs + from each object, push loss is computed by each object with all + other objects. We use confusion matrix with 0 in diagonal to + compute the push loss. + + Args: + tl_preds (tensor): Embedding feature map of left-top corner. + br_preds (tensor): Embedding feature map of bottim-right corner. + match (list): Downsampled coordinates pair of each ground truth box. + """ + + tl_list, br_list, me_list = [], [], [] + if len(match) == 0: # no object in image + pull_loss = tl_preds.sum() * 0. + push_loss = tl_preds.sum() * 0. + else: + for m in match: + [tl_y, tl_x], [br_y, br_x] = m + tl_e = tl_preds[:, tl_y, tl_x].view(-1, 1) + br_e = br_preds[:, br_y, br_x].view(-1, 1) + tl_list.append(tl_e) + br_list.append(br_e) + me_list.append((tl_e + br_e) / 2.0) + + tl_list = torch.cat(tl_list) + br_list = torch.cat(br_list) + me_list = torch.cat(me_list) + + assert tl_list.size() == br_list.size() + + # N is object number in image, M is dimension of embedding vector + N, M = tl_list.size() + + pull_loss = (tl_list - me_list).pow(2) + (br_list - me_list).pow(2) + pull_loss = pull_loss.sum() / N + + margin = 1 # exp setting of CornerNet, details in section 3.3 of paper + + # confusion matrix of push loss + conf_mat = me_list.expand((N, N, M)).permute(1, 0, 2) - me_list + conf_weight = 1 - torch.eye(N).type_as(me_list) + conf_mat = conf_weight * (margin - conf_mat.sum(-1).abs()) + + if N > 1: # more than one object in current image + push_loss = F.relu(conf_mat).sum() / (N * (N - 1)) + else: + push_loss = tl_preds.sum() * 0. + + return pull_loss, push_loss + + +@LOSSES.register_module() +class AssociativeEmbeddingLoss(nn.Module): + """Associative Embedding Loss. + + More details can be found in + `Associative Embedding `_ and + `CornerNet `_ . + Code is modified from `kp_utils.py `_ # noqa: E501 + + Args: + pull_weight (float): Loss weight for corners from same object. + push_weight (float): Loss weight for corners from different object. + """ + + def __init__(self, pull_weight=0.25, push_weight=0.25): + super(AssociativeEmbeddingLoss, self).__init__() + self.pull_weight = pull_weight + self.push_weight = push_weight + + def forward(self, pred, target, match): + """Forward function.""" + batch = pred.size(0) + pull_all, push_all = 0.0, 0.0 + for i in range(batch): + pull, push = ae_loss_per_image(pred[i], target[i], match[i]) + + pull_all += self.pull_weight * pull + push_all += self.push_weight * push + + return pull_all, push_all diff --git a/detection/mmdet/models/losses/balanced_l1_loss.py b/detection/mmdet/models/losses/balanced_l1_loss.py new file mode 100644 index 0000000..7bcd13f --- /dev/null +++ b/detection/mmdet/models/losses/balanced_l1_loss.py @@ -0,0 +1,120 @@ +import mmcv +import numpy as np +import torch +import torch.nn as nn + +from ..builder import LOSSES +from .utils import weighted_loss + + +@mmcv.jit(derivate=True, coderize=True) +@weighted_loss +def balanced_l1_loss(pred, + target, + beta=1.0, + alpha=0.5, + gamma=1.5, + reduction='mean'): + """Calculate balanced L1 loss. + + Please see the `Libra R-CNN `_ + + Args: + pred (torch.Tensor): The prediction with shape (N, 4). + target (torch.Tensor): The learning target of the prediction with + shape (N, 4). + beta (float): The loss is a piecewise function of prediction and target + and ``beta`` serves as a threshold for the difference between the + prediction and target. Defaults to 1.0. + alpha (float): The denominator ``alpha`` in the balanced L1 loss. + Defaults to 0.5. + gamma (float): The ``gamma`` in the balanced L1 loss. + Defaults to 1.5. + reduction (str, optional): The method that reduces the loss to a + scalar. Options are "none", "mean" and "sum". + + Returns: + torch.Tensor: The calculated loss + """ + assert beta > 0 + assert pred.size() == target.size() and target.numel() > 0 + + diff = torch.abs(pred - target) + b = np.e**(gamma / alpha) - 1 + loss = torch.where( + diff < beta, alpha / b * + (b * diff + 1) * torch.log(b * diff / beta + 1) - alpha * diff, + gamma * diff + gamma / b - alpha * beta) + + return loss + + +@LOSSES.register_module() +class BalancedL1Loss(nn.Module): + """Balanced L1 Loss. + + arXiv: https://arxiv.org/pdf/1904.02701.pdf (CVPR 2019) + + Args: + alpha (float): The denominator ``alpha`` in the balanced L1 loss. + Defaults to 0.5. + gamma (float): The ``gamma`` in the balanced L1 loss. Defaults to 1.5. + beta (float, optional): The loss is a piecewise function of prediction + and target. ``beta`` serves as a threshold for the difference + between the prediction and target. Defaults to 1.0. + reduction (str, optional): The method that reduces the loss to a + scalar. Options are "none", "mean" and "sum". + loss_weight (float, optional): The weight of the loss. Defaults to 1.0 + """ + + def __init__(self, + alpha=0.5, + gamma=1.5, + beta=1.0, + reduction='mean', + loss_weight=1.0): + super(BalancedL1Loss, self).__init__() + self.alpha = alpha + self.gamma = gamma + self.beta = beta + self.reduction = reduction + self.loss_weight = loss_weight + + def forward(self, + pred, + target, + weight=None, + avg_factor=None, + reduction_override=None, + **kwargs): + """Forward function of loss. + + Args: + pred (torch.Tensor): The prediction with shape (N, 4). + target (torch.Tensor): The learning target of the prediction with + shape (N, 4). + weight (torch.Tensor, optional): Sample-wise loss weight with + shape (N, ). + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + reduction_override (str, optional): The reduction method used to + override the original reduction method of the loss. + Options are "none", "mean" and "sum". + + Returns: + torch.Tensor: The calculated loss + """ + assert reduction_override in (None, 'none', 'mean', 'sum') + reduction = ( + reduction_override if reduction_override else self.reduction) + loss_bbox = self.loss_weight * balanced_l1_loss( + pred, + target, + weight, + alpha=self.alpha, + gamma=self.gamma, + beta=self.beta, + reduction=reduction, + avg_factor=avg_factor, + **kwargs) + return loss_bbox diff --git a/detection/mmdet/models/losses/cross_entropy_loss.py b/detection/mmdet/models/losses/cross_entropy_loss.py new file mode 100644 index 0000000..5799415 --- /dev/null +++ b/detection/mmdet/models/losses/cross_entropy_loss.py @@ -0,0 +1,214 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ..builder import LOSSES +from .utils import weight_reduce_loss + + +def cross_entropy(pred, + label, + weight=None, + reduction='mean', + avg_factor=None, + class_weight=None): + """Calculate the CrossEntropy loss. + + Args: + pred (torch.Tensor): The prediction with shape (N, C), C is the number + of classes. + label (torch.Tensor): The learning label of the prediction. + weight (torch.Tensor, optional): Sample-wise loss weight. + reduction (str, optional): The method used to reduce the loss. + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + class_weight (list[float], optional): The weight for each class. + + Returns: + torch.Tensor: The calculated loss + """ + # element-wise losses + loss = F.cross_entropy(pred, label, weight=class_weight, reduction='none') + + # apply weights and do the reduction + if weight is not None: + weight = weight.float() + loss = weight_reduce_loss( + loss, weight=weight, reduction=reduction, avg_factor=avg_factor) + + return loss + + +def _expand_onehot_labels(labels, label_weights, label_channels): + bin_labels = labels.new_full((labels.size(0), label_channels), 0) + inds = torch.nonzero( + (labels >= 0) & (labels < label_channels), as_tuple=False).squeeze() + if inds.numel() > 0: + bin_labels[inds, labels[inds]] = 1 + + if label_weights is None: + bin_label_weights = None + else: + bin_label_weights = label_weights.view(-1, 1).expand( + label_weights.size(0), label_channels) + + return bin_labels, bin_label_weights + + +def binary_cross_entropy(pred, + label, + weight=None, + reduction='mean', + avg_factor=None, + class_weight=None): + """Calculate the binary CrossEntropy loss. + + Args: + pred (torch.Tensor): The prediction with shape (N, 1). + label (torch.Tensor): The learning label of the prediction. + weight (torch.Tensor, optional): Sample-wise loss weight. + reduction (str, optional): The method used to reduce the loss. + Options are "none", "mean" and "sum". + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + class_weight (list[float], optional): The weight for each class. + + Returns: + torch.Tensor: The calculated loss + """ + if pred.dim() != label.dim(): + label, weight = _expand_onehot_labels(label, weight, pred.size(-1)) + + # weighted element-wise losses + if weight is not None: + weight = weight.float() + loss = F.binary_cross_entropy_with_logits( + pred, label.float(), pos_weight=class_weight, reduction='none') + # do the reduction for the weighted loss + loss = weight_reduce_loss( + loss, weight, reduction=reduction, avg_factor=avg_factor) + + return loss + + +def mask_cross_entropy(pred, + target, + label, + reduction='mean', + avg_factor=None, + class_weight=None): + """Calculate the CrossEntropy loss for masks. + + Args: + pred (torch.Tensor): The prediction with shape (N, C, *), C is the + number of classes. The trailing * indicates arbitrary shape. + target (torch.Tensor): The learning label of the prediction. + label (torch.Tensor): ``label`` indicates the class label of the mask + corresponding object. This will be used to select the mask in the + of the class which the object belongs to when the mask prediction + if not class-agnostic. + reduction (str, optional): The method used to reduce the loss. + Options are "none", "mean" and "sum". + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + class_weight (list[float], optional): The weight for each class. + + Returns: + torch.Tensor: The calculated loss + + Example: + >>> N, C = 3, 11 + >>> H, W = 2, 2 + >>> pred = torch.randn(N, C, H, W) * 1000 + >>> target = torch.rand(N, H, W) + >>> label = torch.randint(0, C, size=(N,)) + >>> reduction = 'mean' + >>> avg_factor = None + >>> class_weights = None + >>> loss = mask_cross_entropy(pred, target, label, reduction, + >>> avg_factor, class_weights) + >>> assert loss.shape == (1,) + """ + # TODO: handle these two reserved arguments + assert reduction == 'mean' and avg_factor is None + num_rois = pred.size()[0] + inds = torch.arange(0, num_rois, dtype=torch.long, device=pred.device) + pred_slice = pred[inds, label].squeeze(1) + return F.binary_cross_entropy_with_logits( + pred_slice, target, weight=class_weight, reduction='mean')[None] + + +@LOSSES.register_module() +class CrossEntropyLoss(nn.Module): + + def __init__(self, + use_sigmoid=False, + use_mask=False, + reduction='mean', + class_weight=None, + loss_weight=1.0): + """CrossEntropyLoss. + + Args: + use_sigmoid (bool, optional): Whether the prediction uses sigmoid + of softmax. Defaults to False. + use_mask (bool, optional): Whether to use mask cross entropy loss. + Defaults to False. + reduction (str, optional): . Defaults to 'mean'. + Options are "none", "mean" and "sum". + class_weight (list[float], optional): Weight of each class. + Defaults to None. + loss_weight (float, optional): Weight of the loss. Defaults to 1.0. + """ + super(CrossEntropyLoss, self).__init__() + assert (use_sigmoid is False) or (use_mask is False) + self.use_sigmoid = use_sigmoid + self.use_mask = use_mask + self.reduction = reduction + self.loss_weight = loss_weight + self.class_weight = class_weight + + if self.use_sigmoid: + self.cls_criterion = binary_cross_entropy + elif self.use_mask: + self.cls_criterion = mask_cross_entropy + else: + self.cls_criterion = cross_entropy + + def forward(self, + cls_score, + label, + weight=None, + avg_factor=None, + reduction_override=None, + **kwargs): + """Forward function. + + Args: + cls_score (torch.Tensor): The prediction. + label (torch.Tensor): The learning label of the prediction. + weight (torch.Tensor, optional): Sample-wise loss weight. + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + reduction (str, optional): The method used to reduce the loss. + Options are "none", "mean" and "sum". + Returns: + torch.Tensor: The calculated loss + """ + assert reduction_override in (None, 'none', 'mean', 'sum') + reduction = ( + reduction_override if reduction_override else self.reduction) + if self.class_weight is not None: + class_weight = cls_score.new_tensor( + self.class_weight, device=cls_score.device) + else: + class_weight = None + loss_cls = self.loss_weight * self.cls_criterion( + cls_score, + label, + weight, + class_weight=class_weight, + reduction=reduction, + avg_factor=avg_factor, + **kwargs) + return loss_cls diff --git a/detection/mmdet/models/losses/focal_loss.py b/detection/mmdet/models/losses/focal_loss.py new file mode 100644 index 0000000..493907c --- /dev/null +++ b/detection/mmdet/models/losses/focal_loss.py @@ -0,0 +1,181 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.ops import sigmoid_focal_loss as _sigmoid_focal_loss + +from ..builder import LOSSES +from .utils import weight_reduce_loss + + +# This method is only for debugging +def py_sigmoid_focal_loss(pred, + target, + weight=None, + gamma=2.0, + alpha=0.25, + reduction='mean', + avg_factor=None): + """PyTorch version of `Focal Loss `_. + + Args: + pred (torch.Tensor): The prediction with shape (N, C), C is the + number of classes + target (torch.Tensor): The learning label of the prediction. + weight (torch.Tensor, optional): Sample-wise loss weight. + gamma (float, optional): The gamma for calculating the modulating + factor. Defaults to 2.0. + alpha (float, optional): A balanced form for Focal Loss. + Defaults to 0.25. + reduction (str, optional): The method used to reduce the loss into + a scalar. Defaults to 'mean'. + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + """ + pred_sigmoid = pred.sigmoid() + target = target.type_as(pred) + pt = (1 - pred_sigmoid) * target + pred_sigmoid * (1 - target) + focal_weight = (alpha * target + (1 - alpha) * + (1 - target)) * pt.pow(gamma) + loss = F.binary_cross_entropy_with_logits( + pred, target, reduction='none') * focal_weight + if weight is not None: + if weight.shape != loss.shape: + if weight.size(0) == loss.size(0): + # For most cases, weight is of shape (num_priors, ), + # which means it does not have the second axis num_class + weight = weight.view(-1, 1) + else: + # Sometimes, weight per anchor per class is also needed. e.g. + # in FSAF. But it may be flattened of shape + # (num_priors x num_class, ), while loss is still of shape + # (num_priors, num_class). + assert weight.numel() == loss.numel() + weight = weight.view(loss.size(0), -1) + assert weight.ndim == loss.ndim + loss = weight_reduce_loss(loss, weight, reduction, avg_factor) + return loss + + +def sigmoid_focal_loss(pred, + target, + weight=None, + gamma=2.0, + alpha=0.25, + reduction='mean', + avg_factor=None): + r"""A warpper of cuda version `Focal Loss + `_. + + Args: + pred (torch.Tensor): The prediction with shape (N, C), C is the number + of classes. + target (torch.Tensor): The learning label of the prediction. + weight (torch.Tensor, optional): Sample-wise loss weight. + gamma (float, optional): The gamma for calculating the modulating + factor. Defaults to 2.0. + alpha (float, optional): A balanced form for Focal Loss. + Defaults to 0.25. + reduction (str, optional): The method used to reduce the loss into + a scalar. Defaults to 'mean'. Options are "none", "mean" and "sum". + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + """ + # Function.apply does not accept keyword arguments, so the decorator + # "weighted_loss" is not applicable + loss = _sigmoid_focal_loss(pred.contiguous(), target, gamma, alpha, None, + 'none') + if weight is not None: + if weight.shape != loss.shape: + if weight.size(0) == loss.size(0): + # For most cases, weight is of shape (num_priors, ), + # which means it does not have the second axis num_class + weight = weight.view(-1, 1) + else: + # Sometimes, weight per anchor per class is also needed. e.g. + # in FSAF. But it may be flattened of shape + # (num_priors x num_class, ), while loss is still of shape + # (num_priors, num_class). + assert weight.numel() == loss.numel() + weight = weight.view(loss.size(0), -1) + assert weight.ndim == loss.ndim + loss = weight_reduce_loss(loss, weight, reduction, avg_factor) + return loss + + +@LOSSES.register_module() +class FocalLoss(nn.Module): + + def __init__(self, + use_sigmoid=True, + gamma=2.0, + alpha=0.25, + reduction='mean', + loss_weight=1.0): + """`Focal Loss `_ + + Args: + use_sigmoid (bool, optional): Whether to the prediction is + used for sigmoid or softmax. Defaults to True. + gamma (float, optional): The gamma for calculating the modulating + factor. Defaults to 2.0. + alpha (float, optional): A balanced form for Focal Loss. + Defaults to 0.25. + reduction (str, optional): The method used to reduce the loss into + a scalar. Defaults to 'mean'. Options are "none", "mean" and + "sum". + loss_weight (float, optional): Weight of loss. Defaults to 1.0. + """ + super(FocalLoss, self).__init__() + assert use_sigmoid is True, 'Only sigmoid focal loss supported now.' + self.use_sigmoid = use_sigmoid + self.gamma = gamma + self.alpha = alpha + self.reduction = reduction + self.loss_weight = loss_weight + + def forward(self, + pred, + target, + weight=None, + avg_factor=None, + reduction_override=None): + """Forward function. + + Args: + pred (torch.Tensor): The prediction. + target (torch.Tensor): The learning label of the prediction. + weight (torch.Tensor, optional): The weight of loss for each + prediction. Defaults to None. + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + reduction_override (str, optional): The reduction method used to + override the original reduction method of the loss. + Options are "none", "mean" and "sum". + + Returns: + torch.Tensor: The calculated loss + """ + assert reduction_override in (None, 'none', 'mean', 'sum') + reduction = ( + reduction_override if reduction_override else self.reduction) + if self.use_sigmoid: + if torch.cuda.is_available() and pred.is_cuda: + calculate_loss_func = sigmoid_focal_loss + else: + num_classes = pred.size(1) + target = F.one_hot(target, num_classes=num_classes + 1) + target = target[:, :num_classes] + calculate_loss_func = py_sigmoid_focal_loss + + loss_cls = self.loss_weight * calculate_loss_func( + pred, + target, + weight, + gamma=self.gamma, + alpha=self.alpha, + reduction=reduction, + avg_factor=avg_factor) + + else: + raise NotImplementedError + return loss_cls diff --git a/detection/mmdet/models/losses/gaussian_focal_loss.py b/detection/mmdet/models/losses/gaussian_focal_loss.py new file mode 100644 index 0000000..e45506a --- /dev/null +++ b/detection/mmdet/models/losses/gaussian_focal_loss.py @@ -0,0 +1,91 @@ +import mmcv +import torch.nn as nn + +from ..builder import LOSSES +from .utils import weighted_loss + + +@mmcv.jit(derivate=True, coderize=True) +@weighted_loss +def gaussian_focal_loss(pred, gaussian_target, alpha=2.0, gamma=4.0): + """`Focal Loss `_ for targets in gaussian + distribution. + + Args: + pred (torch.Tensor): The prediction. + gaussian_target (torch.Tensor): The learning target of the prediction + in gaussian distribution. + alpha (float, optional): A balanced form for Focal Loss. + Defaults to 2.0. + gamma (float, optional): The gamma for calculating the modulating + factor. Defaults to 4.0. + """ + eps = 1e-12 + pos_weights = gaussian_target.eq(1) + neg_weights = (1 - gaussian_target).pow(gamma) + pos_loss = -(pred + eps).log() * (1 - pred).pow(alpha) * pos_weights + neg_loss = -(1 - pred + eps).log() * pred.pow(alpha) * neg_weights + return pos_loss + neg_loss + + +@LOSSES.register_module() +class GaussianFocalLoss(nn.Module): + """GaussianFocalLoss is a variant of focal loss. + + More details can be found in the `paper + `_ + Code is modified from `kp_utils.py + `_ # noqa: E501 + Please notice that the target in GaussianFocalLoss is a gaussian heatmap, + not 0/1 binary target. + + Args: + alpha (float): Power of prediction. + gamma (float): Power of target for negative samples. + reduction (str): Options are "none", "mean" and "sum". + loss_weight (float): Loss weight of current loss. + """ + + def __init__(self, + alpha=2.0, + gamma=4.0, + reduction='mean', + loss_weight=1.0): + super(GaussianFocalLoss, self).__init__() + self.alpha = alpha + self.gamma = gamma + self.reduction = reduction + self.loss_weight = loss_weight + + def forward(self, + pred, + target, + weight=None, + avg_factor=None, + reduction_override=None): + """Forward function. + + Args: + pred (torch.Tensor): The prediction. + target (torch.Tensor): The learning target of the prediction + in gaussian distribution. + weight (torch.Tensor, optional): The weight of loss for each + prediction. Defaults to None. + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + reduction_override (str, optional): The reduction method used to + override the original reduction method of the loss. + Defaults to None. + """ + assert reduction_override in (None, 'none', 'mean', 'sum') + reduction = ( + reduction_override if reduction_override else self.reduction) + loss_reg = self.loss_weight * gaussian_focal_loss( + pred, + target, + weight, + alpha=self.alpha, + gamma=self.gamma, + reduction=reduction, + avg_factor=avg_factor) + return loss_reg diff --git a/detection/mmdet/models/losses/gfocal_loss.py b/detection/mmdet/models/losses/gfocal_loss.py new file mode 100644 index 0000000..9d3b883 --- /dev/null +++ b/detection/mmdet/models/losses/gfocal_loss.py @@ -0,0 +1,188 @@ +import mmcv +import torch.nn as nn +import torch.nn.functional as F + +from ..builder import LOSSES +from .utils import weighted_loss + + +@mmcv.jit(derivate=True, coderize=True) +@weighted_loss +def quality_focal_loss(pred, target, beta=2.0): + r"""Quality Focal Loss (QFL) is from `Generalized Focal Loss: Learning + Qualified and Distributed Bounding Boxes for Dense Object Detection + `_. + + Args: + pred (torch.Tensor): Predicted joint representation of classification + and quality (IoU) estimation with shape (N, C), C is the number of + classes. + target (tuple([torch.Tensor])): Target category label with shape (N,) + and target quality label with shape (N,). + beta (float): The beta parameter for calculating the modulating factor. + Defaults to 2.0. + + Returns: + torch.Tensor: Loss tensor with shape (N,). + """ + assert len(target) == 2, """target for QFL must be a tuple of two elements, + including category label and quality label, respectively""" + # label denotes the category id, score denotes the quality score + label, score = target + + # negatives are supervised by 0 quality score + pred_sigmoid = pred.sigmoid() + scale_factor = pred_sigmoid + zerolabel = scale_factor.new_zeros(pred.shape) + loss = F.binary_cross_entropy_with_logits( + pred, zerolabel, reduction='none') * scale_factor.pow(beta) + + # FG cat_id: [0, num_classes -1], BG cat_id: num_classes + bg_class_ind = pred.size(1) + pos = ((label >= 0) & (label < bg_class_ind)).nonzero().squeeze(1) + pos_label = label[pos].long() + # positives are supervised by bbox quality (IoU) score + scale_factor = score[pos] - pred_sigmoid[pos, pos_label] + loss[pos, pos_label] = F.binary_cross_entropy_with_logits( + pred[pos, pos_label], score[pos], + reduction='none') * scale_factor.abs().pow(beta) + + loss = loss.sum(dim=1, keepdim=False) + return loss + + +@mmcv.jit(derivate=True, coderize=True) +@weighted_loss +def distribution_focal_loss(pred, label): + r"""Distribution Focal Loss (DFL) is from `Generalized Focal Loss: Learning + Qualified and Distributed Bounding Boxes for Dense Object Detection + `_. + + Args: + pred (torch.Tensor): Predicted general distribution of bounding boxes + (before softmax) with shape (N, n+1), n is the max value of the + integral set `{0, ..., n}` in paper. + label (torch.Tensor): Target distance label for bounding boxes with + shape (N,). + + Returns: + torch.Tensor: Loss tensor with shape (N,). + """ + dis_left = label.long() + dis_right = dis_left + 1 + weight_left = dis_right.float() - label + weight_right = label - dis_left.float() + loss = F.cross_entropy(pred, dis_left, reduction='none') * weight_left \ + + F.cross_entropy(pred, dis_right, reduction='none') * weight_right + return loss + + +@LOSSES.register_module() +class QualityFocalLoss(nn.Module): + r"""Quality Focal Loss (QFL) is a variant of `Generalized Focal Loss: + Learning Qualified and Distributed Bounding Boxes for Dense Object + Detection `_. + + Args: + use_sigmoid (bool): Whether sigmoid operation is conducted in QFL. + Defaults to True. + beta (float): The beta parameter for calculating the modulating factor. + Defaults to 2.0. + reduction (str): Options are "none", "mean" and "sum". + loss_weight (float): Loss weight of current loss. + """ + + def __init__(self, + use_sigmoid=True, + beta=2.0, + reduction='mean', + loss_weight=1.0): + super(QualityFocalLoss, self).__init__() + assert use_sigmoid is True, 'Only sigmoid in QFL supported now.' + self.use_sigmoid = use_sigmoid + self.beta = beta + self.reduction = reduction + self.loss_weight = loss_weight + + def forward(self, + pred, + target, + weight=None, + avg_factor=None, + reduction_override=None): + """Forward function. + + Args: + pred (torch.Tensor): Predicted joint representation of + classification and quality (IoU) estimation with shape (N, C), + C is the number of classes. + target (tuple([torch.Tensor])): Target category label with shape + (N,) and target quality label with shape (N,). + weight (torch.Tensor, optional): The weight of loss for each + prediction. Defaults to None. + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + reduction_override (str, optional): The reduction method used to + override the original reduction method of the loss. + Defaults to None. + """ + assert reduction_override in (None, 'none', 'mean', 'sum') + reduction = ( + reduction_override if reduction_override else self.reduction) + if self.use_sigmoid: + loss_cls = self.loss_weight * quality_focal_loss( + pred, + target, + weight, + beta=self.beta, + reduction=reduction, + avg_factor=avg_factor) + else: + raise NotImplementedError + return loss_cls + + +@LOSSES.register_module() +class DistributionFocalLoss(nn.Module): + r"""Distribution Focal Loss (DFL) is a variant of `Generalized Focal Loss: + Learning Qualified and Distributed Bounding Boxes for Dense Object + Detection `_. + + Args: + reduction (str): Options are `'none'`, `'mean'` and `'sum'`. + loss_weight (float): Loss weight of current loss. + """ + + def __init__(self, reduction='mean', loss_weight=1.0): + super(DistributionFocalLoss, self).__init__() + self.reduction = reduction + self.loss_weight = loss_weight + + def forward(self, + pred, + target, + weight=None, + avg_factor=None, + reduction_override=None): + """Forward function. + + Args: + pred (torch.Tensor): Predicted general distribution of bounding + boxes (before softmax) with shape (N, n+1), n is the max value + of the integral set `{0, ..., n}` in paper. + target (torch.Tensor): Target distance label for bounding boxes + with shape (N,). + weight (torch.Tensor, optional): The weight of loss for each + prediction. Defaults to None. + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + reduction_override (str, optional): The reduction method used to + override the original reduction method of the loss. + Defaults to None. + """ + assert reduction_override in (None, 'none', 'mean', 'sum') + reduction = ( + reduction_override if reduction_override else self.reduction) + loss_cls = self.loss_weight * distribution_focal_loss( + pred, target, weight, reduction=reduction, avg_factor=avg_factor) + return loss_cls diff --git a/detection/mmdet/models/losses/ghm_loss.py b/detection/mmdet/models/losses/ghm_loss.py new file mode 100644 index 0000000..8969a23 --- /dev/null +++ b/detection/mmdet/models/losses/ghm_loss.py @@ -0,0 +1,172 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ..builder import LOSSES + + +def _expand_onehot_labels(labels, label_weights, label_channels): + bin_labels = labels.new_full((labels.size(0), label_channels), 0) + inds = torch.nonzero( + (labels >= 0) & (labels < label_channels), as_tuple=False).squeeze() + if inds.numel() > 0: + bin_labels[inds, labels[inds]] = 1 + bin_label_weights = label_weights.view(-1, 1).expand( + label_weights.size(0), label_channels) + return bin_labels, bin_label_weights + + +# TODO: code refactoring to make it consistent with other losses +@LOSSES.register_module() +class GHMC(nn.Module): + """GHM Classification Loss. + + Details of the theorem can be viewed in the paper + `Gradient Harmonized Single-stage Detector + `_. + + Args: + bins (int): Number of the unit regions for distribution calculation. + momentum (float): The parameter for moving average. + use_sigmoid (bool): Can only be true for BCE based loss now. + loss_weight (float): The weight of the total GHM-C loss. + """ + + def __init__(self, bins=10, momentum=0, use_sigmoid=True, loss_weight=1.0): + super(GHMC, self).__init__() + self.bins = bins + self.momentum = momentum + edges = torch.arange(bins + 1).float() / bins + self.register_buffer('edges', edges) + self.edges[-1] += 1e-6 + if momentum > 0: + acc_sum = torch.zeros(bins) + self.register_buffer('acc_sum', acc_sum) + self.use_sigmoid = use_sigmoid + if not self.use_sigmoid: + raise NotImplementedError + self.loss_weight = loss_weight + + def forward(self, pred, target, label_weight, *args, **kwargs): + """Calculate the GHM-C loss. + + Args: + pred (float tensor of size [batch_num, class_num]): + The direct prediction of classification fc layer. + target (float tensor of size [batch_num, class_num]): + Binary class target for each sample. + label_weight (float tensor of size [batch_num, class_num]): + the value is 1 if the sample is valid and 0 if ignored. + Returns: + The gradient harmonized loss. + """ + # the target should be binary class label + if pred.dim() != target.dim(): + target, label_weight = _expand_onehot_labels( + target, label_weight, pred.size(-1)) + target, label_weight = target.float(), label_weight.float() + edges = self.edges + mmt = self.momentum + weights = torch.zeros_like(pred) + + # gradient length + g = torch.abs(pred.sigmoid().detach() - target) + + valid = label_weight > 0 + tot = max(valid.float().sum().item(), 1.0) + n = 0 # n valid bins + for i in range(self.bins): + inds = (g >= edges[i]) & (g < edges[i + 1]) & valid + num_in_bin = inds.sum().item() + if num_in_bin > 0: + if mmt > 0: + self.acc_sum[i] = mmt * self.acc_sum[i] \ + + (1 - mmt) * num_in_bin + weights[inds] = tot / self.acc_sum[i] + else: + weights[inds] = tot / num_in_bin + n += 1 + if n > 0: + weights = weights / n + + loss = F.binary_cross_entropy_with_logits( + pred, target, weights, reduction='sum') / tot + return loss * self.loss_weight + + +# TODO: code refactoring to make it consistent with other losses +@LOSSES.register_module() +class GHMR(nn.Module): + """GHM Regression Loss. + + Details of the theorem can be viewed in the paper + `Gradient Harmonized Single-stage Detector + `_. + + Args: + mu (float): The parameter for the Authentic Smooth L1 loss. + bins (int): Number of the unit regions for distribution calculation. + momentum (float): The parameter for moving average. + loss_weight (float): The weight of the total GHM-R loss. + """ + + def __init__(self, mu=0.02, bins=10, momentum=0, loss_weight=1.0): + super(GHMR, self).__init__() + self.mu = mu + self.bins = bins + edges = torch.arange(bins + 1).float() / bins + self.register_buffer('edges', edges) + self.edges[-1] = 1e3 + self.momentum = momentum + if momentum > 0: + acc_sum = torch.zeros(bins) + self.register_buffer('acc_sum', acc_sum) + self.loss_weight = loss_weight + + # TODO: support reduction parameter + def forward(self, pred, target, label_weight, avg_factor=None): + """Calculate the GHM-R loss. + + Args: + pred (float tensor of size [batch_num, 4 (* class_num)]): + The prediction of box regression layer. Channel number can be 4 + or 4 * class_num depending on whether it is class-agnostic. + target (float tensor of size [batch_num, 4 (* class_num)]): + The target regression values with the same size of pred. + label_weight (float tensor of size [batch_num, 4 (* class_num)]): + The weight of each sample, 0 if ignored. + Returns: + The gradient harmonized loss. + """ + mu = self.mu + edges = self.edges + mmt = self.momentum + + # ASL1 loss + diff = pred - target + loss = torch.sqrt(diff * diff + mu * mu) - mu + + # gradient length + g = torch.abs(diff / torch.sqrt(mu * mu + diff * diff)).detach() + weights = torch.zeros_like(g) + + valid = label_weight > 0 + tot = max(label_weight.float().sum().item(), 1.0) + n = 0 # n: valid bins + for i in range(self.bins): + inds = (g >= edges[i]) & (g < edges[i + 1]) & valid + num_in_bin = inds.sum().item() + if num_in_bin > 0: + n += 1 + if mmt > 0: + self.acc_sum[i] = mmt * self.acc_sum[i] \ + + (1 - mmt) * num_in_bin + weights[inds] = tot / self.acc_sum[i] + else: + weights[inds] = tot / num_in_bin + if n > 0: + weights /= n + + loss = loss * weights + loss = loss.sum() / tot + return loss * self.loss_weight diff --git a/detection/mmdet/models/losses/iou_loss.py b/detection/mmdet/models/losses/iou_loss.py new file mode 100644 index 0000000..eba6f18 --- /dev/null +++ b/detection/mmdet/models/losses/iou_loss.py @@ -0,0 +1,436 @@ +import math + +import mmcv +import torch +import torch.nn as nn + +from mmdet.core import bbox_overlaps +from ..builder import LOSSES +from .utils import weighted_loss + + +@mmcv.jit(derivate=True, coderize=True) +@weighted_loss +def iou_loss(pred, target, linear=False, eps=1e-6): + """IoU loss. + + Computing the IoU loss between a set of predicted bboxes and target bboxes. + The loss is calculated as negative log of IoU. + + Args: + pred (torch.Tensor): Predicted bboxes of format (x1, y1, x2, y2), + shape (n, 4). + target (torch.Tensor): Corresponding gt bboxes, shape (n, 4). + linear (bool, optional): If True, use linear scale of loss instead of + log scale. Default: False. + eps (float): Eps to avoid log(0). + + Return: + torch.Tensor: Loss tensor. + """ + ious = bbox_overlaps(pred, target, is_aligned=True).clamp(min=eps) + if linear: + loss = 1 - ious + else: + loss = -ious.log() + return loss + + +@mmcv.jit(derivate=True, coderize=True) +@weighted_loss +def bounded_iou_loss(pred, target, beta=0.2, eps=1e-3): + """BIoULoss. + + This is an implementation of paper + `Improving Object Localization with Fitness NMS and Bounded IoU Loss. + `_. + + Args: + pred (torch.Tensor): Predicted bboxes. + target (torch.Tensor): Target bboxes. + beta (float): beta parameter in smoothl1. + eps (float): eps to avoid NaN. + """ + pred_ctrx = (pred[:, 0] + pred[:, 2]) * 0.5 + pred_ctry = (pred[:, 1] + pred[:, 3]) * 0.5 + pred_w = pred[:, 2] - pred[:, 0] + pred_h = pred[:, 3] - pred[:, 1] + with torch.no_grad(): + target_ctrx = (target[:, 0] + target[:, 2]) * 0.5 + target_ctry = (target[:, 1] + target[:, 3]) * 0.5 + target_w = target[:, 2] - target[:, 0] + target_h = target[:, 3] - target[:, 1] + + dx = target_ctrx - pred_ctrx + dy = target_ctry - pred_ctry + + loss_dx = 1 - torch.max( + (target_w - 2 * dx.abs()) / + (target_w + 2 * dx.abs() + eps), torch.zeros_like(dx)) + loss_dy = 1 - torch.max( + (target_h - 2 * dy.abs()) / + (target_h + 2 * dy.abs() + eps), torch.zeros_like(dy)) + loss_dw = 1 - torch.min(target_w / (pred_w + eps), pred_w / + (target_w + eps)) + loss_dh = 1 - torch.min(target_h / (pred_h + eps), pred_h / + (target_h + eps)) + loss_comb = torch.stack([loss_dx, loss_dy, loss_dw, loss_dh], + dim=-1).view(loss_dx.size(0), -1) + + loss = torch.where(loss_comb < beta, 0.5 * loss_comb * loss_comb / beta, + loss_comb - 0.5 * beta) + return loss + + +@mmcv.jit(derivate=True, coderize=True) +@weighted_loss +def giou_loss(pred, target, eps=1e-7): + r"""`Generalized Intersection over Union: A Metric and A Loss for Bounding + Box Regression `_. + + Args: + pred (torch.Tensor): Predicted bboxes of format (x1, y1, x2, y2), + shape (n, 4). + target (torch.Tensor): Corresponding gt bboxes, shape (n, 4). + eps (float): Eps to avoid log(0). + + Return: + Tensor: Loss tensor. + """ + gious = bbox_overlaps(pred, target, mode='giou', is_aligned=True, eps=eps) + loss = 1 - gious + return loss + + +@mmcv.jit(derivate=True, coderize=True) +@weighted_loss +def diou_loss(pred, target, eps=1e-7): + r"""`Implementation of Distance-IoU Loss: Faster and Better + Learning for Bounding Box Regression, https://arxiv.org/abs/1911.08287`_. + + Code is modified from https://github.com/Zzh-tju/DIoU. + + Args: + pred (Tensor): Predicted bboxes of format (x1, y1, x2, y2), + shape (n, 4). + target (Tensor): Corresponding gt bboxes, shape (n, 4). + eps (float): Eps to avoid log(0). + Return: + Tensor: Loss tensor. + """ + # overlap + lt = torch.max(pred[:, :2], target[:, :2]) + rb = torch.min(pred[:, 2:], target[:, 2:]) + wh = (rb - lt).clamp(min=0) + overlap = wh[:, 0] * wh[:, 1] + + # union + ap = (pred[:, 2] - pred[:, 0]) * (pred[:, 3] - pred[:, 1]) + ag = (target[:, 2] - target[:, 0]) * (target[:, 3] - target[:, 1]) + union = ap + ag - overlap + eps + + # IoU + ious = overlap / union + + # enclose area + enclose_x1y1 = torch.min(pred[:, :2], target[:, :2]) + enclose_x2y2 = torch.max(pred[:, 2:], target[:, 2:]) + enclose_wh = (enclose_x2y2 - enclose_x1y1).clamp(min=0) + + cw = enclose_wh[:, 0] + ch = enclose_wh[:, 1] + + c2 = cw**2 + ch**2 + eps + + b1_x1, b1_y1 = pred[:, 0], pred[:, 1] + b1_x2, b1_y2 = pred[:, 2], pred[:, 3] + b2_x1, b2_y1 = target[:, 0], target[:, 1] + b2_x2, b2_y2 = target[:, 2], target[:, 3] + + left = ((b2_x1 + b2_x2) - (b1_x1 + b1_x2))**2 / 4 + right = ((b2_y1 + b2_y2) - (b1_y1 + b1_y2))**2 / 4 + rho2 = left + right + + # DIoU + dious = ious - rho2 / c2 + loss = 1 - dious + return loss + + +@mmcv.jit(derivate=True, coderize=True) +@weighted_loss +def ciou_loss(pred, target, eps=1e-7): + r"""`Implementation of paper `Enhancing Geometric Factors into + Model Learning and Inference for Object Detection and Instance + Segmentation `_. + + Code is modified from https://github.com/Zzh-tju/CIoU. + + Args: + pred (Tensor): Predicted bboxes of format (x1, y1, x2, y2), + shape (n, 4). + target (Tensor): Corresponding gt bboxes, shape (n, 4). + eps (float): Eps to avoid log(0). + Return: + Tensor: Loss tensor. + """ + # overlap + lt = torch.max(pred[:, :2], target[:, :2]) + rb = torch.min(pred[:, 2:], target[:, 2:]) + wh = (rb - lt).clamp(min=0) + overlap = wh[:, 0] * wh[:, 1] + + # union + ap = (pred[:, 2] - pred[:, 0]) * (pred[:, 3] - pred[:, 1]) + ag = (target[:, 2] - target[:, 0]) * (target[:, 3] - target[:, 1]) + union = ap + ag - overlap + eps + + # IoU + ious = overlap / union + + # enclose area + enclose_x1y1 = torch.min(pred[:, :2], target[:, :2]) + enclose_x2y2 = torch.max(pred[:, 2:], target[:, 2:]) + enclose_wh = (enclose_x2y2 - enclose_x1y1).clamp(min=0) + + cw = enclose_wh[:, 0] + ch = enclose_wh[:, 1] + + c2 = cw**2 + ch**2 + eps + + b1_x1, b1_y1 = pred[:, 0], pred[:, 1] + b1_x2, b1_y2 = pred[:, 2], pred[:, 3] + b2_x1, b2_y1 = target[:, 0], target[:, 1] + b2_x2, b2_y2 = target[:, 2], target[:, 3] + + w1, h1 = b1_x2 - b1_x1, b1_y2 - b1_y1 + eps + w2, h2 = b2_x2 - b2_x1, b2_y2 - b2_y1 + eps + + left = ((b2_x1 + b2_x2) - (b1_x1 + b1_x2))**2 / 4 + right = ((b2_y1 + b2_y2) - (b1_y1 + b1_y2))**2 / 4 + rho2 = left + right + + factor = 4 / math.pi**2 + v = factor * torch.pow(torch.atan(w2 / h2) - torch.atan(w1 / h1), 2) + + # CIoU + cious = ious - (rho2 / c2 + v**2 / (1 - ious + v)) + loss = 1 - cious + return loss + + +@LOSSES.register_module() +class IoULoss(nn.Module): + """IoULoss. + + Computing the IoU loss between a set of predicted bboxes and target bboxes. + + Args: + linear (bool): If True, use linear scale of loss instead of log scale. + Default: False. + eps (float): Eps to avoid log(0). + reduction (str): Options are "none", "mean" and "sum". + loss_weight (float): Weight of loss. + """ + + def __init__(self, + linear=False, + eps=1e-6, + reduction='mean', + loss_weight=1.0): + super(IoULoss, self).__init__() + self.linear = linear + self.eps = eps + self.reduction = reduction + self.loss_weight = loss_weight + + def forward(self, + pred, + target, + weight=None, + avg_factor=None, + reduction_override=None, + **kwargs): + """Forward function. + + Args: + pred (torch.Tensor): The prediction. + target (torch.Tensor): The learning target of the prediction. + weight (torch.Tensor, optional): The weight of loss for each + prediction. Defaults to None. + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + reduction_override (str, optional): The reduction method used to + override the original reduction method of the loss. + Defaults to None. Options are "none", "mean" and "sum". + """ + assert reduction_override in (None, 'none', 'mean', 'sum') + reduction = ( + reduction_override if reduction_override else self.reduction) + if (weight is not None) and (not torch.any(weight > 0)) and ( + reduction != 'none'): + return (pred * weight).sum() # 0 + if weight is not None and weight.dim() > 1: + # TODO: remove this in the future + # reduce the weight of shape (n, 4) to (n,) to match the + # iou_loss of shape (n,) + assert weight.shape == pred.shape + weight = weight.mean(-1) + loss = self.loss_weight * iou_loss( + pred, + target, + weight, + linear=self.linear, + eps=self.eps, + reduction=reduction, + avg_factor=avg_factor, + **kwargs) + return loss + + +@LOSSES.register_module() +class BoundedIoULoss(nn.Module): + + def __init__(self, beta=0.2, eps=1e-3, reduction='mean', loss_weight=1.0): + super(BoundedIoULoss, self).__init__() + self.beta = beta + self.eps = eps + self.reduction = reduction + self.loss_weight = loss_weight + + def forward(self, + pred, + target, + weight=None, + avg_factor=None, + reduction_override=None, + **kwargs): + if weight is not None and not torch.any(weight > 0): + return (pred * weight).sum() # 0 + assert reduction_override in (None, 'none', 'mean', 'sum') + reduction = ( + reduction_override if reduction_override else self.reduction) + loss = self.loss_weight * bounded_iou_loss( + pred, + target, + weight, + beta=self.beta, + eps=self.eps, + reduction=reduction, + avg_factor=avg_factor, + **kwargs) + return loss + + +@LOSSES.register_module() +class GIoULoss(nn.Module): + + def __init__(self, eps=1e-6, reduction='mean', loss_weight=1.0): + super(GIoULoss, self).__init__() + self.eps = eps + self.reduction = reduction + self.loss_weight = loss_weight + + def forward(self, + pred, + target, + weight=None, + avg_factor=None, + reduction_override=None, + **kwargs): + if weight is not None and not torch.any(weight > 0): + return (pred * weight).sum() # 0 + assert reduction_override in (None, 'none', 'mean', 'sum') + reduction = ( + reduction_override if reduction_override else self.reduction) + if weight is not None and weight.dim() > 1: + # TODO: remove this in the future + # reduce the weight of shape (n, 4) to (n,) to match the + # giou_loss of shape (n,) + assert weight.shape == pred.shape + weight = weight.mean(-1) + loss = self.loss_weight * giou_loss( + pred, + target, + weight, + eps=self.eps, + reduction=reduction, + avg_factor=avg_factor, + **kwargs) + return loss + + +@LOSSES.register_module() +class DIoULoss(nn.Module): + + def __init__(self, eps=1e-6, reduction='mean', loss_weight=1.0): + super(DIoULoss, self).__init__() + self.eps = eps + self.reduction = reduction + self.loss_weight = loss_weight + + def forward(self, + pred, + target, + weight=None, + avg_factor=None, + reduction_override=None, + **kwargs): + if weight is not None and not torch.any(weight > 0): + return (pred * weight).sum() # 0 + assert reduction_override in (None, 'none', 'mean', 'sum') + reduction = ( + reduction_override if reduction_override else self.reduction) + if weight is not None and weight.dim() > 1: + # TODO: remove this in the future + # reduce the weight of shape (n, 4) to (n,) to match the + # giou_loss of shape (n,) + assert weight.shape == pred.shape + weight = weight.mean(-1) + loss = self.loss_weight * diou_loss( + pred, + target, + weight, + eps=self.eps, + reduction=reduction, + avg_factor=avg_factor, + **kwargs) + return loss + + +@LOSSES.register_module() +class CIoULoss(nn.Module): + + def __init__(self, eps=1e-6, reduction='mean', loss_weight=1.0): + super(CIoULoss, self).__init__() + self.eps = eps + self.reduction = reduction + self.loss_weight = loss_weight + + def forward(self, + pred, + target, + weight=None, + avg_factor=None, + reduction_override=None, + **kwargs): + if weight is not None and not torch.any(weight > 0): + return (pred * weight).sum() # 0 + assert reduction_override in (None, 'none', 'mean', 'sum') + reduction = ( + reduction_override if reduction_override else self.reduction) + if weight is not None and weight.dim() > 1: + # TODO: remove this in the future + # reduce the weight of shape (n, 4) to (n,) to match the + # giou_loss of shape (n,) + assert weight.shape == pred.shape + weight = weight.mean(-1) + loss = self.loss_weight * ciou_loss( + pred, + target, + weight, + eps=self.eps, + reduction=reduction, + avg_factor=avg_factor, + **kwargs) + return loss diff --git a/detection/mmdet/models/losses/kd_loss.py b/detection/mmdet/models/losses/kd_loss.py new file mode 100644 index 0000000..f3abb68 --- /dev/null +++ b/detection/mmdet/models/losses/kd_loss.py @@ -0,0 +1,87 @@ +import mmcv +import torch.nn as nn +import torch.nn.functional as F + +from ..builder import LOSSES +from .utils import weighted_loss + + +@mmcv.jit(derivate=True, coderize=True) +@weighted_loss +def knowledge_distillation_kl_div_loss(pred, + soft_label, + T, + detach_target=True): + r"""Loss function for knowledge distilling using KL divergence. + + Args: + pred (Tensor): Predicted logits with shape (N, n + 1). + soft_label (Tensor): Target logits with shape (N, N + 1). + T (int): Temperature for distillation. + detach_target (bool): Remove soft_label from automatic differentiation + + Returns: + torch.Tensor: Loss tensor with shape (N,). + """ + assert pred.size() == soft_label.size() + target = F.softmax(soft_label / T, dim=1) + if detach_target: + target = target.detach() + + kd_loss = F.kl_div( + F.log_softmax(pred / T, dim=1), target, reduction='none').mean(1) * ( + T * T) + + return kd_loss + + +@LOSSES.register_module() +class KnowledgeDistillationKLDivLoss(nn.Module): + """Loss function for knowledge distilling using KL divergence. + + Args: + reduction (str): Options are `'none'`, `'mean'` and `'sum'`. + loss_weight (float): Loss weight of current loss. + T (int): Temperature for distillation. + """ + + def __init__(self, reduction='mean', loss_weight=1.0, T=10): + super(KnowledgeDistillationKLDivLoss, self).__init__() + assert T >= 1 + self.reduction = reduction + self.loss_weight = loss_weight + self.T = T + + def forward(self, + pred, + soft_label, + weight=None, + avg_factor=None, + reduction_override=None): + """Forward function. + + Args: + pred (Tensor): Predicted logits with shape (N, n + 1). + soft_label (Tensor): Target logits with shape (N, N + 1). + weight (torch.Tensor, optional): The weight of loss for each + prediction. Defaults to None. + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + reduction_override (str, optional): The reduction method used to + override the original reduction method of the loss. + Defaults to None. + """ + assert reduction_override in (None, 'none', 'mean', 'sum') + + reduction = ( + reduction_override if reduction_override else self.reduction) + + loss_kd = self.loss_weight * knowledge_distillation_kl_div_loss( + pred, + soft_label, + weight, + reduction=reduction, + avg_factor=avg_factor, + T=self.T) + + return loss_kd diff --git a/detection/mmdet/models/losses/mse_loss.py b/detection/mmdet/models/losses/mse_loss.py new file mode 100644 index 0000000..68d0575 --- /dev/null +++ b/detection/mmdet/models/losses/mse_loss.py @@ -0,0 +1,49 @@ +import torch.nn as nn +import torch.nn.functional as F + +from ..builder import LOSSES +from .utils import weighted_loss + + +@weighted_loss +def mse_loss(pred, target): + """Warpper of mse loss.""" + return F.mse_loss(pred, target, reduction='none') + + +@LOSSES.register_module() +class MSELoss(nn.Module): + """MSELoss. + + Args: + reduction (str, optional): The method that reduces the loss to a + scalar. Options are "none", "mean" and "sum". + loss_weight (float, optional): The weight of the loss. Defaults to 1.0 + """ + + def __init__(self, reduction='mean', loss_weight=1.0): + super().__init__() + self.reduction = reduction + self.loss_weight = loss_weight + + def forward(self, pred, target, weight=None, avg_factor=None): + """Forward function of loss. + + Args: + pred (torch.Tensor): The prediction. + target (torch.Tensor): The learning target of the prediction. + weight (torch.Tensor, optional): Weight of the loss for each + prediction. Defaults to None. + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + + Returns: + torch.Tensor: The calculated loss + """ + loss = self.loss_weight * mse_loss( + pred, + target, + weight, + reduction=self.reduction, + avg_factor=avg_factor) + return loss diff --git a/detection/mmdet/models/losses/pisa_loss.py b/detection/mmdet/models/losses/pisa_loss.py new file mode 100644 index 0000000..4a48adf --- /dev/null +++ b/detection/mmdet/models/losses/pisa_loss.py @@ -0,0 +1,183 @@ +import mmcv +import torch + +from mmdet.core import bbox_overlaps + + +@mmcv.jit(derivate=True, coderize=True) +def isr_p(cls_score, + bbox_pred, + bbox_targets, + rois, + sampling_results, + loss_cls, + bbox_coder, + k=2, + bias=0, + num_class=80): + """Importance-based Sample Reweighting (ISR_P), positive part. + + Args: + cls_score (Tensor): Predicted classification scores. + bbox_pred (Tensor): Predicted bbox deltas. + bbox_targets (tuple[Tensor]): A tuple of bbox targets, the are + labels, label_weights, bbox_targets, bbox_weights, respectively. + rois (Tensor): Anchors (single_stage) in shape (n, 4) or RoIs + (two_stage) in shape (n, 5). + sampling_results (obj): Sampling results. + loss_cls (func): Classification loss func of the head. + bbox_coder (obj): BBox coder of the head. + k (float): Power of the non-linear mapping. + bias (float): Shift of the non-linear mapping. + num_class (int): Number of classes, default: 80. + + Return: + tuple([Tensor]): labels, imp_based_label_weights, bbox_targets, + bbox_target_weights + """ + + labels, label_weights, bbox_targets, bbox_weights = bbox_targets + pos_label_inds = ((labels >= 0) & + (labels < num_class)).nonzero().reshape(-1) + pos_labels = labels[pos_label_inds] + + # if no positive samples, return the original targets + num_pos = float(pos_label_inds.size(0)) + if num_pos == 0: + return labels, label_weights, bbox_targets, bbox_weights + + # merge pos_assigned_gt_inds of per image to a single tensor + gts = list() + last_max_gt = 0 + for i in range(len(sampling_results)): + gt_i = sampling_results[i].pos_assigned_gt_inds + gts.append(gt_i + last_max_gt) + if len(gt_i) != 0: + last_max_gt = gt_i.max() + 1 + gts = torch.cat(gts) + assert len(gts) == num_pos + + cls_score = cls_score.detach() + bbox_pred = bbox_pred.detach() + + # For single stage detectors, rois here indicate anchors, in shape (N, 4) + # For two stage detectors, rois are in shape (N, 5) + if rois.size(-1) == 5: + pos_rois = rois[pos_label_inds][:, 1:] + else: + pos_rois = rois[pos_label_inds] + + if bbox_pred.size(-1) > 4: + bbox_pred = bbox_pred.view(bbox_pred.size(0), -1, 4) + pos_delta_pred = bbox_pred[pos_label_inds, pos_labels].view(-1, 4) + else: + pos_delta_pred = bbox_pred[pos_label_inds].view(-1, 4) + + # compute iou of the predicted bbox and the corresponding GT + pos_delta_target = bbox_targets[pos_label_inds].view(-1, 4) + pos_bbox_pred = bbox_coder.decode(pos_rois, pos_delta_pred) + target_bbox_pred = bbox_coder.decode(pos_rois, pos_delta_target) + ious = bbox_overlaps(pos_bbox_pred, target_bbox_pred, is_aligned=True) + + pos_imp_weights = label_weights[pos_label_inds] + # Two steps to compute IoU-HLR. Samples are first sorted by IoU locally, + # then sorted again within the same-rank group + max_l_num = pos_labels.bincount().max() + for label in pos_labels.unique(): + l_inds = (pos_labels == label).nonzero().view(-1) + l_gts = gts[l_inds] + for t in l_gts.unique(): + t_inds = l_inds[l_gts == t] + t_ious = ious[t_inds] + _, t_iou_rank_idx = t_ious.sort(descending=True) + _, t_iou_rank = t_iou_rank_idx.sort() + ious[t_inds] += max_l_num - t_iou_rank.float() + l_ious = ious[l_inds] + _, l_iou_rank_idx = l_ious.sort(descending=True) + _, l_iou_rank = l_iou_rank_idx.sort() # IoU-HLR + # linearly map HLR to label weights + pos_imp_weights[l_inds] *= (max_l_num - l_iou_rank.float()) / max_l_num + + pos_imp_weights = (bias + pos_imp_weights * (1 - bias)).pow(k) + + # normalize to make the new weighted loss value equal to the original loss + pos_loss_cls = loss_cls( + cls_score[pos_label_inds], pos_labels, reduction_override='none') + if pos_loss_cls.dim() > 1: + ori_pos_loss_cls = pos_loss_cls * label_weights[pos_label_inds][:, + None] + new_pos_loss_cls = pos_loss_cls * pos_imp_weights[:, None] + else: + ori_pos_loss_cls = pos_loss_cls * label_weights[pos_label_inds] + new_pos_loss_cls = pos_loss_cls * pos_imp_weights + pos_loss_cls_ratio = ori_pos_loss_cls.sum() / new_pos_loss_cls.sum() + pos_imp_weights = pos_imp_weights * pos_loss_cls_ratio + label_weights[pos_label_inds] = pos_imp_weights + + bbox_targets = labels, label_weights, bbox_targets, bbox_weights + return bbox_targets + + +@mmcv.jit(derivate=True, coderize=True) +def carl_loss(cls_score, + labels, + bbox_pred, + bbox_targets, + loss_bbox, + k=1, + bias=0.2, + avg_factor=None, + sigmoid=False, + num_class=80): + """Classification-Aware Regression Loss (CARL). + + Args: + cls_score (Tensor): Predicted classification scores. + labels (Tensor): Targets of classification. + bbox_pred (Tensor): Predicted bbox deltas. + bbox_targets (Tensor): Target of bbox regression. + loss_bbox (func): Regression loss func of the head. + bbox_coder (obj): BBox coder of the head. + k (float): Power of the non-linear mapping. + bias (float): Shift of the non-linear mapping. + avg_factor (int): Average factor used in regression loss. + sigmoid (bool): Activation of the classification score. + num_class (int): Number of classes, default: 80. + + Return: + dict: CARL loss dict. + """ + pos_label_inds = ((labels >= 0) & + (labels < num_class)).nonzero().reshape(-1) + if pos_label_inds.numel() == 0: + return dict(loss_carl=cls_score.sum()[None] * 0.) + pos_labels = labels[pos_label_inds] + + # multiply pos_cls_score with the corresponding bbox weight + # and remain gradient + if sigmoid: + pos_cls_score = cls_score.sigmoid()[pos_label_inds, pos_labels] + else: + pos_cls_score = cls_score.softmax(-1)[pos_label_inds, pos_labels] + carl_loss_weights = (bias + (1 - bias) * pos_cls_score).pow(k) + + # normalize carl_loss_weight to make its sum equal to num positive + num_pos = float(pos_cls_score.size(0)) + weight_ratio = num_pos / carl_loss_weights.sum() + carl_loss_weights *= weight_ratio + + if avg_factor is None: + avg_factor = bbox_targets.size(0) + # if is class agnostic, bbox pred is in shape (N, 4) + # otherwise, bbox pred is in shape (N, #classes, 4) + if bbox_pred.size(-1) > 4: + bbox_pred = bbox_pred.view(bbox_pred.size(0), -1, 4) + pos_bbox_preds = bbox_pred[pos_label_inds, pos_labels] + else: + pos_bbox_preds = bbox_pred[pos_label_inds] + ori_loss_reg = loss_bbox( + pos_bbox_preds, + bbox_targets[pos_label_inds], + reduction_override='none') / avg_factor + loss_carl = (ori_loss_reg * carl_loss_weights[:, None]).sum() + return dict(loss_carl=loss_carl[None]) diff --git a/detection/mmdet/models/losses/smooth_l1_loss.py b/detection/mmdet/models/losses/smooth_l1_loss.py new file mode 100644 index 0000000..ec9c98a --- /dev/null +++ b/detection/mmdet/models/losses/smooth_l1_loss.py @@ -0,0 +1,139 @@ +import mmcv +import torch +import torch.nn as nn + +from ..builder import LOSSES +from .utils import weighted_loss + + +@mmcv.jit(derivate=True, coderize=True) +@weighted_loss +def smooth_l1_loss(pred, target, beta=1.0): + """Smooth L1 loss. + + Args: + pred (torch.Tensor): The prediction. + target (torch.Tensor): The learning target of the prediction. + beta (float, optional): The threshold in the piecewise function. + Defaults to 1.0. + + Returns: + torch.Tensor: Calculated loss + """ + assert beta > 0 + assert pred.size() == target.size() and target.numel() > 0 + diff = torch.abs(pred - target) + loss = torch.where(diff < beta, 0.5 * diff * diff / beta, + diff - 0.5 * beta) + return loss + + +@mmcv.jit(derivate=True, coderize=True) +@weighted_loss +def l1_loss(pred, target): + """L1 loss. + + Args: + pred (torch.Tensor): The prediction. + target (torch.Tensor): The learning target of the prediction. + + Returns: + torch.Tensor: Calculated loss + """ + assert pred.size() == target.size() and target.numel() > 0 + loss = torch.abs(pred - target) + return loss + + +@LOSSES.register_module() +class SmoothL1Loss(nn.Module): + """Smooth L1 loss. + + Args: + beta (float, optional): The threshold in the piecewise function. + Defaults to 1.0. + reduction (str, optional): The method to reduce the loss. + Options are "none", "mean" and "sum". Defaults to "mean". + loss_weight (float, optional): The weight of loss. + """ + + def __init__(self, beta=1.0, reduction='mean', loss_weight=1.0): + super(SmoothL1Loss, self).__init__() + self.beta = beta + self.reduction = reduction + self.loss_weight = loss_weight + + def forward(self, + pred, + target, + weight=None, + avg_factor=None, + reduction_override=None, + **kwargs): + """Forward function. + + Args: + pred (torch.Tensor): The prediction. + target (torch.Tensor): The learning target of the prediction. + weight (torch.Tensor, optional): The weight of loss for each + prediction. Defaults to None. + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + reduction_override (str, optional): The reduction method used to + override the original reduction method of the loss. + Defaults to None. + """ + assert reduction_override in (None, 'none', 'mean', 'sum') + reduction = ( + reduction_override if reduction_override else self.reduction) + loss_bbox = self.loss_weight * smooth_l1_loss( + pred, + target, + weight, + beta=self.beta, + reduction=reduction, + avg_factor=avg_factor, + **kwargs) + return loss_bbox + + +@LOSSES.register_module() +class L1Loss(nn.Module): + """L1 loss. + + Args: + reduction (str, optional): The method to reduce the loss. + Options are "none", "mean" and "sum". + loss_weight (float, optional): The weight of loss. + """ + + def __init__(self, reduction='mean', loss_weight=1.0): + super(L1Loss, self).__init__() + self.reduction = reduction + self.loss_weight = loss_weight + + def forward(self, + pred, + target, + weight=None, + avg_factor=None, + reduction_override=None): + """Forward function. + + Args: + pred (torch.Tensor): The prediction. + target (torch.Tensor): The learning target of the prediction. + weight (torch.Tensor, optional): The weight of loss for each + prediction. Defaults to None. + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + reduction_override (str, optional): The reduction method used to + override the original reduction method of the loss. + Defaults to None. + """ + assert reduction_override in (None, 'none', 'mean', 'sum') + reduction = ( + reduction_override if reduction_override else self.reduction) + loss_bbox = self.loss_weight * l1_loss( + pred, target, weight, reduction=reduction, avg_factor=avg_factor) + return loss_bbox diff --git a/detection/mmdet/models/losses/utils.py b/detection/mmdet/models/losses/utils.py new file mode 100644 index 0000000..4756d7f --- /dev/null +++ b/detection/mmdet/models/losses/utils.py @@ -0,0 +1,100 @@ +import functools + +import mmcv +import torch.nn.functional as F + + +def reduce_loss(loss, reduction): + """Reduce loss as specified. + + Args: + loss (Tensor): Elementwise loss tensor. + reduction (str): Options are "none", "mean" and "sum". + + Return: + Tensor: Reduced loss tensor. + """ + reduction_enum = F._Reduction.get_enum(reduction) + # none: 0, elementwise_mean:1, sum: 2 + if reduction_enum == 0: + return loss + elif reduction_enum == 1: + return loss.mean() + elif reduction_enum == 2: + return loss.sum() + + +@mmcv.jit(derivate=True, coderize=True) +def weight_reduce_loss(loss, weight=None, reduction='mean', avg_factor=None): + """Apply element-wise weight and reduce loss. + + Args: + loss (Tensor): Element-wise loss. + weight (Tensor): Element-wise weights. + reduction (str): Same as built-in losses of PyTorch. + avg_factor (float): Avarage factor when computing the mean of losses. + + Returns: + Tensor: Processed loss values. + """ + # if weight is specified, apply element-wise weight + if weight is not None: + loss = loss * weight + + # if avg_factor is not specified, just reduce the loss + if avg_factor is None: + loss = reduce_loss(loss, reduction) + else: + # if reduction is mean, then average the loss by avg_factor + if reduction == 'mean': + loss = loss.sum() / avg_factor + # if reduction is 'none', then do nothing, otherwise raise an error + elif reduction != 'none': + raise ValueError('avg_factor can not be used with reduction="sum"') + return loss + + +def weighted_loss(loss_func): + """Create a weighted version of a given loss function. + + To use this decorator, the loss function must have the signature like + `loss_func(pred, target, **kwargs)`. The function only needs to compute + element-wise loss without any reduction. This decorator will add weight + and reduction arguments to the function. The decorated function will have + the signature like `loss_func(pred, target, weight=None, reduction='mean', + avg_factor=None, **kwargs)`. + + :Example: + + >>> import torch + >>> @weighted_loss + >>> def l1_loss(pred, target): + >>> return (pred - target).abs() + + >>> pred = torch.Tensor([0, 2, 3]) + >>> target = torch.Tensor([1, 1, 1]) + >>> weight = torch.Tensor([1, 0, 1]) + + >>> l1_loss(pred, target) + tensor(1.3333) + >>> l1_loss(pred, target, weight) + tensor(1.) + >>> l1_loss(pred, target, reduction='none') + tensor([1., 1., 2.]) + >>> l1_loss(pred, target, weight, avg_factor=2) + tensor(1.5000) + """ + + @functools.wraps(loss_func) + def wrapper(pred, + target, + weight=None, + reduction='mean', + avg_factor=None, + **kwargs): + # get element-wise loss + loss = loss_func(pred, target, **kwargs) + loss = weight_reduce_loss(loss, weight, reduction, avg_factor) + return loss + + return wrapper diff --git a/detection/mmdet/models/losses/varifocal_loss.py b/detection/mmdet/models/losses/varifocal_loss.py new file mode 100644 index 0000000..7f00bd6 --- /dev/null +++ b/detection/mmdet/models/losses/varifocal_loss.py @@ -0,0 +1,133 @@ +import mmcv +import torch.nn as nn +import torch.nn.functional as F + +from ..builder import LOSSES +from .utils import weight_reduce_loss + + +@mmcv.jit(derivate=True, coderize=True) +def varifocal_loss(pred, + target, + weight=None, + alpha=0.75, + gamma=2.0, + iou_weighted=True, + reduction='mean', + avg_factor=None): + """`Varifocal Loss `_ + + Args: + pred (torch.Tensor): The prediction with shape (N, C), C is the + number of classes + target (torch.Tensor): The learning target of the iou-aware + classification score with shape (N, C), C is the number of classes. + weight (torch.Tensor, optional): The weight of loss for each + prediction. Defaults to None. + alpha (float, optional): A balance factor for the negative part of + Varifocal Loss, which is different from the alpha of Focal Loss. + Defaults to 0.75. + gamma (float, optional): The gamma for calculating the modulating + factor. Defaults to 2.0. + iou_weighted (bool, optional): Whether to weight the loss of the + positive example with the iou target. Defaults to True. + reduction (str, optional): The method used to reduce the loss into + a scalar. Defaults to 'mean'. Options are "none", "mean" and + "sum". + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + """ + # pred and target should be of the same size + assert pred.size() == target.size() + pred_sigmoid = pred.sigmoid() + target = target.type_as(pred) + if iou_weighted: + focal_weight = target * (target > 0.0).float() + \ + alpha * (pred_sigmoid - target).abs().pow(gamma) * \ + (target <= 0.0).float() + else: + focal_weight = (target > 0.0).float() + \ + alpha * (pred_sigmoid - target).abs().pow(gamma) * \ + (target <= 0.0).float() + loss = F.binary_cross_entropy_with_logits( + pred, target, reduction='none') * focal_weight + loss = weight_reduce_loss(loss, weight, reduction, avg_factor) + return loss + + +@LOSSES.register_module() +class VarifocalLoss(nn.Module): + + def __init__(self, + use_sigmoid=True, + alpha=0.75, + gamma=2.0, + iou_weighted=True, + reduction='mean', + loss_weight=1.0): + """`Varifocal Loss `_ + + Args: + use_sigmoid (bool, optional): Whether the prediction is + used for sigmoid or softmax. Defaults to True. + alpha (float, optional): A balance factor for the negative part of + Varifocal Loss, which is different from the alpha of Focal + Loss. Defaults to 0.75. + gamma (float, optional): The gamma for calculating the modulating + factor. Defaults to 2.0. + iou_weighted (bool, optional): Whether to weight the loss of the + positive examples with the iou target. Defaults to True. + reduction (str, optional): The method used to reduce the loss into + a scalar. Defaults to 'mean'. Options are "none", "mean" and + "sum". + loss_weight (float, optional): Weight of loss. Defaults to 1.0. + """ + super(VarifocalLoss, self).__init__() + assert use_sigmoid is True, \ + 'Only sigmoid varifocal loss supported now.' + assert alpha >= 0.0 + self.use_sigmoid = use_sigmoid + self.alpha = alpha + self.gamma = gamma + self.iou_weighted = iou_weighted + self.reduction = reduction + self.loss_weight = loss_weight + + def forward(self, + pred, + target, + weight=None, + avg_factor=None, + reduction_override=None): + """Forward function. + + Args: + pred (torch.Tensor): The prediction. + target (torch.Tensor): The learning target of the prediction. + weight (torch.Tensor, optional): The weight of loss for each + prediction. Defaults to None. + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + reduction_override (str, optional): The reduction method used to + override the original reduction method of the loss. + Options are "none", "mean" and "sum". + + Returns: + torch.Tensor: The calculated loss + """ + assert reduction_override in (None, 'none', 'mean', 'sum') + reduction = ( + reduction_override if reduction_override else self.reduction) + if self.use_sigmoid: + loss_cls = self.loss_weight * varifocal_loss( + pred, + target, + weight, + alpha=self.alpha, + gamma=self.gamma, + iou_weighted=self.iou_weighted, + reduction=reduction, + avg_factor=avg_factor) + else: + raise NotImplementedError + return loss_cls diff --git a/detection/mmdet/models/necks/__init__.py b/detection/mmdet/models/necks/__init__.py new file mode 100644 index 0000000..02f833a --- /dev/null +++ b/detection/mmdet/models/necks/__init__.py @@ -0,0 +1,16 @@ +from .bfp import BFP +from .channel_mapper import ChannelMapper +from .fpg import FPG +from .fpn import FPN +from .fpn_carafe import FPN_CARAFE +from .hrfpn import HRFPN +from .nas_fpn import NASFPN +from .nasfcos_fpn import NASFCOS_FPN +from .pafpn import PAFPN +from .rfp import RFP +from .yolo_neck import YOLOV3Neck + +__all__ = [ + 'FPN', 'BFP', 'ChannelMapper', 'HRFPN', 'NASFPN', 'FPN_CARAFE', 'PAFPN', + 'NASFCOS_FPN', 'RFP', 'YOLOV3Neck', 'FPG' +] diff --git a/detection/mmdet/models/necks/bfp.py b/detection/mmdet/models/necks/bfp.py new file mode 100644 index 0000000..123f551 --- /dev/null +++ b/detection/mmdet/models/necks/bfp.py @@ -0,0 +1,104 @@ +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import ConvModule, xavier_init +from mmcv.cnn.bricks import NonLocal2d + +from ..builder import NECKS + + +@NECKS.register_module() +class BFP(nn.Module): + """BFP (Balanced Feature Pyramids) + + BFP takes multi-level features as inputs and gather them into a single one, + then refine the gathered feature and scatter the refined results to + multi-level features. This module is used in Libra R-CNN (CVPR 2019), see + the paper `Libra R-CNN: Towards Balanced Learning for Object Detection + `_ for details. + + Args: + in_channels (int): Number of input channels (feature maps of all levels + should have the same channels). + num_levels (int): Number of input feature levels. + conv_cfg (dict): The config dict for convolution layers. + norm_cfg (dict): The config dict for normalization layers. + refine_level (int): Index of integration and refine level of BSF in + multi-level features from bottom to top. + refine_type (str): Type of the refine op, currently support + [None, 'conv', 'non_local']. + """ + + def __init__(self, + in_channels, + num_levels, + refine_level=2, + refine_type=None, + conv_cfg=None, + norm_cfg=None): + super(BFP, self).__init__() + assert refine_type in [None, 'conv', 'non_local'] + + self.in_channels = in_channels + self.num_levels = num_levels + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + + self.refine_level = refine_level + self.refine_type = refine_type + assert 0 <= self.refine_level < self.num_levels + + if self.refine_type == 'conv': + self.refine = ConvModule( + self.in_channels, + self.in_channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg) + elif self.refine_type == 'non_local': + self.refine = NonLocal2d( + self.in_channels, + reduction=1, + use_scale=False, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg) + + def init_weights(self): + """Initialize the weights of FPN module.""" + for m in self.modules(): + if isinstance(m, nn.Conv2d): + xavier_init(m, distribution='uniform') + + def forward(self, inputs): + """Forward function.""" + assert len(inputs) == self.num_levels + + # step 1: gather multi-level features by resize and average + feats = [] + gather_size = inputs[self.refine_level].size()[2:] + for i in range(self.num_levels): + if i < self.refine_level: + gathered = F.adaptive_max_pool2d( + inputs[i], output_size=gather_size) + else: + gathered = F.interpolate( + inputs[i], size=gather_size, mode='nearest') + feats.append(gathered) + + bsf = sum(feats) / len(feats) + + # step 2: refine gathered features + if self.refine_type is not None: + bsf = self.refine(bsf) + + # step 3: scatter refined features to multi-levels by a residual path + outs = [] + for i in range(self.num_levels): + out_size = inputs[i].size()[2:] + if i < self.refine_level: + residual = F.interpolate(bsf, size=out_size, mode='nearest') + else: + residual = F.adaptive_max_pool2d(bsf, output_size=out_size) + outs.append(residual + inputs[i]) + + return tuple(outs) diff --git a/detection/mmdet/models/necks/channel_mapper.py b/detection/mmdet/models/necks/channel_mapper.py new file mode 100644 index 0000000..a4f5ed4 --- /dev/null +++ b/detection/mmdet/models/necks/channel_mapper.py @@ -0,0 +1,74 @@ +import torch.nn as nn +from mmcv.cnn import ConvModule, xavier_init + +from ..builder import NECKS + + +@NECKS.register_module() +class ChannelMapper(nn.Module): + r"""Channel Mapper to reduce/increase channels of backbone features. + + This is used to reduce/increase channels of backbone features. + + Args: + in_channels (List[int]): Number of input channels per scale. + out_channels (int): Number of output channels (used at each scale). + kernel_size (int, optional): kernel_size for reducing channels (used + at each scale). Default: 3. + conv_cfg (dict, optional): Config dict for convolution layer. + Default: None. + norm_cfg (dict, optional): Config dict for normalization layer. + Default: None. + act_cfg (dict, optional): Config dict for activation layer in + ConvModule. Default: dict(type='ReLU'). + + Example: + >>> import torch + >>> in_channels = [2, 3, 5, 7] + >>> scales = [340, 170, 84, 43] + >>> inputs = [torch.rand(1, c, s, s) + ... for c, s in zip(in_channels, scales)] + >>> self = ChannelMapper(in_channels, 11, 3).eval() + >>> outputs = self.forward(inputs) + >>> for i in range(len(outputs)): + ... print(f'outputs[{i}].shape = {outputs[i].shape}') + outputs[0].shape = torch.Size([1, 11, 340, 340]) + outputs[1].shape = torch.Size([1, 11, 170, 170]) + outputs[2].shape = torch.Size([1, 11, 84, 84]) + outputs[3].shape = torch.Size([1, 11, 43, 43]) + """ + + def __init__(self, + in_channels, + out_channels, + kernel_size=3, + conv_cfg=None, + norm_cfg=None, + act_cfg=dict(type='ReLU')): + super(ChannelMapper, self).__init__() + assert isinstance(in_channels, list) + + self.convs = nn.ModuleList() + for in_channel in in_channels: + self.convs.append( + ConvModule( + in_channel, + out_channels, + kernel_size, + padding=(kernel_size - 1) // 2, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg)) + + # default init_weights for conv(msra) and norm in ConvModule + def init_weights(self): + """Initialize the weights of ChannelMapper module.""" + for m in self.modules(): + if isinstance(m, nn.Conv2d): + xavier_init(m, distribution='uniform') + + def forward(self, inputs): + """Forward function.""" + assert len(inputs) == len(self.convs) + outs = [self.convs[i](inputs[i]) for i in range(len(inputs))] + return tuple(outs) diff --git a/detection/mmdet/models/necks/fpg.py b/detection/mmdet/models/necks/fpg.py new file mode 100644 index 0000000..c8e0d16 --- /dev/null +++ b/detection/mmdet/models/necks/fpg.py @@ -0,0 +1,398 @@ +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import ConvModule, caffe2_xavier_init, constant_init, is_norm + +from ..builder import NECKS + + +class Transition(nn.Module): + """Base class for transition. + + Args: + in_channels (int): Number of input channels. + out_channels (int): Number of output channels. + """ + + def __init__(self, in_channels, out_channels): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + + def forward(x): + pass + + +class UpInterpolationConv(Transition): + """A transition used for up-sampling. + + Up-sample the input by interpolation then refines the feature by + a convolution layer. + + Args: + in_channels (int): Number of input channels. + out_channels (int): Number of output channels. + scale_factor (int): Up-sampling factor. Default: 2. + mode (int): Interpolation mode. Default: nearest. + align_corners (bool): Whether align corners when interpolation. + Default: None. + kernel_size (int): Kernel size for the conv. Default: 3. + """ + + def __init__(self, + in_channels, + out_channels, + scale_factor=2, + mode='nearest', + align_corners=None, + kernel_size=3, + **kwargs): + super().__init__(in_channels, out_channels) + self.mode = mode + self.scale_factor = scale_factor + self.align_corners = align_corners + self.conv = ConvModule( + in_channels, + out_channels, + kernel_size, + padding=(kernel_size - 1) // 2, + **kwargs) + + def forward(self, x): + x = F.interpolate( + x, + scale_factor=self.scale_factor, + mode=self.mode, + align_corners=self.align_corners) + x = self.conv(x) + return x + + +class LastConv(Transition): + """A transition used for refining the output of the last stage. + + Args: + in_channels (int): Number of input channels. + out_channels (int): Number of output channels. + num_inputs (int): Number of inputs of the FPN features. + kernel_size (int): Kernel size for the conv. Default: 3. + """ + + def __init__(self, + in_channels, + out_channels, + num_inputs, + kernel_size=3, + **kwargs): + super().__init__(in_channels, out_channels) + self.num_inputs = num_inputs + self.conv_out = ConvModule( + in_channels, + out_channels, + kernel_size, + padding=(kernel_size - 1) // 2, + **kwargs) + + def forward(self, inputs): + assert len(inputs) == self.num_inputs + return self.conv_out(inputs[-1]) + + +@NECKS.register_module() +class FPG(nn.Module): + """FPG. + + Implementation of `Feature Pyramid Grids (FPG) + `_. + This implementation only gives the basic structure stated in the paper. + But users can implement different type of transitions to fully explore the + the potential power of the structure of FPG. + + Args: + in_channels (int): Number of input channels (feature maps of all levels + should have the same channels). + out_channels (int): Number of output channels (used at each scale) + num_outs (int): Number of output scales. + stack_times (int): The number of times the pyramid architecture will + be stacked. + paths (list[str]): Specify the path order of each stack level. + Each element in the list should be either 'bu' (bottom-up) or + 'td' (top-down). + inter_channels (int): Number of inter channels. + same_up_trans (dict): Transition that goes down at the same stage. + same_down_trans (dict): Transition that goes up at the same stage. + across_lateral_trans (dict): Across-pathway same-stage + across_down_trans (dict): Across-pathway bottom-up connection. + across_up_trans (dict): Across-pathway top-down connection. + across_skip_trans (dict): Across-pathway skip connection. + output_trans (dict): Transition that trans the output of the + last stage. + start_level (int): Index of the start input backbone level used to + build the feature pyramid. Default: 0. + end_level (int): Index of the end input backbone level (exclusive) to + build the feature pyramid. Default: -1, which means the last level. + add_extra_convs (bool): It decides whether to add conv + layers on top of the original feature maps. Default to False. + If True, its actual mode is specified by `extra_convs_on_inputs`. + norm_cfg (dict): Config dict for normalization layer. Default: None. + """ + + transition_types = { + 'conv': ConvModule, + 'interpolation_conv': UpInterpolationConv, + 'last_conv': LastConv, + } + + def __init__(self, + in_channels, + out_channels, + num_outs, + stack_times, + paths, + inter_channels=None, + same_down_trans=None, + same_up_trans=dict( + type='conv', kernel_size=3, stride=2, padding=1), + across_lateral_trans=dict(type='conv', kernel_size=1), + across_down_trans=dict(type='conv', kernel_size=3), + across_up_trans=None, + across_skip_trans=dict(type='identity'), + output_trans=dict(type='last_conv', kernel_size=3), + start_level=0, + end_level=-1, + add_extra_convs=False, + norm_cfg=None, + skip_inds=None): + super(FPG, self).__init__() + assert isinstance(in_channels, list) + self.in_channels = in_channels + self.out_channels = out_channels + self.num_ins = len(in_channels) + self.num_outs = num_outs + if inter_channels is None: + self.inter_channels = [out_channels for _ in range(num_outs)] + elif isinstance(inter_channels, int): + self.inter_channels = [inter_channels for _ in range(num_outs)] + else: + assert isinstance(inter_channels, list) + assert len(inter_channels) == num_outs + self.inter_channels = inter_channels + self.stack_times = stack_times + self.paths = paths + assert isinstance(paths, list) and len(paths) == stack_times + for d in paths: + assert d in ('bu', 'td') + + self.same_down_trans = same_down_trans + self.same_up_trans = same_up_trans + self.across_lateral_trans = across_lateral_trans + self.across_down_trans = across_down_trans + self.across_up_trans = across_up_trans + self.output_trans = output_trans + self.across_skip_trans = across_skip_trans + + self.with_bias = norm_cfg is None + # skip inds must be specified if across skip trans is not None + if self.across_skip_trans is not None: + skip_inds is not None + self.skip_inds = skip_inds + assert len(self.skip_inds[0]) <= self.stack_times + + if end_level == -1: + self.backbone_end_level = self.num_ins + assert num_outs >= self.num_ins - start_level + else: + # if end_level < inputs, no extra level is allowed + self.backbone_end_level = end_level + assert end_level <= len(in_channels) + assert num_outs == end_level - start_level + self.start_level = start_level + self.end_level = end_level + self.add_extra_convs = add_extra_convs + + # build lateral 1x1 convs to reduce channels + self.lateral_convs = nn.ModuleList() + for i in range(self.start_level, self.backbone_end_level): + l_conv = nn.Conv2d(self.in_channels[i], + self.inter_channels[i - self.start_level], 1) + self.lateral_convs.append(l_conv) + + extra_levels = num_outs - self.backbone_end_level + self.start_level + self.extra_downsamples = nn.ModuleList() + for i in range(extra_levels): + if self.add_extra_convs: + fpn_idx = self.backbone_end_level - self.start_level + i + extra_conv = nn.Conv2d( + self.inter_channels[fpn_idx - 1], + self.inter_channels[fpn_idx], + 3, + stride=2, + padding=1) + self.extra_downsamples.append(extra_conv) + else: + self.extra_downsamples.append(nn.MaxPool2d(1, stride=2)) + + self.fpn_transitions = nn.ModuleList() # stack times + for s in range(self.stack_times): + stage_trans = nn.ModuleList() # num of feature levels + for i in range(self.num_outs): + # same, across_lateral, across_down, across_up + trans = nn.ModuleDict() + if s in self.skip_inds[i]: + stage_trans.append(trans) + continue + # build same-stage down trans (used in bottom-up paths) + if i == 0 or self.same_up_trans is None: + same_up_trans = None + else: + same_up_trans = self.build_trans( + self.same_up_trans, self.inter_channels[i - 1], + self.inter_channels[i]) + trans['same_up'] = same_up_trans + # build same-stage up trans (used in top-down paths) + if i == self.num_outs - 1 or self.same_down_trans is None: + same_down_trans = None + else: + same_down_trans = self.build_trans( + self.same_down_trans, self.inter_channels[i + 1], + self.inter_channels[i]) + trans['same_down'] = same_down_trans + # build across lateral trans + across_lateral_trans = self.build_trans( + self.across_lateral_trans, self.inter_channels[i], + self.inter_channels[i]) + trans['across_lateral'] = across_lateral_trans + # build across down trans + if i == self.num_outs - 1 or self.across_down_trans is None: + across_down_trans = None + else: + across_down_trans = self.build_trans( + self.across_down_trans, self.inter_channels[i + 1], + self.inter_channels[i]) + trans['across_down'] = across_down_trans + # build across up trans + if i == 0 or self.across_up_trans is None: + across_up_trans = None + else: + across_up_trans = self.build_trans( + self.across_up_trans, self.inter_channels[i - 1], + self.inter_channels[i]) + trans['across_up'] = across_up_trans + if self.across_skip_trans is None: + across_skip_trans = None + else: + across_skip_trans = self.build_trans( + self.across_skip_trans, self.inter_channels[i - 1], + self.inter_channels[i]) + trans['across_skip'] = across_skip_trans + # build across_skip trans + stage_trans.append(trans) + self.fpn_transitions.append(stage_trans) + + self.output_transition = nn.ModuleList() # output levels + for i in range(self.num_outs): + trans = self.build_trans( + self.output_trans, + self.inter_channels[i], + self.out_channels, + num_inputs=self.stack_times + 1) + self.output_transition.append(trans) + + self.relu = nn.ReLU(inplace=True) + + def build_trans(self, cfg, in_channels, out_channels, **extra_args): + cfg_ = cfg.copy() + trans_type = cfg_.pop('type') + trans_cls = self.transition_types[trans_type] + return trans_cls(in_channels, out_channels, **cfg_, **extra_args) + + def init_weights(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + caffe2_xavier_init(m) + elif is_norm(m): + constant_init(m, 1.0) + + def fuse(self, fuse_dict): + out = None + for item in fuse_dict.values(): + if item is not None: + if out is None: + out = item + else: + out = out + item + return out + + def forward(self, inputs): + assert len(inputs) == len(self.in_channels) + + # build all levels from original feature maps + feats = [ + lateral_conv(inputs[i + self.start_level]) + for i, lateral_conv in enumerate(self.lateral_convs) + ] + for downsample in self.extra_downsamples: + feats.append(downsample(feats[-1])) + + outs = [feats] + + for i in range(self.stack_times): + current_outs = outs[-1] + next_outs = [] + direction = self.paths[i] + for j in range(self.num_outs): + if i in self.skip_inds[j]: + next_outs.append(outs[-1][j]) + continue + # feature level + if direction == 'td': + lvl = self.num_outs - j - 1 + else: + lvl = j + # get transitions + if direction == 'td': + same_trans = self.fpn_transitions[i][lvl]['same_down'] + else: + same_trans = self.fpn_transitions[i][lvl]['same_up'] + across_lateral_trans = self.fpn_transitions[i][lvl][ + 'across_lateral'] + across_down_trans = self.fpn_transitions[i][lvl]['across_down'] + across_up_trans = self.fpn_transitions[i][lvl]['across_up'] + across_skip_trans = self.fpn_transitions[i][lvl]['across_skip'] + # init output + to_fuse = dict( + same=None, lateral=None, across_up=None, across_down=None) + # same downsample/upsample + if same_trans is not None: + to_fuse['same'] = same_trans(next_outs[-1]) + # across lateral + if across_lateral_trans is not None: + to_fuse['lateral'] = across_lateral_trans( + current_outs[lvl]) + # across downsample + if lvl > 0 and across_up_trans is not None: + to_fuse['across_up'] = across_up_trans(current_outs[lvl - + 1]) + # across upsample + if (lvl < self.num_outs - 1 and across_down_trans is not None): + to_fuse['across_down'] = across_down_trans( + current_outs[lvl + 1]) + if across_skip_trans is not None: + to_fuse['across_skip'] = across_skip_trans(outs[0][lvl]) + x = self.fuse(to_fuse) + next_outs.append(x) + + if direction == 'td': + outs.append(next_outs[::-1]) + else: + outs.append(next_outs) + + # output trans + final_outs = [] + for i in range(self.num_outs): + lvl_out_list = [] + for s in range(len(outs)): + lvl_out_list.append(outs[s][i]) + lvl_out = self.output_transition[i](lvl_out_list) + final_outs.append(lvl_out) + + return final_outs diff --git a/detection/mmdet/models/necks/fpn.py b/detection/mmdet/models/necks/fpn.py new file mode 100644 index 0000000..5e5dfe6 --- /dev/null +++ b/detection/mmdet/models/necks/fpn.py @@ -0,0 +1,221 @@ +import warnings + +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import ConvModule, xavier_init +from mmcv.runner import auto_fp16 + +from ..builder import NECKS + + +@NECKS.register_module() +class FPN(nn.Module): + r"""Feature Pyramid Network. + + This is an implementation of paper `Feature Pyramid Networks for Object + Detection `_. + + Args: + in_channels (List[int]): Number of input channels per scale. + out_channels (int): Number of output channels (used at each scale) + num_outs (int): Number of output scales. + start_level (int): Index of the start input backbone level used to + build the feature pyramid. Default: 0. + end_level (int): Index of the end input backbone level (exclusive) to + build the feature pyramid. Default: -1, which means the last level. + add_extra_convs (bool | str): If bool, it decides whether to add conv + layers on top of the original feature maps. Default to False. + If True, its actual mode is specified by `extra_convs_on_inputs`. + If str, it specifies the source feature map of the extra convs. + Only the following options are allowed + + - 'on_input': Last feat map of neck inputs (i.e. backbone feature). + - 'on_lateral': Last feature map after lateral convs. + - 'on_output': The last output feature map after fpn convs. + extra_convs_on_inputs (bool, deprecated): Whether to apply extra convs + on the original feature from the backbone. If True, + it is equivalent to `add_extra_convs='on_input'`. If False, it is + equivalent to set `add_extra_convs='on_output'`. Default to True. + relu_before_extra_convs (bool): Whether to apply relu before the extra + conv. Default: False. + no_norm_on_lateral (bool): Whether to apply norm on lateral. + Default: False. + conv_cfg (dict): Config dict for convolution layer. Default: None. + norm_cfg (dict): Config dict for normalization layer. Default: None. + act_cfg (str): Config dict for activation layer in ConvModule. + Default: None. + upsample_cfg (dict): Config dict for interpolate layer. + Default: `dict(mode='nearest')` + + Example: + >>> import torch + >>> in_channels = [2, 3, 5, 7] + >>> scales = [340, 170, 84, 43] + >>> inputs = [torch.rand(1, c, s, s) + ... for c, s in zip(in_channels, scales)] + >>> self = FPN(in_channels, 11, len(in_channels)).eval() + >>> outputs = self.forward(inputs) + >>> for i in range(len(outputs)): + ... print(f'outputs[{i}].shape = {outputs[i].shape}') + outputs[0].shape = torch.Size([1, 11, 340, 340]) + outputs[1].shape = torch.Size([1, 11, 170, 170]) + outputs[2].shape = torch.Size([1, 11, 84, 84]) + outputs[3].shape = torch.Size([1, 11, 43, 43]) + """ + + def __init__(self, + in_channels, + out_channels, + num_outs, + start_level=0, + end_level=-1, + add_extra_convs=False, + extra_convs_on_inputs=True, + relu_before_extra_convs=False, + no_norm_on_lateral=False, + conv_cfg=None, + norm_cfg=None, + act_cfg=None, + upsample_cfg=dict(mode='nearest')): + super(FPN, self).__init__() + assert isinstance(in_channels, list) + self.in_channels = in_channels + self.out_channels = out_channels + self.num_ins = len(in_channels) + self.num_outs = num_outs + self.relu_before_extra_convs = relu_before_extra_convs + self.no_norm_on_lateral = no_norm_on_lateral + self.fp16_enabled = False + self.upsample_cfg = upsample_cfg.copy() + + if end_level == -1: + self.backbone_end_level = self.num_ins + assert num_outs >= self.num_ins - start_level + else: + # if end_level < inputs, no extra level is allowed + self.backbone_end_level = end_level + assert end_level <= len(in_channels) + assert num_outs == end_level - start_level + self.start_level = start_level + self.end_level = end_level + self.add_extra_convs = add_extra_convs + assert isinstance(add_extra_convs, (str, bool)) + if isinstance(add_extra_convs, str): + # Extra_convs_source choices: 'on_input', 'on_lateral', 'on_output' + assert add_extra_convs in ('on_input', 'on_lateral', 'on_output') + elif add_extra_convs: # True + if extra_convs_on_inputs: + # TODO: deprecate `extra_convs_on_inputs` + warnings.simplefilter('once') + warnings.warn( + '"extra_convs_on_inputs" will be deprecated in v2.9.0,' + 'Please use "add_extra_convs"', DeprecationWarning) + self.add_extra_convs = 'on_input' + else: + self.add_extra_convs = 'on_output' + + self.lateral_convs = nn.ModuleList() + self.fpn_convs = nn.ModuleList() + + for i in range(self.start_level, self.backbone_end_level): + l_conv = ConvModule( + in_channels[i], + out_channels, + 1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg if not self.no_norm_on_lateral else None, + act_cfg=act_cfg, + inplace=False) + fpn_conv = ConvModule( + out_channels, + out_channels, + 3, + padding=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + inplace=False) + + self.lateral_convs.append(l_conv) + self.fpn_convs.append(fpn_conv) + + # add extra conv layers (e.g., RetinaNet) + extra_levels = num_outs - self.backbone_end_level + self.start_level + if self.add_extra_convs and extra_levels >= 1: + for i in range(extra_levels): + if i == 0 and self.add_extra_convs == 'on_input': + in_channels = self.in_channels[self.backbone_end_level - 1] + else: + in_channels = out_channels + extra_fpn_conv = ConvModule( + in_channels, + out_channels, + 3, + stride=2, + padding=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + inplace=False) + self.fpn_convs.append(extra_fpn_conv) + + # default init_weights for conv(msra) and norm in ConvModule + def init_weights(self): + """Initialize the weights of FPN module.""" + for m in self.modules(): + if isinstance(m, nn.Conv2d): + xavier_init(m, distribution='uniform') + + @auto_fp16() + def forward(self, inputs): + """Forward function.""" + assert len(inputs) == len(self.in_channels) + + # build laterals + laterals = [ + lateral_conv(inputs[i + self.start_level]) + for i, lateral_conv in enumerate(self.lateral_convs) + ] + + # build top-down path + used_backbone_levels = len(laterals) + for i in range(used_backbone_levels - 1, 0, -1): + # In some cases, fixing `scale factor` (e.g. 2) is preferred, but + # it cannot co-exist with `size` in `F.interpolate`. + if 'scale_factor' in self.upsample_cfg: + laterals[i - 1] += F.interpolate(laterals[i], + **self.upsample_cfg) + else: + prev_shape = laterals[i - 1].shape[2:] + laterals[i - 1] += F.interpolate( + laterals[i], size=prev_shape, **self.upsample_cfg) + + # build outputs + # part 1: from original levels + outs = [ + self.fpn_convs[i](laterals[i]) for i in range(used_backbone_levels) + ] + # part 2: add extra levels + if self.num_outs > len(outs): + # use max pool to get more levels on top of outputs + # (e.g., Faster R-CNN, Mask R-CNN) + if not self.add_extra_convs: + for i in range(self.num_outs - used_backbone_levels): + outs.append(F.max_pool2d(outs[-1], 1, stride=2)) + # add conv layers on top of original feature maps (RetinaNet) + else: + if self.add_extra_convs == 'on_input': + extra_source = inputs[self.backbone_end_level - 1] + elif self.add_extra_convs == 'on_lateral': + extra_source = laterals[-1] + elif self.add_extra_convs == 'on_output': + extra_source = outs[-1] + else: + raise NotImplementedError + outs.append(self.fpn_convs[used_backbone_levels](extra_source)) + for i in range(used_backbone_levels + 1, self.num_outs): + if self.relu_before_extra_convs: + outs.append(self.fpn_convs[i](F.relu(outs[-1]))) + else: + outs.append(self.fpn_convs[i](outs[-1])) + return tuple(outs) diff --git a/detection/mmdet/models/necks/fpn_carafe.py b/detection/mmdet/models/necks/fpn_carafe.py new file mode 100644 index 0000000..302e657 --- /dev/null +++ b/detection/mmdet/models/necks/fpn_carafe.py @@ -0,0 +1,267 @@ +import torch.nn as nn +from mmcv.cnn import ConvModule, build_upsample_layer, xavier_init +from mmcv.ops.carafe import CARAFEPack + +from ..builder import NECKS + + +@NECKS.register_module() +class FPN_CARAFE(nn.Module): + """FPN_CARAFE is a more flexible implementation of FPN. It allows more + choice for upsample methods during the top-down pathway. + + It can reproduce the performance of ICCV 2019 paper + CARAFE: Content-Aware ReAssembly of FEatures + Please refer to https://arxiv.org/abs/1905.02188 for more details. + + Args: + in_channels (list[int]): Number of channels for each input feature map. + out_channels (int): Output channels of feature pyramids. + num_outs (int): Number of output stages. + start_level (int): Start level of feature pyramids. + (Default: 0) + end_level (int): End level of feature pyramids. + (Default: -1 indicates the last level). + norm_cfg (dict): Dictionary to construct and config norm layer. + activate (str): Type of activation function in ConvModule + (Default: None indicates w/o activation). + order (dict): Order of components in ConvModule. + upsample (str): Type of upsample layer. + upsample_cfg (dict): Dictionary to construct and config upsample layer. + """ + + def __init__(self, + in_channels, + out_channels, + num_outs, + start_level=0, + end_level=-1, + norm_cfg=None, + act_cfg=None, + order=('conv', 'norm', 'act'), + upsample_cfg=dict( + type='carafe', + up_kernel=5, + up_group=1, + encoder_kernel=3, + encoder_dilation=1)): + super(FPN_CARAFE, self).__init__() + assert isinstance(in_channels, list) + self.in_channels = in_channels + self.out_channels = out_channels + self.num_ins = len(in_channels) + self.num_outs = num_outs + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + self.with_bias = norm_cfg is None + self.upsample_cfg = upsample_cfg.copy() + self.upsample = self.upsample_cfg.get('type') + self.relu = nn.ReLU(inplace=False) + + self.order = order + assert order in [('conv', 'norm', 'act'), ('act', 'conv', 'norm')] + + assert self.upsample in [ + 'nearest', 'bilinear', 'deconv', 'pixel_shuffle', 'carafe', None + ] + if self.upsample in ['deconv', 'pixel_shuffle']: + assert hasattr( + self.upsample_cfg, + 'upsample_kernel') and self.upsample_cfg.upsample_kernel > 0 + self.upsample_kernel = self.upsample_cfg.pop('upsample_kernel') + + if end_level == -1: + self.backbone_end_level = self.num_ins + assert num_outs >= self.num_ins - start_level + else: + # if end_level < inputs, no extra level is allowed + self.backbone_end_level = end_level + assert end_level <= len(in_channels) + assert num_outs == end_level - start_level + self.start_level = start_level + self.end_level = end_level + + self.lateral_convs = nn.ModuleList() + self.fpn_convs = nn.ModuleList() + self.upsample_modules = nn.ModuleList() + + for i in range(self.start_level, self.backbone_end_level): + l_conv = ConvModule( + in_channels[i], + out_channels, + 1, + norm_cfg=norm_cfg, + bias=self.with_bias, + act_cfg=act_cfg, + inplace=False, + order=self.order) + fpn_conv = ConvModule( + out_channels, + out_channels, + 3, + padding=1, + norm_cfg=self.norm_cfg, + bias=self.with_bias, + act_cfg=act_cfg, + inplace=False, + order=self.order) + if i != self.backbone_end_level - 1: + upsample_cfg_ = self.upsample_cfg.copy() + if self.upsample == 'deconv': + upsample_cfg_.update( + in_channels=out_channels, + out_channels=out_channels, + kernel_size=self.upsample_kernel, + stride=2, + padding=(self.upsample_kernel - 1) // 2, + output_padding=(self.upsample_kernel - 1) // 2) + elif self.upsample == 'pixel_shuffle': + upsample_cfg_.update( + in_channels=out_channels, + out_channels=out_channels, + scale_factor=2, + upsample_kernel=self.upsample_kernel) + elif self.upsample == 'carafe': + upsample_cfg_.update(channels=out_channels, scale_factor=2) + else: + # suppress warnings + align_corners = (None + if self.upsample == 'nearest' else False) + upsample_cfg_.update( + scale_factor=2, + mode=self.upsample, + align_corners=align_corners) + upsample_module = build_upsample_layer(upsample_cfg_) + self.upsample_modules.append(upsample_module) + self.lateral_convs.append(l_conv) + self.fpn_convs.append(fpn_conv) + + # add extra conv layers (e.g., RetinaNet) + extra_out_levels = ( + num_outs - self.backbone_end_level + self.start_level) + if extra_out_levels >= 1: + for i in range(extra_out_levels): + in_channels = ( + self.in_channels[self.backbone_end_level - + 1] if i == 0 else out_channels) + extra_l_conv = ConvModule( + in_channels, + out_channels, + 3, + stride=2, + padding=1, + norm_cfg=norm_cfg, + bias=self.with_bias, + act_cfg=act_cfg, + inplace=False, + order=self.order) + if self.upsample == 'deconv': + upsampler_cfg_ = dict( + in_channels=out_channels, + out_channels=out_channels, + kernel_size=self.upsample_kernel, + stride=2, + padding=(self.upsample_kernel - 1) // 2, + output_padding=(self.upsample_kernel - 1) // 2) + elif self.upsample == 'pixel_shuffle': + upsampler_cfg_ = dict( + in_channels=out_channels, + out_channels=out_channels, + scale_factor=2, + upsample_kernel=self.upsample_kernel) + elif self.upsample == 'carafe': + upsampler_cfg_ = dict( + channels=out_channels, + scale_factor=2, + **self.upsample_cfg) + else: + # suppress warnings + align_corners = (None + if self.upsample == 'nearest' else False) + upsampler_cfg_ = dict( + scale_factor=2, + mode=self.upsample, + align_corners=align_corners) + upsampler_cfg_['type'] = self.upsample + upsample_module = build_upsample_layer(upsampler_cfg_) + extra_fpn_conv = ConvModule( + out_channels, + out_channels, + 3, + padding=1, + norm_cfg=self.norm_cfg, + bias=self.with_bias, + act_cfg=act_cfg, + inplace=False, + order=self.order) + self.upsample_modules.append(upsample_module) + self.fpn_convs.append(extra_fpn_conv) + self.lateral_convs.append(extra_l_conv) + + # default init_weights for conv(msra) and norm in ConvModule + def init_weights(self): + """Initialize the weights of module.""" + for m in self.modules(): + if isinstance(m, (nn.Conv2d, nn.ConvTranspose2d)): + xavier_init(m, distribution='uniform') + for m in self.modules(): + if isinstance(m, CARAFEPack): + m.init_weights() + + def slice_as(self, src, dst): + """Slice ``src`` as ``dst`` + + Note: + ``src`` should have the same or larger size than ``dst``. + + Args: + src (torch.Tensor): Tensors to be sliced. + dst (torch.Tensor): ``src`` will be sliced to have the same + size as ``dst``. + + Returns: + torch.Tensor: Sliced tensor. + """ + assert (src.size(2) >= dst.size(2)) and (src.size(3) >= dst.size(3)) + if src.size(2) == dst.size(2) and src.size(3) == dst.size(3): + return src + else: + return src[:, :, :dst.size(2), :dst.size(3)] + + def tensor_add(self, a, b): + """Add tensors ``a`` and ``b`` that might have different sizes.""" + if a.size() == b.size(): + c = a + b + else: + c = a + self.slice_as(b, a) + return c + + def forward(self, inputs): + """Forward function.""" + assert len(inputs) == len(self.in_channels) + + # build laterals + laterals = [] + for i, lateral_conv in enumerate(self.lateral_convs): + if i <= self.backbone_end_level - self.start_level: + input = inputs[min(i + self.start_level, len(inputs) - 1)] + else: + input = laterals[-1] + lateral = lateral_conv(input) + laterals.append(lateral) + + # build top-down path + for i in range(len(laterals) - 1, 0, -1): + if self.upsample is not None: + upsample_feat = self.upsample_modules[i - 1](laterals[i]) + else: + upsample_feat = laterals[i] + laterals[i - 1] = self.tensor_add(laterals[i - 1], upsample_feat) + + # build outputs + num_conv_outs = len(self.fpn_convs) + outs = [] + for i in range(num_conv_outs): + out = self.fpn_convs[i](laterals[i]) + outs.append(out) + return tuple(outs) diff --git a/detection/mmdet/models/necks/hrfpn.py b/detection/mmdet/models/necks/hrfpn.py new file mode 100644 index 0000000..ed4f194 --- /dev/null +++ b/detection/mmdet/models/necks/hrfpn.py @@ -0,0 +1,102 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import ConvModule, caffe2_xavier_init +from torch.utils.checkpoint import checkpoint + +from ..builder import NECKS + + +@NECKS.register_module() +class HRFPN(nn.Module): + """HRFPN (High Resolution Feature Pyramids) + + paper: `High-Resolution Representations for Labeling Pixels and Regions + `_. + + Args: + in_channels (list): number of channels for each branch. + out_channels (int): output channels of feature pyramids. + num_outs (int): number of output stages. + pooling_type (str): pooling for generating feature pyramids + from {MAX, AVG}. + conv_cfg (dict): dictionary to construct and config conv layer. + norm_cfg (dict): dictionary to construct and config norm layer. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. + stride (int): stride of 3x3 convolutional layers + """ + + def __init__(self, + in_channels, + out_channels, + num_outs=5, + pooling_type='AVG', + conv_cfg=None, + norm_cfg=None, + with_cp=False, + stride=1): + super(HRFPN, self).__init__() + assert isinstance(in_channels, list) + self.in_channels = in_channels + self.out_channels = out_channels + self.num_ins = len(in_channels) + self.num_outs = num_outs + self.with_cp = with_cp + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + + self.reduction_conv = ConvModule( + sum(in_channels), + out_channels, + kernel_size=1, + conv_cfg=self.conv_cfg, + act_cfg=None) + + self.fpn_convs = nn.ModuleList() + for i in range(self.num_outs): + self.fpn_convs.append( + ConvModule( + out_channels, + out_channels, + kernel_size=3, + padding=1, + stride=stride, + conv_cfg=self.conv_cfg, + act_cfg=None)) + + if pooling_type == 'MAX': + self.pooling = F.max_pool2d + else: + self.pooling = F.avg_pool2d + + def init_weights(self): + """Initialize the weights of module.""" + for m in self.modules(): + if isinstance(m, nn.Conv2d): + caffe2_xavier_init(m) + + def forward(self, inputs): + """Forward function.""" + assert len(inputs) == self.num_ins + outs = [inputs[0]] + for i in range(1, self.num_ins): + outs.append( + F.interpolate(inputs[i], scale_factor=2**i, mode='bilinear')) + out = torch.cat(outs, dim=1) + if out.requires_grad and self.with_cp: + out = checkpoint(self.reduction_conv, out) + else: + out = self.reduction_conv(out) + outs = [out] + for i in range(1, self.num_outs): + outs.append(self.pooling(out, kernel_size=2**i, stride=2**i)) + outputs = [] + + for i in range(self.num_outs): + if outs[i].requires_grad and self.with_cp: + tmp_out = checkpoint(self.fpn_convs[i], outs[i]) + else: + tmp_out = self.fpn_convs[i](outs[i]) + outputs.append(tmp_out) + return tuple(outputs) diff --git a/detection/mmdet/models/necks/nas_fpn.py b/detection/mmdet/models/necks/nas_fpn.py new file mode 100644 index 0000000..8e333ce --- /dev/null +++ b/detection/mmdet/models/necks/nas_fpn.py @@ -0,0 +1,160 @@ +import torch.nn as nn +from mmcv.cnn import ConvModule, caffe2_xavier_init +from mmcv.ops.merge_cells import GlobalPoolingCell, SumCell + +from ..builder import NECKS + + +@NECKS.register_module() +class NASFPN(nn.Module): + """NAS-FPN. + + Implementation of `NAS-FPN: Learning Scalable Feature Pyramid Architecture + for Object Detection `_ + + Args: + in_channels (List[int]): Number of input channels per scale. + out_channels (int): Number of output channels (used at each scale) + num_outs (int): Number of output scales. + stack_times (int): The number of times the pyramid architecture will + be stacked. + start_level (int): Index of the start input backbone level used to + build the feature pyramid. Default: 0. + end_level (int): Index of the end input backbone level (exclusive) to + build the feature pyramid. Default: -1, which means the last level. + add_extra_convs (bool): It decides whether to add conv + layers on top of the original feature maps. Default to False. + If True, its actual mode is specified by `extra_convs_on_inputs`. + """ + + def __init__(self, + in_channels, + out_channels, + num_outs, + stack_times, + start_level=0, + end_level=-1, + add_extra_convs=False, + norm_cfg=None): + super(NASFPN, self).__init__() + assert isinstance(in_channels, list) + self.in_channels = in_channels + self.out_channels = out_channels + self.num_ins = len(in_channels) # num of input feature levels + self.num_outs = num_outs # num of output feature levels + self.stack_times = stack_times + self.norm_cfg = norm_cfg + + if end_level == -1: + self.backbone_end_level = self.num_ins + assert num_outs >= self.num_ins - start_level + else: + # if end_level < inputs, no extra level is allowed + self.backbone_end_level = end_level + assert end_level <= len(in_channels) + assert num_outs == end_level - start_level + self.start_level = start_level + self.end_level = end_level + self.add_extra_convs = add_extra_convs + + # add lateral connections + self.lateral_convs = nn.ModuleList() + for i in range(self.start_level, self.backbone_end_level): + l_conv = ConvModule( + in_channels[i], + out_channels, + 1, + norm_cfg=norm_cfg, + act_cfg=None) + self.lateral_convs.append(l_conv) + + # add extra downsample layers (stride-2 pooling or conv) + extra_levels = num_outs - self.backbone_end_level + self.start_level + self.extra_downsamples = nn.ModuleList() + for i in range(extra_levels): + extra_conv = ConvModule( + out_channels, out_channels, 1, norm_cfg=norm_cfg, act_cfg=None) + self.extra_downsamples.append( + nn.Sequential(extra_conv, nn.MaxPool2d(2, 2))) + + # add NAS FPN connections + self.fpn_stages = nn.ModuleList() + for _ in range(self.stack_times): + stage = nn.ModuleDict() + # gp(p6, p4) -> p4_1 + stage['gp_64_4'] = GlobalPoolingCell( + in_channels=out_channels, + out_channels=out_channels, + out_norm_cfg=norm_cfg) + # sum(p4_1, p4) -> p4_2 + stage['sum_44_4'] = SumCell( + in_channels=out_channels, + out_channels=out_channels, + out_norm_cfg=norm_cfg) + # sum(p4_2, p3) -> p3_out + stage['sum_43_3'] = SumCell( + in_channels=out_channels, + out_channels=out_channels, + out_norm_cfg=norm_cfg) + # sum(p3_out, p4_2) -> p4_out + stage['sum_34_4'] = SumCell( + in_channels=out_channels, + out_channels=out_channels, + out_norm_cfg=norm_cfg) + # sum(p5, gp(p4_out, p3_out)) -> p5_out + stage['gp_43_5'] = GlobalPoolingCell(with_out_conv=False) + stage['sum_55_5'] = SumCell( + in_channels=out_channels, + out_channels=out_channels, + out_norm_cfg=norm_cfg) + # sum(p7, gp(p5_out, p4_2)) -> p7_out + stage['gp_54_7'] = GlobalPoolingCell(with_out_conv=False) + stage['sum_77_7'] = SumCell( + in_channels=out_channels, + out_channels=out_channels, + out_norm_cfg=norm_cfg) + # gp(p7_out, p5_out) -> p6_out + stage['gp_75_6'] = GlobalPoolingCell( + in_channels=out_channels, + out_channels=out_channels, + out_norm_cfg=norm_cfg) + self.fpn_stages.append(stage) + + def init_weights(self): + """Initialize the weights of module.""" + for m in self.modules(): + if isinstance(m, nn.Conv2d): + caffe2_xavier_init(m) + + def forward(self, inputs): + """Forward function.""" + # build P3-P5 + feats = [ + lateral_conv(inputs[i + self.start_level]) + for i, lateral_conv in enumerate(self.lateral_convs) + ] + # build P6-P7 on top of P5 + for downsample in self.extra_downsamples: + feats.append(downsample(feats[-1])) + + p3, p4, p5, p6, p7 = feats + + for stage in self.fpn_stages: + # gp(p6, p4) -> p4_1 + p4_1 = stage['gp_64_4'](p6, p4, out_size=p4.shape[-2:]) + # sum(p4_1, p4) -> p4_2 + p4_2 = stage['sum_44_4'](p4_1, p4, out_size=p4.shape[-2:]) + # sum(p4_2, p3) -> p3_out + p3 = stage['sum_43_3'](p4_2, p3, out_size=p3.shape[-2:]) + # sum(p3_out, p4_2) -> p4_out + p4 = stage['sum_34_4'](p3, p4_2, out_size=p4.shape[-2:]) + # sum(p5, gp(p4_out, p3_out)) -> p5_out + p5_tmp = stage['gp_43_5'](p4, p3, out_size=p5.shape[-2:]) + p5 = stage['sum_55_5'](p5, p5_tmp, out_size=p5.shape[-2:]) + # sum(p7, gp(p5_out, p4_2)) -> p7_out + p7_tmp = stage['gp_54_7'](p5, p4_2, out_size=p7.shape[-2:]) + p7 = stage['sum_77_7'](p7, p7_tmp, out_size=p7.shape[-2:]) + # gp(p7_out, p5_out) -> p6_out + p6 = stage['gp_75_6'](p7, p5, out_size=p6.shape[-2:]) + + return p3, p4, p5, p6, p7 diff --git a/detection/mmdet/models/necks/nasfcos_fpn.py b/detection/mmdet/models/necks/nasfcos_fpn.py new file mode 100644 index 0000000..2daf79e --- /dev/null +++ b/detection/mmdet/models/necks/nasfcos_fpn.py @@ -0,0 +1,161 @@ +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import ConvModule, caffe2_xavier_init +from mmcv.ops.merge_cells import ConcatCell + +from ..builder import NECKS + + +@NECKS.register_module() +class NASFCOS_FPN(nn.Module): + """FPN structure in NASFPN. + + Implementation of paper `NAS-FCOS: Fast Neural Architecture Search for + Object Detection `_ + + Args: + in_channels (List[int]): Number of input channels per scale. + out_channels (int): Number of output channels (used at each scale) + num_outs (int): Number of output scales. + start_level (int): Index of the start input backbone level used to + build the feature pyramid. Default: 0. + end_level (int): Index of the end input backbone level (exclusive) to + build the feature pyramid. Default: -1, which means the last level. + add_extra_convs (bool): It decides whether to add conv + layers on top of the original feature maps. Default to False. + If True, its actual mode is specified by `extra_convs_on_inputs`. + conv_cfg (dict): dictionary to construct and config conv layer. + norm_cfg (dict): dictionary to construct and config norm layer. + """ + + def __init__(self, + in_channels, + out_channels, + num_outs, + start_level=1, + end_level=-1, + add_extra_convs=False, + conv_cfg=None, + norm_cfg=None): + super(NASFCOS_FPN, self).__init__() + assert isinstance(in_channels, list) + self.in_channels = in_channels + self.out_channels = out_channels + self.num_ins = len(in_channels) + self.num_outs = num_outs + self.norm_cfg = norm_cfg + self.conv_cfg = conv_cfg + + if end_level == -1: + self.backbone_end_level = self.num_ins + assert num_outs >= self.num_ins - start_level + else: + self.backbone_end_level = end_level + assert end_level <= len(in_channels) + assert num_outs == end_level - start_level + self.start_level = start_level + self.end_level = end_level + self.add_extra_convs = add_extra_convs + + self.adapt_convs = nn.ModuleList() + for i in range(self.start_level, self.backbone_end_level): + adapt_conv = ConvModule( + in_channels[i], + out_channels, + 1, + stride=1, + padding=0, + bias=False, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU', inplace=False)) + self.adapt_convs.append(adapt_conv) + + # C2 is omitted according to the paper + extra_levels = num_outs - self.backbone_end_level + self.start_level + + def build_concat_cell(with_input1_conv, with_input2_conv): + cell_conv_cfg = dict( + kernel_size=1, padding=0, bias=False, groups=out_channels) + return ConcatCell( + in_channels=out_channels, + out_channels=out_channels, + with_out_conv=True, + out_conv_cfg=cell_conv_cfg, + out_norm_cfg=dict(type='BN'), + out_conv_order=('norm', 'act', 'conv'), + with_input1_conv=with_input1_conv, + with_input2_conv=with_input2_conv, + input_conv_cfg=conv_cfg, + input_norm_cfg=norm_cfg, + upsample_mode='nearest') + + # Denote c3=f0, c4=f1, c5=f2 for convince + self.fpn = nn.ModuleDict() + self.fpn['c22_1'] = build_concat_cell(True, True) + self.fpn['c22_2'] = build_concat_cell(True, True) + self.fpn['c32'] = build_concat_cell(True, False) + self.fpn['c02'] = build_concat_cell(True, False) + self.fpn['c42'] = build_concat_cell(True, True) + self.fpn['c36'] = build_concat_cell(True, True) + self.fpn['c61'] = build_concat_cell(True, True) # f9 + self.extra_downsamples = nn.ModuleList() + for i in range(extra_levels): + extra_act_cfg = None if i == 0 \ + else dict(type='ReLU', inplace=False) + self.extra_downsamples.append( + ConvModule( + out_channels, + out_channels, + 3, + stride=2, + padding=1, + act_cfg=extra_act_cfg, + order=('act', 'norm', 'conv'))) + + def forward(self, inputs): + """Forward function.""" + feats = [ + adapt_conv(inputs[i + self.start_level]) + for i, adapt_conv in enumerate(self.adapt_convs) + ] + + for (i, module_name) in enumerate(self.fpn): + idx_1, idx_2 = int(module_name[1]), int(module_name[2]) + res = self.fpn[module_name](feats[idx_1], feats[idx_2]) + feats.append(res) + + ret = [] + for (idx, input_idx) in zip([9, 8, 7], [1, 2, 3]): # add P3, P4, P5 + feats1, feats2 = feats[idx], feats[5] + feats2_resize = F.interpolate( + feats2, + size=feats1.size()[2:], + mode='bilinear', + align_corners=False) + + feats_sum = feats1 + feats2_resize + ret.append( + F.interpolate( + feats_sum, + size=inputs[input_idx].size()[2:], + mode='bilinear', + align_corners=False)) + + for submodule in self.extra_downsamples: + ret.append(submodule(ret[-1])) + + return tuple(ret) + + def init_weights(self): + """Initialize the weights of module.""" + for module in self.fpn.values(): + if hasattr(module, 'conv_out'): + caffe2_xavier_init(module.out_conv.conv) + + for modules in [ + self.adapt_convs.modules(), + self.extra_downsamples.modules() + ]: + for module in modules: + if isinstance(module, nn.Conv2d): + caffe2_xavier_init(module) diff --git a/detection/mmdet/models/necks/pafpn.py b/detection/mmdet/models/necks/pafpn.py new file mode 100644 index 0000000..d7c0b50 --- /dev/null +++ b/detection/mmdet/models/necks/pafpn.py @@ -0,0 +1,142 @@ +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import ConvModule +from mmcv.runner import auto_fp16 + +from ..builder import NECKS +from .fpn import FPN + + +@NECKS.register_module() +class PAFPN(FPN): + """Path Aggregation Network for Instance Segmentation. + + This is an implementation of the `PAFPN in Path Aggregation Network + `_. + + Args: + in_channels (List[int]): Number of input channels per scale. + out_channels (int): Number of output channels (used at each scale) + num_outs (int): Number of output scales. + start_level (int): Index of the start input backbone level used to + build the feature pyramid. Default: 0. + end_level (int): Index of the end input backbone level (exclusive) to + build the feature pyramid. Default: -1, which means the last level. + add_extra_convs (bool): Whether to add conv layers on top of the + original feature maps. Default: False. + extra_convs_on_inputs (bool): Whether to apply extra conv on + the original feature from the backbone. Default: False. + relu_before_extra_convs (bool): Whether to apply relu before the extra + conv. Default: False. + no_norm_on_lateral (bool): Whether to apply norm on lateral. + Default: False. + conv_cfg (dict): Config dict for convolution layer. Default: None. + norm_cfg (dict): Config dict for normalization layer. Default: None. + act_cfg (str): Config dict for activation layer in ConvModule. + Default: None. + """ + + def __init__(self, + in_channels, + out_channels, + num_outs, + start_level=0, + end_level=-1, + add_extra_convs=False, + extra_convs_on_inputs=True, + relu_before_extra_convs=False, + no_norm_on_lateral=False, + conv_cfg=None, + norm_cfg=None, + act_cfg=None): + super(PAFPN, + self).__init__(in_channels, out_channels, num_outs, start_level, + end_level, add_extra_convs, extra_convs_on_inputs, + relu_before_extra_convs, no_norm_on_lateral, + conv_cfg, norm_cfg, act_cfg) + # add extra bottom up pathway + self.downsample_convs = nn.ModuleList() + self.pafpn_convs = nn.ModuleList() + for i in range(self.start_level + 1, self.backbone_end_level): + d_conv = ConvModule( + out_channels, + out_channels, + 3, + stride=2, + padding=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + inplace=False) + pafpn_conv = ConvModule( + out_channels, + out_channels, + 3, + padding=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + inplace=False) + self.downsample_convs.append(d_conv) + self.pafpn_convs.append(pafpn_conv) + + @auto_fp16() + def forward(self, inputs): + """Forward function.""" + assert len(inputs) == len(self.in_channels) + + # build laterals + laterals = [ + lateral_conv(inputs[i + self.start_level]) + for i, lateral_conv in enumerate(self.lateral_convs) + ] + + # build top-down path + used_backbone_levels = len(laterals) + for i in range(used_backbone_levels - 1, 0, -1): + prev_shape = laterals[i - 1].shape[2:] + laterals[i - 1] += F.interpolate( + laterals[i], size=prev_shape, mode='nearest') + + # build outputs + # part 1: from original levels + inter_outs = [ + self.fpn_convs[i](laterals[i]) for i in range(used_backbone_levels) + ] + + # part 2: add bottom-up path + for i in range(0, used_backbone_levels - 1): + inter_outs[i + 1] += self.downsample_convs[i](inter_outs[i]) + + outs = [] + outs.append(inter_outs[0]) + outs.extend([ + self.pafpn_convs[i - 1](inter_outs[i]) + for i in range(1, used_backbone_levels) + ]) + + # part 3: add extra levels + if self.num_outs > len(outs): + # use max pool to get more levels on top of outputs + # (e.g., Faster R-CNN, Mask R-CNN) + if not self.add_extra_convs: + for i in range(self.num_outs - used_backbone_levels): + outs.append(F.max_pool2d(outs[-1], 1, stride=2)) + # add conv layers on top of original feature maps (RetinaNet) + else: + if self.add_extra_convs == 'on_input': + orig = inputs[self.backbone_end_level - 1] + outs.append(self.fpn_convs[used_backbone_levels](orig)) + elif self.add_extra_convs == 'on_lateral': + outs.append(self.fpn_convs[used_backbone_levels]( + laterals[-1])) + elif self.add_extra_convs == 'on_output': + outs.append(self.fpn_convs[used_backbone_levels](outs[-1])) + else: + raise NotImplementedError + for i in range(used_backbone_levels + 1, self.num_outs): + if self.relu_before_extra_convs: + outs.append(self.fpn_convs[i](F.relu(outs[-1]))) + else: + outs.append(self.fpn_convs[i](outs[-1])) + return tuple(outs) diff --git a/detection/mmdet/models/necks/rfp.py b/detection/mmdet/models/necks/rfp.py new file mode 100644 index 0000000..8a63e63 --- /dev/null +++ b/detection/mmdet/models/necks/rfp.py @@ -0,0 +1,128 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import constant_init, kaiming_init, xavier_init + +from ..builder import NECKS, build_backbone +from .fpn import FPN + + +class ASPP(nn.Module): + """ASPP (Atrous Spatial Pyramid Pooling) + + This is an implementation of the ASPP module used in DetectoRS + (https://arxiv.org/pdf/2006.02334.pdf) + + Args: + in_channels (int): Number of input channels. + out_channels (int): Number of channels produced by this module + dilations (tuple[int]): Dilations of the four branches. + Default: (1, 3, 6, 1) + """ + + def __init__(self, in_channels, out_channels, dilations=(1, 3, 6, 1)): + super().__init__() + assert dilations[-1] == 1 + self.aspp = nn.ModuleList() + for dilation in dilations: + kernel_size = 3 if dilation > 1 else 1 + padding = dilation if dilation > 1 else 0 + conv = nn.Conv2d( + in_channels, + out_channels, + kernel_size=kernel_size, + stride=1, + dilation=dilation, + padding=padding, + bias=True) + self.aspp.append(conv) + self.gap = nn.AdaptiveAvgPool2d(1) + self.init_weights() + + def init_weights(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + kaiming_init(m) + + def forward(self, x): + avg_x = self.gap(x) + out = [] + for aspp_idx in range(len(self.aspp)): + inp = avg_x if (aspp_idx == len(self.aspp) - 1) else x + out.append(F.relu_(self.aspp[aspp_idx](inp))) + out[-1] = out[-1].expand_as(out[-2]) + out = torch.cat(out, dim=1) + return out + + +@NECKS.register_module() +class RFP(FPN): + """RFP (Recursive Feature Pyramid) + + This is an implementation of RFP in `DetectoRS + `_. Different from standard FPN, the + input of RFP should be multi level features along with origin input image + of backbone. + + Args: + rfp_steps (int): Number of unrolled steps of RFP. + rfp_backbone (dict): Configuration of the backbone for RFP. + aspp_out_channels (int): Number of output channels of ASPP module. + aspp_dilations (tuple[int]): Dilation rates of four branches. + Default: (1, 3, 6, 1) + """ + + def __init__(self, + rfp_steps, + rfp_backbone, + aspp_out_channels, + aspp_dilations=(1, 3, 6, 1), + **kwargs): + super().__init__(**kwargs) + self.rfp_steps = rfp_steps + self.rfp_modules = nn.ModuleList() + for rfp_idx in range(1, rfp_steps): + rfp_module = build_backbone(rfp_backbone) + self.rfp_modules.append(rfp_module) + self.rfp_aspp = ASPP(self.out_channels, aspp_out_channels, + aspp_dilations) + self.rfp_weight = nn.Conv2d( + self.out_channels, + 1, + kernel_size=1, + stride=1, + padding=0, + bias=True) + + def init_weights(self): + # Avoid using super().init_weights(), which may alter the default + # initialization of the modules in self.rfp_modules that have missing + # keys in the pretrained checkpoint. + for convs in [self.lateral_convs, self.fpn_convs]: + for m in convs.modules(): + if isinstance(m, nn.Conv2d): + xavier_init(m, distribution='uniform') + for rfp_idx in range(self.rfp_steps - 1): + self.rfp_modules[rfp_idx].init_weights( + self.rfp_modules[rfp_idx].pretrained) + constant_init(self.rfp_weight, 0) + + def forward(self, inputs): + inputs = list(inputs) + assert len(inputs) == len(self.in_channels) + 1 # +1 for input image + img = inputs.pop(0) + # FPN forward + x = super().forward(tuple(inputs)) + for rfp_idx in range(self.rfp_steps - 1): + rfp_feats = [x[0]] + list( + self.rfp_aspp(x[i]) for i in range(1, len(x))) + x_idx = self.rfp_modules[rfp_idx].rfp_forward(img, rfp_feats) + # FPN forward + x_idx = super().forward(x_idx) + x_new = [] + for ft_idx in range(len(x_idx)): + add_weight = torch.sigmoid(self.rfp_weight(x_idx[ft_idx])) + x_new.append(add_weight * x_idx[ft_idx] + + (1 - add_weight) * x[ft_idx]) + x = x_new + return x diff --git a/detection/mmdet/models/necks/yolo_neck.py b/detection/mmdet/models/necks/yolo_neck.py new file mode 100644 index 0000000..c2f9b9e --- /dev/null +++ b/detection/mmdet/models/necks/yolo_neck.py @@ -0,0 +1,136 @@ +# Copyright (c) 2019 Western Digital Corporation or its affiliates. + +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import ConvModule + +from ..builder import NECKS + + +class DetectionBlock(nn.Module): + """Detection block in YOLO neck. + + Let out_channels = n, the DetectionBlock contains: + Six ConvLayers, 1 Conv2D Layer and 1 YoloLayer. + The first 6 ConvLayers are formed the following way: + 1x1xn, 3x3x2n, 1x1xn, 3x3x2n, 1x1xn, 3x3x2n. + The Conv2D layer is 1x1x255. + Some block will have branch after the fifth ConvLayer. + The input channel is arbitrary (in_channels) + + Args: + in_channels (int): The number of input channels. + out_channels (int): The number of output channels. + conv_cfg (dict): Config dict for convolution layer. Default: None. + norm_cfg (dict): Dictionary to construct and config norm layer. + Default: dict(type='BN', requires_grad=True) + act_cfg (dict): Config dict for activation layer. + Default: dict(type='LeakyReLU', negative_slope=0.1). + """ + + def __init__(self, + in_channels, + out_channels, + conv_cfg=None, + norm_cfg=dict(type='BN', requires_grad=True), + act_cfg=dict(type='LeakyReLU', negative_slope=0.1)): + super(DetectionBlock, self).__init__() + double_out_channels = out_channels * 2 + + # shortcut + cfg = dict(conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg) + self.conv1 = ConvModule(in_channels, out_channels, 1, **cfg) + self.conv2 = ConvModule( + out_channels, double_out_channels, 3, padding=1, **cfg) + self.conv3 = ConvModule(double_out_channels, out_channels, 1, **cfg) + self.conv4 = ConvModule( + out_channels, double_out_channels, 3, padding=1, **cfg) + self.conv5 = ConvModule(double_out_channels, out_channels, 1, **cfg) + + def forward(self, x): + tmp = self.conv1(x) + tmp = self.conv2(tmp) + tmp = self.conv3(tmp) + tmp = self.conv4(tmp) + out = self.conv5(tmp) + return out + + +@NECKS.register_module() +class YOLOV3Neck(nn.Module): + """The neck of YOLOV3. + + It can be treated as a simplified version of FPN. It + will take the result from Darknet backbone and do some upsampling and + concatenation. It will finally output the detection result. + + Note: + The input feats should be from top to bottom. + i.e., from high-lvl to low-lvl + But YOLOV3Neck will process them in reversed order. + i.e., from bottom (high-lvl) to top (low-lvl) + + Args: + num_scales (int): The number of scales / stages. + in_channels (int): The number of input channels. + out_channels (int): The number of output channels. + conv_cfg (dict): Config dict for convolution layer. Default: None. + norm_cfg (dict): Dictionary to construct and config norm layer. + Default: dict(type='BN', requires_grad=True) + act_cfg (dict): Config dict for activation layer. + Default: dict(type='LeakyReLU', negative_slope=0.1). + """ + + def __init__(self, + num_scales, + in_channels, + out_channels, + conv_cfg=None, + norm_cfg=dict(type='BN', requires_grad=True), + act_cfg=dict(type='LeakyReLU', negative_slope=0.1)): + super(YOLOV3Neck, self).__init__() + assert (num_scales == len(in_channels) == len(out_channels)) + self.num_scales = num_scales + self.in_channels = in_channels + self.out_channels = out_channels + + # shortcut + cfg = dict(conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg) + + # To support arbitrary scales, the code looks awful, but it works. + # Better solution is welcomed. + self.detect1 = DetectionBlock(in_channels[0], out_channels[0], **cfg) + for i in range(1, self.num_scales): + in_c, out_c = self.in_channels[i], self.out_channels[i] + self.add_module(f'conv{i}', ConvModule(in_c, out_c, 1, **cfg)) + # in_c + out_c : High-lvl feats will be cat with low-lvl feats + self.add_module(f'detect{i+1}', + DetectionBlock(in_c + out_c, out_c, **cfg)) + + def forward(self, feats): + assert len(feats) == self.num_scales + + # processed from bottom (high-lvl) to top (low-lvl) + outs = [] + out = self.detect1(feats[-1]) + outs.append(out) + + for i, x in enumerate(reversed(feats[:-1])): + conv = getattr(self, f'conv{i+1}') + tmp = conv(out) + + # Cat with low-lvl feats + tmp = F.interpolate(tmp, scale_factor=2) + tmp = torch.cat((tmp, x), 1) + + detect = getattr(self, f'detect{i+2}') + out = detect(tmp) + outs.append(out) + + return tuple(outs) + + def init_weights(self): + """Initialize the weights of module.""" + # init is done in ConvModule + pass diff --git a/detection/mmdet/models/roi_heads/__init__.py b/detection/mmdet/models/roi_heads/__init__.py new file mode 100644 index 0000000..ca0a38e --- /dev/null +++ b/detection/mmdet/models/roi_heads/__init__.py @@ -0,0 +1,34 @@ +from .base_roi_head import BaseRoIHead +from .bbox_heads import (BBoxHead, ConvFCBBoxHead, DoubleConvFCBBoxHead, + SCNetBBoxHead, Shared2FCBBoxHead, + Shared4Conv1FCBBoxHead) +from .cascade_roi_head import CascadeRoIHead +from .double_roi_head import DoubleHeadRoIHead +from .dynamic_roi_head import DynamicRoIHead +from .grid_roi_head import GridRoIHead +from .htc_roi_head import HybridTaskCascadeRoIHead +from .mask_heads import (CoarseMaskHead, FCNMaskHead, FeatureRelayHead, + FusedSemanticHead, GlobalContextHead, GridHead, + HTCMaskHead, MaskIoUHead, MaskPointHead, + SCNetMaskHead, SCNetSemanticHead) +from .mask_scoring_roi_head import MaskScoringRoIHead +from .pisa_roi_head import PISARoIHead +from .point_rend_roi_head import PointRendRoIHead +from .roi_extractors import SingleRoIExtractor +from .scnet_roi_head import SCNetRoIHead +from .shared_heads import ResLayer +from .sparse_roi_head import SparseRoIHead +from .standard_roi_head import StandardRoIHead +from .trident_roi_head import TridentRoIHead + +__all__ = [ + 'BaseRoIHead', 'CascadeRoIHead', 'DoubleHeadRoIHead', 'MaskScoringRoIHead', + 'HybridTaskCascadeRoIHead', 'GridRoIHead', 'ResLayer', 'BBoxHead', + 'ConvFCBBoxHead', 'Shared2FCBBoxHead', 'StandardRoIHead', + 'Shared4Conv1FCBBoxHead', 'DoubleConvFCBBoxHead', 'FCNMaskHead', + 'HTCMaskHead', 'FusedSemanticHead', 'GridHead', 'MaskIoUHead', + 'SingleRoIExtractor', 'PISARoIHead', 'PointRendRoIHead', 'MaskPointHead', + 'CoarseMaskHead', 'DynamicRoIHead', 'SparseRoIHead', 'TridentRoIHead', + 'SCNetRoIHead', 'SCNetMaskHead', 'SCNetSemanticHead', 'SCNetBBoxHead', + 'FeatureRelayHead', 'GlobalContextHead' +] diff --git a/detection/mmdet/models/roi_heads/base_roi_head.py b/detection/mmdet/models/roi_heads/base_roi_head.py new file mode 100644 index 0000000..2d61cc0 --- /dev/null +++ b/detection/mmdet/models/roi_heads/base_roi_head.py @@ -0,0 +1,103 @@ +from abc import ABCMeta, abstractmethod + +import torch.nn as nn + +from ..builder import build_shared_head + + +class BaseRoIHead(nn.Module, metaclass=ABCMeta): + """Base class for RoIHeads.""" + + def __init__(self, + bbox_roi_extractor=None, + bbox_head=None, + mask_roi_extractor=None, + mask_head=None, + shared_head=None, + train_cfg=None, + test_cfg=None): + super(BaseRoIHead, self).__init__() + self.train_cfg = train_cfg + self.test_cfg = test_cfg + if shared_head is not None: + self.shared_head = build_shared_head(shared_head) + + if bbox_head is not None: + self.init_bbox_head(bbox_roi_extractor, bbox_head) + + if mask_head is not None: + self.init_mask_head(mask_roi_extractor, mask_head) + + self.init_assigner_sampler() + + @property + def with_bbox(self): + """bool: whether the RoI head contains a `bbox_head`""" + return hasattr(self, 'bbox_head') and self.bbox_head is not None + + @property + def with_mask(self): + """bool: whether the RoI head contains a `mask_head`""" + return hasattr(self, 'mask_head') and self.mask_head is not None + + @property + def with_shared_head(self): + """bool: whether the RoI head contains a `shared_head`""" + return hasattr(self, 'shared_head') and self.shared_head is not None + + @abstractmethod + def init_weights(self, pretrained): + """Initialize the weights in head. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + pass + + @abstractmethod + def init_bbox_head(self): + """Initialize ``bbox_head``""" + pass + + @abstractmethod + def init_mask_head(self): + """Initialize ``mask_head``""" + pass + + @abstractmethod + def init_assigner_sampler(self): + """Initialize assigner and sampler.""" + pass + + @abstractmethod + def forward_train(self, + x, + img_meta, + proposal_list, + gt_bboxes, + gt_labels, + gt_bboxes_ignore=None, + gt_masks=None, + **kwargs): + """Forward function during training.""" + + async def async_simple_test(self, x, img_meta, **kwargs): + """Asynchronized test function.""" + raise NotImplementedError + + def simple_test(self, + x, + proposal_list, + img_meta, + proposals=None, + rescale=False, + **kwargs): + """Test without augmentation.""" + + def aug_test(self, x, proposal_list, img_metas, rescale=False, **kwargs): + """Test with augmentations. + + If rescale is False, then returned bboxes and masks will fit the scale + of imgs[0]. + """ diff --git a/detection/mmdet/models/roi_heads/bbox_heads/__init__.py b/detection/mmdet/models/roi_heads/bbox_heads/__init__.py new file mode 100644 index 0000000..bc5d29e --- /dev/null +++ b/detection/mmdet/models/roi_heads/bbox_heads/__init__.py @@ -0,0 +1,13 @@ +from .bbox_head import BBoxHead +from .convfc_bbox_head import (ConvFCBBoxHead, Shared2FCBBoxHead, + Shared4Conv1FCBBoxHead) +from .dii_head import DIIHead +from .double_bbox_head import DoubleConvFCBBoxHead +from .sabl_head import SABLHead +from .scnet_bbox_head import SCNetBBoxHead + +__all__ = [ + 'BBoxHead', 'ConvFCBBoxHead', 'Shared2FCBBoxHead', + 'Shared4Conv1FCBBoxHead', 'DoubleConvFCBBoxHead', 'SABLHead', 'DIIHead', + 'SCNetBBoxHead' +] diff --git a/detection/mmdet/models/roi_heads/bbox_heads/bbox_head.py b/detection/mmdet/models/roi_heads/bbox_heads/bbox_head.py new file mode 100644 index 0000000..408abef --- /dev/null +++ b/detection/mmdet/models/roi_heads/bbox_heads/bbox_head.py @@ -0,0 +1,483 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.runner import auto_fp16, force_fp32 +from torch.nn.modules.utils import _pair + +from mmdet.core import build_bbox_coder, multi_apply, multiclass_nms +from mmdet.models.builder import HEADS, build_loss +from mmdet.models.losses import accuracy + + +@HEADS.register_module() +class BBoxHead(nn.Module): + """Simplest RoI head, with only two fc layers for classification and + regression respectively.""" + + def __init__(self, + with_avg_pool=False, + with_cls=True, + with_reg=True, + roi_feat_size=7, + in_channels=256, + num_classes=80, + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + clip_border=True, + target_means=[0., 0., 0., 0.], + target_stds=[0.1, 0.1, 0.2, 0.2]), + reg_class_agnostic=False, + reg_decoded_bbox=False, + loss_cls=dict( + type='CrossEntropyLoss', + use_sigmoid=False, + loss_weight=1.0), + loss_bbox=dict( + type='SmoothL1Loss', beta=1.0, loss_weight=1.0)): + super(BBoxHead, self).__init__() + assert with_cls or with_reg + self.with_avg_pool = with_avg_pool + self.with_cls = with_cls + self.with_reg = with_reg + self.roi_feat_size = _pair(roi_feat_size) + self.roi_feat_area = self.roi_feat_size[0] * self.roi_feat_size[1] + self.in_channels = in_channels + self.num_classes = num_classes + self.reg_class_agnostic = reg_class_agnostic + self.reg_decoded_bbox = reg_decoded_bbox + self.fp16_enabled = False + + self.bbox_coder = build_bbox_coder(bbox_coder) + self.loss_cls = build_loss(loss_cls) + self.loss_bbox = build_loss(loss_bbox) + + in_channels = self.in_channels + if self.with_avg_pool: + self.avg_pool = nn.AvgPool2d(self.roi_feat_size) + else: + in_channels *= self.roi_feat_area + if self.with_cls: + # need to add background class + self.fc_cls = nn.Linear(in_channels, num_classes + 1) + if self.with_reg: + out_dim_reg = 4 if reg_class_agnostic else 4 * num_classes + self.fc_reg = nn.Linear(in_channels, out_dim_reg) + self.debug_imgs = None + + def init_weights(self): + # conv layers are already initialized by ConvModule + if self.with_cls: + nn.init.normal_(self.fc_cls.weight, 0, 0.01) + nn.init.constant_(self.fc_cls.bias, 0) + if self.with_reg: + nn.init.normal_(self.fc_reg.weight, 0, 0.001) + nn.init.constant_(self.fc_reg.bias, 0) + + @auto_fp16() + def forward(self, x): + if self.with_avg_pool: + x = self.avg_pool(x) + x = x.view(x.size(0), -1) + cls_score = self.fc_cls(x) if self.with_cls else None + bbox_pred = self.fc_reg(x) if self.with_reg else None + return cls_score, bbox_pred + + def _get_target_single(self, pos_bboxes, neg_bboxes, pos_gt_bboxes, + pos_gt_labels, cfg): + """Calculate the ground truth for proposals in the single image + according to the sampling results. + + Args: + pos_bboxes (Tensor): Contains all the positive boxes, + has shape (num_pos, 4), the last dimension 4 + represents [tl_x, tl_y, br_x, br_y]. + neg_bboxes (Tensor): Contains all the negative boxes, + has shape (num_neg, 4), the last dimension 4 + represents [tl_x, tl_y, br_x, br_y]. + pos_gt_bboxes (Tensor): Contains all the gt_boxes, + has shape (num_gt, 4), the last dimension 4 + represents [tl_x, tl_y, br_x, br_y]. + pos_gt_labels (Tensor): Contains all the gt_labels, + has shape (num_gt). + cfg (obj:`ConfigDict`): `train_cfg` of R-CNN. + + Returns: + Tuple[Tensor]: Ground truth for proposals + in a single image. Containing the following Tensors: + + - labels(Tensor): Gt_labels for all proposals, has + shape (num_proposals,). + - label_weights(Tensor): Labels_weights for all + proposals, has shape (num_proposals,). + - bbox_targets(Tensor):Regression target for all + proposals, has shape (num_proposals, 4), the + last dimension 4 represents [tl_x, tl_y, br_x, br_y]. + - bbox_weights(Tensor):Regression weights for all + proposals, has shape (num_proposals, 4). + """ + num_pos = pos_bboxes.size(0) + num_neg = neg_bboxes.size(0) + num_samples = num_pos + num_neg + + # original implementation uses new_zeros since BG are set to be 0 + # now use empty & fill because BG cat_id = num_classes, + # FG cat_id = [0, num_classes-1] + labels = pos_bboxes.new_full((num_samples, ), + self.num_classes, + dtype=torch.long) + label_weights = pos_bboxes.new_zeros(num_samples) + bbox_targets = pos_bboxes.new_zeros(num_samples, 4) + bbox_weights = pos_bboxes.new_zeros(num_samples, 4) + if num_pos > 0: + labels[:num_pos] = pos_gt_labels + pos_weight = 1.0 if cfg.pos_weight <= 0 else cfg.pos_weight + label_weights[:num_pos] = pos_weight + if not self.reg_decoded_bbox: + pos_bbox_targets = self.bbox_coder.encode( + pos_bboxes, pos_gt_bboxes) + else: + # When the regression loss (e.g. `IouLoss`, `GIouLoss`) + # is applied directly on the decoded bounding boxes, both + # the predicted boxes and regression targets should be with + # absolute coordinate format. + pos_bbox_targets = pos_gt_bboxes + bbox_targets[:num_pos, :] = pos_bbox_targets + bbox_weights[:num_pos, :] = 1 + if num_neg > 0: + label_weights[-num_neg:] = 1.0 + + return labels, label_weights, bbox_targets, bbox_weights + + def get_targets(self, + sampling_results, + gt_bboxes, + gt_labels, + rcnn_train_cfg, + concat=True): + """Calculate the ground truth for all samples in a batch according to + the sampling_results. + + Almost the same as the implementation in bbox_head, we passed + additional parameters pos_inds_list and neg_inds_list to + `_get_target_single` function. + + Args: + sampling_results (List[obj:SamplingResults]): Assign results of + all images in a batch after sampling. + gt_bboxes (list[Tensor]): Gt_bboxes of all images in a batch, + each tensor has shape (num_gt, 4), the last dimension 4 + represents [tl_x, tl_y, br_x, br_y]. + gt_labels (list[Tensor]): Gt_labels of all images in a batch, + each tensor has shape (num_gt,). + rcnn_train_cfg (obj:ConfigDict): `train_cfg` of RCNN. + concat (bool): Whether to concatenate the results of all + the images in a single batch. + + Returns: + Tuple[Tensor]: Ground truth for proposals in a single image. + Containing the following list of Tensors: + + - labels (list[Tensor],Tensor): Gt_labels for all + proposals in a batch, each tensor in list has + shape (num_proposals,) when `concat=False`, otherwise + just a single tensor has shape (num_all_proposals,). + - label_weights (list[Tensor]): Labels_weights for + all proposals in a batch, each tensor in list has + shape (num_proposals,) when `concat=False`, otherwise + just a single tensor has shape (num_all_proposals,). + - bbox_targets (list[Tensor],Tensor): Regression target + for all proposals in a batch, each tensor in list + has shape (num_proposals, 4) when `concat=False`, + otherwise just a single tensor has shape + (num_all_proposals, 4), the last dimension 4 represents + [tl_x, tl_y, br_x, br_y]. + - bbox_weights (list[tensor],Tensor): Regression weights for + all proposals in a batch, each tensor in list has shape + (num_proposals, 4) when `concat=False`, otherwise just a + single tensor has shape (num_all_proposals, 4). + """ + pos_bboxes_list = [res.pos_bboxes for res in sampling_results] + neg_bboxes_list = [res.neg_bboxes for res in sampling_results] + pos_gt_bboxes_list = [res.pos_gt_bboxes for res in sampling_results] + pos_gt_labels_list = [res.pos_gt_labels for res in sampling_results] + labels, label_weights, bbox_targets, bbox_weights = multi_apply( + self._get_target_single, + pos_bboxes_list, + neg_bboxes_list, + pos_gt_bboxes_list, + pos_gt_labels_list, + cfg=rcnn_train_cfg) + + if concat: + labels = torch.cat(labels, 0) + label_weights = torch.cat(label_weights, 0) + bbox_targets = torch.cat(bbox_targets, 0) + bbox_weights = torch.cat(bbox_weights, 0) + return labels, label_weights, bbox_targets, bbox_weights + + @force_fp32(apply_to=('cls_score', 'bbox_pred')) + def loss(self, + cls_score, + bbox_pred, + rois, + labels, + label_weights, + bbox_targets, + bbox_weights, + reduction_override=None): + losses = dict() + if cls_score is not None: + avg_factor = max(torch.sum(label_weights > 0).float().item(), 1.) + if cls_score.numel() > 0: + losses['loss_cls'] = self.loss_cls( + cls_score, + labels, + label_weights, + avg_factor=avg_factor, + reduction_override=reduction_override) + losses['acc'] = accuracy(cls_score, labels) + if bbox_pred is not None: + bg_class_ind = self.num_classes + # 0~self.num_classes-1 are FG, self.num_classes is BG + pos_inds = (labels >= 0) & (labels < bg_class_ind) + # do not perform bounding box regression for BG anymore. + if pos_inds.any(): + if self.reg_decoded_bbox: + # When the regression loss (e.g. `IouLoss`, + # `GIouLoss`, `DIouLoss`) is applied directly on + # the decoded bounding boxes, it decodes the + # already encoded coordinates to absolute format. + bbox_pred = self.bbox_coder.decode(rois[:, 1:], bbox_pred) + if self.reg_class_agnostic: + pos_bbox_pred = bbox_pred.view( + bbox_pred.size(0), 4)[pos_inds.type(torch.bool)] + else: + pos_bbox_pred = bbox_pred.view( + bbox_pred.size(0), -1, + 4)[pos_inds.type(torch.bool), + labels[pos_inds.type(torch.bool)]] + losses['loss_bbox'] = self.loss_bbox( + pos_bbox_pred, + bbox_targets[pos_inds.type(torch.bool)], + bbox_weights[pos_inds.type(torch.bool)], + avg_factor=bbox_targets.size(0), + reduction_override=reduction_override) + else: + losses['loss_bbox'] = bbox_pred[pos_inds].sum() + return losses + + @force_fp32(apply_to=('cls_score', 'bbox_pred')) + def get_bboxes(self, + rois, + cls_score, + bbox_pred, + img_shape, + scale_factor, + rescale=False, + cfg=None): + """Transform network output for a batch into bbox predictions. + + If the input rois has batch dimension, the function would be in + `batch_mode` and return is a tuple[list[Tensor], list[Tensor]], + otherwise, the return is a tuple[Tensor, Tensor]. + + Args: + rois (Tensor): Boxes to be transformed. Has shape (num_boxes, 5) + or (B, num_boxes, 5) + cls_score (list[Tensor] or Tensor): Box scores for + each scale level, each is a 4D-tensor, the channel number is + num_points * num_classes. + bbox_pred (Tensor, optional): Box energies / deltas for each scale + level, each is a 4D-tensor, the channel number is + num_classes * 4. + img_shape (Sequence[int] or torch.Tensor or Sequence[ + Sequence[int]], optional): Maximum bounds for boxes, specifies + (H, W, C) or (H, W). If rois shape is (B, num_boxes, 4), then + the max_shape should be a Sequence[Sequence[int]] + and the length of max_shape should also be B. + scale_factor (tuple[ndarray] or ndarray): Scale factor of the + image arange as (w_scale, h_scale, w_scale, h_scale). In + `batch_mode`, the scale_factor shape is tuple[ndarray]. + rescale (bool): If True, return boxes in original image space. + Default: False. + cfg (obj:`ConfigDict`): `test_cfg` of Bbox Head. Default: None + + Returns: + tuple[list[Tensor], list[Tensor]] or tuple[Tensor, Tensor]: + If the input has a batch dimension, the return value is + a tuple of the list. The first list contains the boxes of + the corresponding image in a batch, each tensor has the + shape (num_boxes, 5) and last dimension 5 represent + (tl_x, tl_y, br_x, br_y, score). Each Tensor in the second + list is the labels with shape (num_boxes, ). The length of + both lists should be equal to batch_size. Otherwise return + value is a tuple of two tensors, the first tensor is the + boxes with scores, the second tensor is the labels, both + have the same shape as the first case. + """ + if isinstance(cls_score, list): + cls_score = sum(cls_score) / float(len(cls_score)) + + scores = F.softmax( + cls_score, dim=-1) if cls_score is not None else None + + batch_mode = True + if rois.ndim == 2: + # e.g. AugTest, Cascade R-CNN, HTC, SCNet... + batch_mode = False + + # add batch dimension + if scores is not None: + scores = scores.unsqueeze(0) + if bbox_pred is not None: + bbox_pred = bbox_pred.unsqueeze(0) + rois = rois.unsqueeze(0) + + if bbox_pred is not None: + bboxes = self.bbox_coder.decode( + rois[..., 1:], bbox_pred, max_shape=img_shape) + else: + bboxes = rois[..., 1:].clone() + if img_shape is not None: + max_shape = bboxes.new_tensor(img_shape)[..., :2] + min_xy = bboxes.new_tensor(0) + max_xy = torch.cat( + [max_shape] * 2, dim=-1).flip(-1).unsqueeze(-2) + bboxes = torch.where(bboxes < min_xy, min_xy, bboxes) + bboxes = torch.where(bboxes > max_xy, max_xy, bboxes) + + if rescale and bboxes.size(-2) > 0: + if not isinstance(scale_factor, tuple): + scale_factor = tuple([scale_factor]) + # B, 1, bboxes.size(-1) + scale_factor = bboxes.new_tensor(scale_factor).unsqueeze(1).repeat( + 1, 1, + bboxes.size(-1) // 4) + bboxes /= scale_factor + + det_bboxes = [] + det_labels = [] + for (bbox, score) in zip(bboxes, scores): + if cfg is not None: + det_bbox, det_label = multiclass_nms(bbox, score, + cfg.score_thr, cfg.nms, + cfg.max_per_img) + else: + det_bbox, det_label = bbox, score + det_bboxes.append(det_bbox) + det_labels.append(det_label) + + if not batch_mode: + det_bboxes = det_bboxes[0] + det_labels = det_labels[0] + return det_bboxes, det_labels + + @force_fp32(apply_to=('bbox_preds', )) + def refine_bboxes(self, rois, labels, bbox_preds, pos_is_gts, img_metas): + """Refine bboxes during training. + + Args: + rois (Tensor): Shape (n*bs, 5), where n is image number per GPU, + and bs is the sampled RoIs per image. The first column is + the image id and the next 4 columns are x1, y1, x2, y2. + labels (Tensor): Shape (n*bs, ). + bbox_preds (Tensor): Shape (n*bs, 4) or (n*bs, 4*#class). + pos_is_gts (list[Tensor]): Flags indicating if each positive bbox + is a gt bbox. + img_metas (list[dict]): Meta info of each image. + + Returns: + list[Tensor]: Refined bboxes of each image in a mini-batch. + + Example: + >>> # xdoctest: +REQUIRES(module:kwarray) + >>> import kwarray + >>> import numpy as np + >>> from mmdet.core.bbox.demodata import random_boxes + >>> self = BBoxHead(reg_class_agnostic=True) + >>> n_roi = 2 + >>> n_img = 4 + >>> scale = 512 + >>> rng = np.random.RandomState(0) + >>> img_metas = [{'img_shape': (scale, scale)} + ... for _ in range(n_img)] + >>> # Create rois in the expected format + >>> roi_boxes = random_boxes(n_roi, scale=scale, rng=rng) + >>> img_ids = torch.randint(0, n_img, (n_roi,)) + >>> img_ids = img_ids.float() + >>> rois = torch.cat([img_ids[:, None], roi_boxes], dim=1) + >>> # Create other args + >>> labels = torch.randint(0, 2, (n_roi,)).long() + >>> bbox_preds = random_boxes(n_roi, scale=scale, rng=rng) + >>> # For each image, pretend random positive boxes are gts + >>> is_label_pos = (labels.numpy() > 0).astype(np.int) + >>> lbl_per_img = kwarray.group_items(is_label_pos, + ... img_ids.numpy()) + >>> pos_per_img = [sum(lbl_per_img.get(gid, [])) + ... for gid in range(n_img)] + >>> pos_is_gts = [ + >>> torch.randint(0, 2, (npos,)).byte().sort( + >>> descending=True)[0] + >>> for npos in pos_per_img + >>> ] + >>> bboxes_list = self.refine_bboxes(rois, labels, bbox_preds, + >>> pos_is_gts, img_metas) + >>> print(bboxes_list) + """ + img_ids = rois[:, 0].long().unique(sorted=True) + assert img_ids.numel() <= len(img_metas) + + bboxes_list = [] + for i in range(len(img_metas)): + inds = torch.nonzero( + rois[:, 0] == i, as_tuple=False).squeeze(dim=1) + num_rois = inds.numel() + + bboxes_ = rois[inds, 1:] + label_ = labels[inds] + bbox_pred_ = bbox_preds[inds] + img_meta_ = img_metas[i] + pos_is_gts_ = pos_is_gts[i] + + bboxes = self.regress_by_class(bboxes_, label_, bbox_pred_, + img_meta_) + + # filter gt bboxes + pos_keep = 1 - pos_is_gts_ + keep_inds = pos_is_gts_.new_ones(num_rois) + keep_inds[:len(pos_is_gts_)] = pos_keep + + bboxes_list.append(bboxes[keep_inds.type(torch.bool)]) + + return bboxes_list + + @force_fp32(apply_to=('bbox_pred', )) + def regress_by_class(self, rois, label, bbox_pred, img_meta): + """Regress the bbox for the predicted class. Used in Cascade R-CNN. + + Args: + rois (Tensor): shape (n, 4) or (n, 5) + label (Tensor): shape (n, ) + bbox_pred (Tensor): shape (n, 4*(#class)) or (n, 4) + img_meta (dict): Image meta info. + + Returns: + Tensor: Regressed bboxes, the same shape as input rois. + """ + assert rois.size(1) == 4 or rois.size(1) == 5, repr(rois.shape) + + if not self.reg_class_agnostic: + label = label * 4 + inds = torch.stack((label, label + 1, label + 2, label + 3), 1) + bbox_pred = torch.gather(bbox_pred, 1, inds) + assert bbox_pred.size(1) == 4 + + if rois.size(1) == 4: + new_rois = self.bbox_coder.decode( + rois, bbox_pred, max_shape=img_meta['img_shape']) + else: + bboxes = self.bbox_coder.decode( + rois[:, 1:], bbox_pred, max_shape=img_meta['img_shape']) + new_rois = torch.cat((rois[:, [0]], bboxes), dim=1) + + return new_rois diff --git a/detection/mmdet/models/roi_heads/bbox_heads/convfc_bbox_head.py b/detection/mmdet/models/roi_heads/bbox_heads/convfc_bbox_head.py new file mode 100644 index 0000000..0e86d2e --- /dev/null +++ b/detection/mmdet/models/roi_heads/bbox_heads/convfc_bbox_head.py @@ -0,0 +1,205 @@ +import torch.nn as nn +from mmcv.cnn import ConvModule + +from mmdet.models.builder import HEADS +from .bbox_head import BBoxHead + + +@HEADS.register_module() +class ConvFCBBoxHead(BBoxHead): + r"""More general bbox head, with shared conv and fc layers and two optional + separated branches. + + .. code-block:: none + + /-> cls convs -> cls fcs -> cls + shared convs -> shared fcs + \-> reg convs -> reg fcs -> reg + """ # noqa: W605 + + def __init__(self, + num_shared_convs=0, + num_shared_fcs=0, + num_cls_convs=0, + num_cls_fcs=0, + num_reg_convs=0, + num_reg_fcs=0, + conv_out_channels=256, + fc_out_channels=1024, + conv_cfg=None, + norm_cfg=None, + *args, + **kwargs): + super(ConvFCBBoxHead, self).__init__(*args, **kwargs) + assert (num_shared_convs + num_shared_fcs + num_cls_convs + + num_cls_fcs + num_reg_convs + num_reg_fcs > 0) + if num_cls_convs > 0 or num_reg_convs > 0: + assert num_shared_fcs == 0 + if not self.with_cls: + assert num_cls_convs == 0 and num_cls_fcs == 0 + if not self.with_reg: + assert num_reg_convs == 0 and num_reg_fcs == 0 + self.num_shared_convs = num_shared_convs + self.num_shared_fcs = num_shared_fcs + self.num_cls_convs = num_cls_convs + self.num_cls_fcs = num_cls_fcs + self.num_reg_convs = num_reg_convs + self.num_reg_fcs = num_reg_fcs + self.conv_out_channels = conv_out_channels + self.fc_out_channels = fc_out_channels + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + + # add shared convs and fcs + self.shared_convs, self.shared_fcs, last_layer_dim = \ + self._add_conv_fc_branch( + self.num_shared_convs, self.num_shared_fcs, self.in_channels, + True) + self.shared_out_channels = last_layer_dim + + # add cls specific branch + self.cls_convs, self.cls_fcs, self.cls_last_dim = \ + self._add_conv_fc_branch( + self.num_cls_convs, self.num_cls_fcs, self.shared_out_channels) + + # add reg specific branch + self.reg_convs, self.reg_fcs, self.reg_last_dim = \ + self._add_conv_fc_branch( + self.num_reg_convs, self.num_reg_fcs, self.shared_out_channels) + + if self.num_shared_fcs == 0 and not self.with_avg_pool: + if self.num_cls_fcs == 0: + self.cls_last_dim *= self.roi_feat_area + if self.num_reg_fcs == 0: + self.reg_last_dim *= self.roi_feat_area + + self.relu = nn.ReLU(inplace=True) + # reconstruct fc_cls and fc_reg since input channels are changed + if self.with_cls: + self.fc_cls = nn.Linear(self.cls_last_dim, self.num_classes + 1) + if self.with_reg: + out_dim_reg = (4 if self.reg_class_agnostic else 4 * + self.num_classes) + self.fc_reg = nn.Linear(self.reg_last_dim, out_dim_reg) + + def _add_conv_fc_branch(self, + num_branch_convs, + num_branch_fcs, + in_channels, + is_shared=False): + """Add shared or separable branch. + + convs -> avg pool (optional) -> fcs + """ + last_layer_dim = in_channels + # add branch specific conv layers + branch_convs = nn.ModuleList() + if num_branch_convs > 0: + for i in range(num_branch_convs): + conv_in_channels = ( + last_layer_dim if i == 0 else self.conv_out_channels) + branch_convs.append( + ConvModule( + conv_in_channels, + self.conv_out_channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg)) + last_layer_dim = self.conv_out_channels + # add branch specific fc layers + branch_fcs = nn.ModuleList() + if num_branch_fcs > 0: + # for shared branch, only consider self.with_avg_pool + # for separated branches, also consider self.num_shared_fcs + if (is_shared + or self.num_shared_fcs == 0) and not self.with_avg_pool: + last_layer_dim *= self.roi_feat_area + for i in range(num_branch_fcs): + fc_in_channels = ( + last_layer_dim if i == 0 else self.fc_out_channels) + branch_fcs.append( + nn.Linear(fc_in_channels, self.fc_out_channels)) + last_layer_dim = self.fc_out_channels + return branch_convs, branch_fcs, last_layer_dim + + def init_weights(self): + super(ConvFCBBoxHead, self).init_weights() + # conv layers are already initialized by ConvModule + for module_list in [self.shared_fcs, self.cls_fcs, self.reg_fcs]: + for m in module_list.modules(): + if isinstance(m, nn.Linear): + nn.init.xavier_uniform_(m.weight) + nn.init.constant_(m.bias, 0) + + def forward(self, x): + # shared part + if self.num_shared_convs > 0: + for conv in self.shared_convs: + x = conv(x) + + if self.num_shared_fcs > 0: + if self.with_avg_pool: + x = self.avg_pool(x) + + x = x.flatten(1) + + for fc in self.shared_fcs: + x = self.relu(fc(x)) + # separate branches + x_cls = x + x_reg = x + + for conv in self.cls_convs: + x_cls = conv(x_cls) + if x_cls.dim() > 2: + if self.with_avg_pool: + x_cls = self.avg_pool(x_cls) + x_cls = x_cls.flatten(1) + for fc in self.cls_fcs: + x_cls = self.relu(fc(x_cls)) + + for conv in self.reg_convs: + x_reg = conv(x_reg) + if x_reg.dim() > 2: + if self.with_avg_pool: + x_reg = self.avg_pool(x_reg) + x_reg = x_reg.flatten(1) + for fc in self.reg_fcs: + x_reg = self.relu(fc(x_reg)) + + cls_score = self.fc_cls(x_cls) if self.with_cls else None + bbox_pred = self.fc_reg(x_reg) if self.with_reg else None + return cls_score, bbox_pred + + +@HEADS.register_module() +class Shared2FCBBoxHead(ConvFCBBoxHead): + + def __init__(self, fc_out_channels=1024, *args, **kwargs): + super(Shared2FCBBoxHead, self).__init__( + num_shared_convs=0, + num_shared_fcs=2, + num_cls_convs=0, + num_cls_fcs=0, + num_reg_convs=0, + num_reg_fcs=0, + fc_out_channels=fc_out_channels, + *args, + **kwargs) + + +@HEADS.register_module() +class Shared4Conv1FCBBoxHead(ConvFCBBoxHead): + + def __init__(self, fc_out_channels=1024, *args, **kwargs): + super(Shared4Conv1FCBBoxHead, self).__init__( + num_shared_convs=4, + num_shared_fcs=1, + num_cls_convs=0, + num_cls_fcs=0, + num_reg_convs=0, + num_reg_fcs=0, + fc_out_channels=fc_out_channels, + *args, + **kwargs) diff --git a/detection/mmdet/models/roi_heads/bbox_heads/dii_head.py b/detection/mmdet/models/roi_heads/bbox_heads/dii_head.py new file mode 100644 index 0000000..8c970a7 --- /dev/null +++ b/detection/mmdet/models/roi_heads/bbox_heads/dii_head.py @@ -0,0 +1,415 @@ +import torch +import torch.nn as nn +from mmcv.cnn import (bias_init_with_prob, build_activation_layer, + build_norm_layer) +from mmcv.runner import auto_fp16, force_fp32 + +from mmdet.core import multi_apply +from mmdet.models.builder import HEADS, build_loss +from mmdet.models.dense_heads.atss_head import reduce_mean +from mmdet.models.losses import accuracy +from mmdet.models.utils import FFN, MultiheadAttention, build_transformer +from .bbox_head import BBoxHead + + +@HEADS.register_module() +class DIIHead(BBoxHead): + r"""Dynamic Instance Interactive Head for `Sparse R-CNN: End-to-End Object + Detection with Learnable Proposals `_ + + Args: + num_classes (int): Number of class in dataset. + Defaults to 80. + num_ffn_fcs (int): The number of fully-connected + layers in FFNs. Defaults to 2. + num_heads (int): The hidden dimension of FFNs. + Defaults to 8. + num_cls_fcs (int): The number of fully-connected + layers in classification subnet. Defaults to 1. + num_reg_fcs (int): The number of fully-connected + layers in regression subnet. Defaults to 3. + feedforward_channels (int): The hidden dimension + of FFNs. Defaults to 2048 + in_channels (int): Hidden_channels of MultiheadAttention. + Defaults to 256. + dropout (float): Probability of drop the channel. + Defaults to 0.0 + ffn_act_cfg (dict): The activation config for FFNs. + dynamic_conv_cfg (dict): The convolution config + for DynamicConv. + loss_iou (dict): The config for iou or giou loss. + + """ + + def __init__(self, + num_classes=80, + num_ffn_fcs=2, + num_heads=8, + num_cls_fcs=1, + num_reg_fcs=3, + feedforward_channels=2048, + in_channels=256, + dropout=0.0, + ffn_act_cfg=dict(type='ReLU', inplace=True), + dynamic_conv_cfg=dict( + type='DynamicConv', + in_channels=256, + feat_channels=64, + out_channels=256, + input_feat_shape=7, + act_cfg=dict(type='ReLU', inplace=True), + norm_cfg=dict(type='LN')), + loss_iou=dict(type='GIoULoss', loss_weight=2.0), + **kwargs): + super(DIIHead, self).__init__( + num_classes=num_classes, + reg_decoded_bbox=True, + reg_class_agnostic=True, + **kwargs) + self.loss_iou = build_loss(loss_iou) + self.in_channels = in_channels + self.fp16_enabled = False + self.attention = MultiheadAttention(in_channels, num_heads, dropout) + self.attention_norm = build_norm_layer(dict(type='LN'), in_channels)[1] + + self.instance_interactive_conv = build_transformer(dynamic_conv_cfg) + self.instance_interactive_conv_dropout = nn.Dropout(dropout) + self.instance_interactive_conv_norm = build_norm_layer( + dict(type='LN'), in_channels)[1] + + self.ffn = FFN( + in_channels, + feedforward_channels, + num_ffn_fcs, + act_cfg=ffn_act_cfg, + dropout=dropout) + self.ffn_norm = build_norm_layer(dict(type='LN'), in_channels)[1] + + self.cls_fcs = nn.ModuleList() + for _ in range(num_cls_fcs): + self.cls_fcs.append( + nn.Linear(in_channels, in_channels, bias=False)) + self.cls_fcs.append( + build_norm_layer(dict(type='LN'), in_channels)[1]) + self.cls_fcs.append( + build_activation_layer(dict(type='ReLU', inplace=True))) + + # over load the self.fc_cls in BBoxHead + if self.loss_cls.use_sigmoid: + self.fc_cls = nn.Linear(in_channels, self.num_classes) + else: + self.fc_cls = nn.Linear(in_channels, self.num_classes + 1) + + self.reg_fcs = nn.ModuleList() + for _ in range(num_reg_fcs): + self.reg_fcs.append( + nn.Linear(in_channels, in_channels, bias=False)) + self.reg_fcs.append( + build_norm_layer(dict(type='LN'), in_channels)[1]) + self.reg_fcs.append( + build_activation_layer(dict(type='ReLU', inplace=True))) + # over load the self.fc_cls in BBoxHead + self.fc_reg = nn.Linear(in_channels, 4) + + assert self.reg_class_agnostic, 'DIIHead only ' \ + 'suppport `reg_class_agnostic=True` ' + assert self.reg_decoded_bbox, 'DIIHead only ' \ + 'suppport `reg_decoded_bbox=True`' + + def init_weights(self): + """Use xavier initialization for all weight parameter and set + classification head bias as a specific value when use focal loss.""" + for p in self.parameters(): + if p.dim() > 1: + nn.init.xavier_uniform_(p) + else: + # adopt the default initialization for + # the weight and bias of the layer norm + pass + if self.loss_cls.use_sigmoid: + bias_init = bias_init_with_prob(0.01) + nn.init.constant_(self.fc_cls.bias, bias_init) + + @auto_fp16() + def forward(self, roi_feat, proposal_feat): + """Forward function of Dynamic Instance Interactive Head. + + Args: + roi_feat (Tensor): Roi-pooling features with shape + (batch_size*num_proposals, feature_dimensions, + pooling_h , pooling_w). + proposal_feat (Tensor): Intermediate feature get from + diihead in last stage, has shape + (batch_size, num_proposals, feature_dimensions) + + Returns: + tuple[Tensor]: Usually a tuple of classification scores + and bbox prediction and a intermediate feature. + + - cls_scores (Tensor): Classification scores for + all proposals, has shape + (batch_size, num_proposals, num_classes). + - bbox_preds (Tensor): Box energies / deltas for + all proposals, has shape + (batch_size, num_proposals, 4). + - obj_feat (Tensor): Object feature before classification + and regression subnet, has shape + (batch_size, num_proposal, feature_dimensions). + """ + N, num_proposals = proposal_feat.shape[:2] + + # Self attention + proposal_feat = proposal_feat.permute(1, 0, 2) + proposal_feat = self.attention_norm(self.attention(proposal_feat)) + + # instance interactive + proposal_feat = proposal_feat.permute(1, 0, + 2).reshape(-1, self.in_channels) + proposal_feat_iic = self.instance_interactive_conv( + proposal_feat, roi_feat) + proposal_feat = proposal_feat + self.instance_interactive_conv_dropout( + proposal_feat_iic) + obj_feat = self.instance_interactive_conv_norm(proposal_feat) + + # FFN + obj_feat = self.ffn_norm(self.ffn(obj_feat)) + + cls_feat = obj_feat + reg_feat = obj_feat + + for cls_layer in self.cls_fcs: + cls_feat = cls_layer(cls_feat) + for reg_layer in self.reg_fcs: + reg_feat = reg_layer(reg_feat) + + cls_score = self.fc_cls(cls_feat).view(N, num_proposals, -1) + bbox_delta = self.fc_reg(reg_feat).view(N, num_proposals, -1) + + return cls_score, bbox_delta, obj_feat.view(N, num_proposals, -1) + + @force_fp32(apply_to=('cls_score', 'bbox_pred')) + def loss(self, + cls_score, + bbox_pred, + labels, + label_weights, + bbox_targets, + bbox_weights, + imgs_whwh=None, + reduction_override=None, + **kwargs): + """"Loss function of DIIHead, get loss of all images. + + Args: + cls_score (Tensor): Classification prediction + results of all class, has shape + (batch_size * num_proposals_single_image, num_classes) + bbox_pred (Tensor): Regression prediction results, + has shape + (batch_size * num_proposals_single_image, 4), the last + dimension 4 represents [tl_x, tl_y, br_x, br_y]. + labels (Tensor): Label of each proposals, has shape + (batch_size * num_proposals_single_image + label_weights (Tensor): Classification loss + weight of each proposals, has shape + (batch_size * num_proposals_single_image + bbox_targets (Tensor): Regression targets of each + proposals, has shape + (batch_size * num_proposals_single_image, 4), + the last dimension 4 represents + [tl_x, tl_y, br_x, br_y]. + bbox_weights (Tensor): Regression loss weight of each + proposals's coordinate, has shape + (batch_size * num_proposals_single_image, 4), + imgs_whwh (Tensor): imgs_whwh (Tensor): Tensor with\ + shape (batch_size, num_proposals, 4), the last + dimension means + [img_width,img_height, img_width, img_height]. + reduction_override (str, optional): The reduction + method used to override the original reduction + method of the loss. Options are "none", + "mean" and "sum". Defaults to None, + + Returns: + dict[str, Tensor]: Dictionary of loss components + """ + losses = dict() + bg_class_ind = self.num_classes + # note in spare rcnn num_gt == num_pos + pos_inds = (labels >= 0) & (labels < bg_class_ind) + num_pos = pos_inds.sum().float() + avg_factor = reduce_mean(num_pos) + if cls_score is not None: + if cls_score.numel() > 0: + losses['loss_cls'] = self.loss_cls( + cls_score, + labels, + label_weights, + avg_factor=avg_factor, + reduction_override=reduction_override) + losses['pos_acc'] = accuracy(cls_score[pos_inds], + labels[pos_inds]) + if bbox_pred is not None: + # 0~self.num_classes-1 are FG, self.num_classes is BG + # do not perform bounding box regression for BG anymore. + if pos_inds.any(): + pos_bbox_pred = bbox_pred.reshape(bbox_pred.size(0), + 4)[pos_inds.type(torch.bool)] + imgs_whwh = imgs_whwh.reshape(bbox_pred.size(0), + 4)[pos_inds.type(torch.bool)] + losses['loss_bbox'] = self.loss_bbox( + pos_bbox_pred / imgs_whwh, + bbox_targets[pos_inds.type(torch.bool)] / imgs_whwh, + bbox_weights[pos_inds.type(torch.bool)], + avg_factor=avg_factor) + losses['loss_iou'] = self.loss_iou( + pos_bbox_pred, + bbox_targets[pos_inds.type(torch.bool)], + bbox_weights[pos_inds.type(torch.bool)], + avg_factor=avg_factor) + else: + losses['loss_bbox'] = bbox_pred.sum() * 0 + losses['loss_iou'] = bbox_pred.sum() * 0 + return losses + + def _get_target_single(self, pos_inds, neg_inds, pos_bboxes, neg_bboxes, + pos_gt_bboxes, pos_gt_labels, cfg): + """Calculate the ground truth for proposals in the single image + according to the sampling results. + + Almost the same as the implementation in `bbox_head`, + we add pos_inds and neg_inds to select positive and + negative samples instead of selecting the first num_pos + as positive samples. + + Args: + pos_inds (Tensor): The length is equal to the + positive sample numbers contain all index + of the positive sample in the origin proposal set. + neg_inds (Tensor): The length is equal to the + negative sample numbers contain all index + of the negative sample in the origin proposal set. + pos_bboxes (Tensor): Contains all the positive boxes, + has shape (num_pos, 4), the last dimension 4 + represents [tl_x, tl_y, br_x, br_y]. + neg_bboxes (Tensor): Contains all the negative boxes, + has shape (num_neg, 4), the last dimension 4 + represents [tl_x, tl_y, br_x, br_y]. + pos_gt_bboxes (Tensor): Contains all the gt_boxes, + has shape (num_gt, 4), the last dimension 4 + represents [tl_x, tl_y, br_x, br_y]. + pos_gt_labels (Tensor): Contains all the gt_labels, + has shape (num_gt). + cfg (obj:`ConfigDict`): `train_cfg` of R-CNN. + + Returns: + Tuple[Tensor]: Ground truth for proposals in a single image. + Containing the following Tensors: + + - labels(Tensor): Gt_labels for all proposals, has + shape (num_proposals,). + - label_weights(Tensor): Labels_weights for all proposals, has + shape (num_proposals,). + - bbox_targets(Tensor):Regression target for all proposals, has + shape (num_proposals, 4), the last dimension 4 + represents [tl_x, tl_y, br_x, br_y]. + - bbox_weights(Tensor):Regression weights for all proposals, + has shape (num_proposals, 4). + """ + num_pos = pos_bboxes.size(0) + num_neg = neg_bboxes.size(0) + num_samples = num_pos + num_neg + + # original implementation uses new_zeros since BG are set to be 0 + # now use empty & fill because BG cat_id = num_classes, + # FG cat_id = [0, num_classes-1] + labels = pos_bboxes.new_full((num_samples, ), + self.num_classes, + dtype=torch.long) + label_weights = pos_bboxes.new_zeros(num_samples) + bbox_targets = pos_bboxes.new_zeros(num_samples, 4) + bbox_weights = pos_bboxes.new_zeros(num_samples, 4) + if num_pos > 0: + labels[pos_inds] = pos_gt_labels + pos_weight = 1.0 if cfg.pos_weight <= 0 else cfg.pos_weight + label_weights[pos_inds] = pos_weight + if not self.reg_decoded_bbox: + pos_bbox_targets = self.bbox_coder.encode( + pos_bboxes, pos_gt_bboxes) + else: + pos_bbox_targets = pos_gt_bboxes + bbox_targets[pos_inds, :] = pos_bbox_targets + bbox_weights[pos_inds, :] = 1 + if num_neg > 0: + label_weights[neg_inds] = 1.0 + + return labels, label_weights, bbox_targets, bbox_weights + + def get_targets(self, + sampling_results, + gt_bboxes, + gt_labels, + rcnn_train_cfg, + concat=True): + """Calculate the ground truth for all samples in a batch according to + the sampling_results. + + Almost the same as the implementation in bbox_head, we passed + additional parameters pos_inds_list and neg_inds_list to + `_get_target_single` function. + + Args: + sampling_results (List[obj:SamplingResults]): Assign results of + all images in a batch after sampling. + gt_bboxes (list[Tensor]): Gt_bboxes of all images in a batch, + each tensor has shape (num_gt, 4), the last dimension 4 + represents [tl_x, tl_y, br_x, br_y]. + gt_labels (list[Tensor]): Gt_labels of all images in a batch, + each tensor has shape (num_gt,). + rcnn_train_cfg (obj:`ConfigDict`): `train_cfg` of RCNN. + concat (bool): Whether to concatenate the results of all + the images in a single batch. + + Returns: + Tuple[Tensor]: Ground truth for proposals in a single image. + Containing the following list of Tensors: + + - labels (list[Tensor],Tensor): Gt_labels for all + proposals in a batch, each tensor in list has + shape (num_proposals,) when `concat=False`, otherwise just + a single tensor has shape (num_all_proposals,). + - label_weights (list[Tensor]): Labels_weights for + all proposals in a batch, each tensor in list has shape + (num_proposals,) when `concat=False`, otherwise just a + single tensor has shape (num_all_proposals,). + - bbox_targets (list[Tensor],Tensor): Regression target + for all proposals in a batch, each tensor in list has + shape (num_proposals, 4) when `concat=False`, otherwise + just a single tensor has shape (num_all_proposals, 4), + the last dimension 4 represents [tl_x, tl_y, br_x, br_y]. + - bbox_weights (list[tensor],Tensor): Regression weights for + all proposals in a batch, each tensor in list has shape + (num_proposals, 4) when `concat=False`, otherwise just a + single tensor has shape (num_all_proposals, 4). + """ + pos_inds_list = [res.pos_inds for res in sampling_results] + neg_inds_list = [res.neg_inds for res in sampling_results] + pos_bboxes_list = [res.pos_bboxes for res in sampling_results] + neg_bboxes_list = [res.neg_bboxes for res in sampling_results] + pos_gt_bboxes_list = [res.pos_gt_bboxes for res in sampling_results] + pos_gt_labels_list = [res.pos_gt_labels for res in sampling_results] + labels, label_weights, bbox_targets, bbox_weights = multi_apply( + self._get_target_single, + pos_inds_list, + neg_inds_list, + pos_bboxes_list, + neg_bboxes_list, + pos_gt_bboxes_list, + pos_gt_labels_list, + cfg=rcnn_train_cfg) + if concat: + labels = torch.cat(labels, 0) + label_weights = torch.cat(label_weights, 0) + bbox_targets = torch.cat(bbox_targets, 0) + bbox_weights = torch.cat(bbox_weights, 0) + return labels, label_weights, bbox_targets, bbox_weights diff --git a/detection/mmdet/models/roi_heads/bbox_heads/double_bbox_head.py b/detection/mmdet/models/roi_heads/bbox_heads/double_bbox_head.py new file mode 100644 index 0000000..6c154cb --- /dev/null +++ b/detection/mmdet/models/roi_heads/bbox_heads/double_bbox_head.py @@ -0,0 +1,172 @@ +import torch.nn as nn +from mmcv.cnn import ConvModule, normal_init, xavier_init + +from mmdet.models.backbones.resnet import Bottleneck +from mmdet.models.builder import HEADS +from .bbox_head import BBoxHead + + +class BasicResBlock(nn.Module): + """Basic residual block. + + This block is a little different from the block in the ResNet backbone. + The kernel size of conv1 is 1 in this block while 3 in ResNet BasicBlock. + + Args: + in_channels (int): Channels of the input feature map. + out_channels (int): Channels of the output feature map. + conv_cfg (dict): The config dict for convolution layers. + norm_cfg (dict): The config dict for normalization layers. + """ + + def __init__(self, + in_channels, + out_channels, + conv_cfg=None, + norm_cfg=dict(type='BN')): + super(BasicResBlock, self).__init__() + + # main path + self.conv1 = ConvModule( + in_channels, + in_channels, + kernel_size=3, + padding=1, + bias=False, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg) + self.conv2 = ConvModule( + in_channels, + out_channels, + kernel_size=1, + bias=False, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=None) + + # identity path + self.conv_identity = ConvModule( + in_channels, + out_channels, + kernel_size=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=None) + + self.relu = nn.ReLU(inplace=True) + + def forward(self, x): + identity = x + + x = self.conv1(x) + x = self.conv2(x) + + identity = self.conv_identity(identity) + out = x + identity + + out = self.relu(out) + return out + + +@HEADS.register_module() +class DoubleConvFCBBoxHead(BBoxHead): + r"""Bbox head used in Double-Head R-CNN + + .. code-block:: none + + /-> cls + /-> shared convs -> + \-> reg + roi features + /-> cls + \-> shared fc -> + \-> reg + """ # noqa: W605 + + def __init__(self, + num_convs=0, + num_fcs=0, + conv_out_channels=1024, + fc_out_channels=1024, + conv_cfg=None, + norm_cfg=dict(type='BN'), + **kwargs): + kwargs.setdefault('with_avg_pool', True) + super(DoubleConvFCBBoxHead, self).__init__(**kwargs) + assert self.with_avg_pool + assert num_convs > 0 + assert num_fcs > 0 + self.num_convs = num_convs + self.num_fcs = num_fcs + self.conv_out_channels = conv_out_channels + self.fc_out_channels = fc_out_channels + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + + # increase the channel of input features + self.res_block = BasicResBlock(self.in_channels, + self.conv_out_channels) + + # add conv heads + self.conv_branch = self._add_conv_branch() + # add fc heads + self.fc_branch = self._add_fc_branch() + + out_dim_reg = 4 if self.reg_class_agnostic else 4 * self.num_classes + self.fc_reg = nn.Linear(self.conv_out_channels, out_dim_reg) + + self.fc_cls = nn.Linear(self.fc_out_channels, self.num_classes + 1) + self.relu = nn.ReLU(inplace=True) + + def _add_conv_branch(self): + """Add the fc branch which consists of a sequential of conv layers.""" + branch_convs = nn.ModuleList() + for i in range(self.num_convs): + branch_convs.append( + Bottleneck( + inplanes=self.conv_out_channels, + planes=self.conv_out_channels // 4, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg)) + return branch_convs + + def _add_fc_branch(self): + """Add the fc branch which consists of a sequential of fc layers.""" + branch_fcs = nn.ModuleList() + for i in range(self.num_fcs): + fc_in_channels = ( + self.in_channels * + self.roi_feat_area if i == 0 else self.fc_out_channels) + branch_fcs.append(nn.Linear(fc_in_channels, self.fc_out_channels)) + return branch_fcs + + def init_weights(self): + # conv layers are already initialized by ConvModule + normal_init(self.fc_cls, std=0.01) + normal_init(self.fc_reg, std=0.001) + + for m in self.fc_branch.modules(): + if isinstance(m, nn.Linear): + xavier_init(m, distribution='uniform') + + def forward(self, x_cls, x_reg): + # conv head + x_conv = self.res_block(x_reg) + + for conv in self.conv_branch: + x_conv = conv(x_conv) + + if self.with_avg_pool: + x_conv = self.avg_pool(x_conv) + + x_conv = x_conv.view(x_conv.size(0), -1) + bbox_pred = self.fc_reg(x_conv) + + # fc head + x_fc = x_cls.view(x_cls.size(0), -1) + for fc in self.fc_branch: + x_fc = self.relu(fc(x_fc)) + + cls_score = self.fc_cls(x_fc) + + return cls_score, bbox_pred diff --git a/detection/mmdet/models/roi_heads/bbox_heads/sabl_head.py b/detection/mmdet/models/roi_heads/bbox_heads/sabl_head.py new file mode 100644 index 0000000..5153996 --- /dev/null +++ b/detection/mmdet/models/roi_heads/bbox_heads/sabl_head.py @@ -0,0 +1,572 @@ +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import ConvModule, kaiming_init, normal_init, xavier_init +from mmcv.runner import force_fp32 + +from mmdet.core import build_bbox_coder, multi_apply, multiclass_nms +from mmdet.models.builder import HEADS, build_loss +from mmdet.models.losses import accuracy + + +@HEADS.register_module() +class SABLHead(nn.Module): + """Side-Aware Boundary Localization (SABL) for RoI-Head. + + Side-Aware features are extracted by conv layers + with an attention mechanism. + Boundary Localization with Bucketing and Bucketing Guided Rescoring + are implemented in BucketingBBoxCoder. + + Please refer to https://arxiv.org/abs/1912.04260 for more details. + + Args: + cls_in_channels (int): Input channels of cls RoI feature. \ + Defaults to 256. + reg_in_channels (int): Input channels of reg RoI feature. \ + Defaults to 256. + roi_feat_size (int): Size of RoI features. Defaults to 7. + reg_feat_up_ratio (int): Upsample ratio of reg features. \ + Defaults to 2. + reg_pre_kernel (int): Kernel of 2D conv layers before \ + attention pooling. Defaults to 3. + reg_post_kernel (int): Kernel of 1D conv layers after \ + attention pooling. Defaults to 3. + reg_pre_num (int): Number of pre convs. Defaults to 2. + reg_post_num (int): Number of post convs. Defaults to 1. + num_classes (int): Number of classes in dataset. Defaults to 80. + cls_out_channels (int): Hidden channels in cls fcs. Defaults to 1024. + reg_offset_out_channels (int): Hidden and output channel \ + of reg offset branch. Defaults to 256. + reg_cls_out_channels (int): Hidden and output channel \ + of reg cls branch. Defaults to 256. + num_cls_fcs (int): Number of fcs for cls branch. Defaults to 1. + num_reg_fcs (int): Number of fcs for reg branch.. Defaults to 0. + reg_class_agnostic (bool): Class agnostic regresion or not. \ + Defaults to True. + norm_cfg (dict): Config of norm layers. Defaults to None. + bbox_coder (dict): Config of bbox coder. Defaults 'BucketingBBoxCoder'. + loss_cls (dict): Config of classification loss. + loss_bbox_cls (dict): Config of classification loss for bbox branch. + loss_bbox_reg (dict): Config of regression loss for bbox branch. + """ + + def __init__(self, + num_classes, + cls_in_channels=256, + reg_in_channels=256, + roi_feat_size=7, + reg_feat_up_ratio=2, + reg_pre_kernel=3, + reg_post_kernel=3, + reg_pre_num=2, + reg_post_num=1, + cls_out_channels=1024, + reg_offset_out_channels=256, + reg_cls_out_channels=256, + num_cls_fcs=1, + num_reg_fcs=0, + reg_class_agnostic=True, + norm_cfg=None, + bbox_coder=dict( + type='BucketingBBoxCoder', + num_buckets=14, + scale_factor=1.7), + loss_cls=dict( + type='CrossEntropyLoss', + use_sigmoid=False, + loss_weight=1.0), + loss_bbox_cls=dict( + type='CrossEntropyLoss', + use_sigmoid=True, + loss_weight=1.0), + loss_bbox_reg=dict( + type='SmoothL1Loss', beta=0.1, loss_weight=1.0)): + super(SABLHead, self).__init__() + self.cls_in_channels = cls_in_channels + self.reg_in_channels = reg_in_channels + self.roi_feat_size = roi_feat_size + self.reg_feat_up_ratio = int(reg_feat_up_ratio) + self.num_buckets = bbox_coder['num_buckets'] + assert self.reg_feat_up_ratio // 2 >= 1 + self.up_reg_feat_size = roi_feat_size * self.reg_feat_up_ratio + assert self.up_reg_feat_size == bbox_coder['num_buckets'] + self.reg_pre_kernel = reg_pre_kernel + self.reg_post_kernel = reg_post_kernel + self.reg_pre_num = reg_pre_num + self.reg_post_num = reg_post_num + self.num_classes = num_classes + self.cls_out_channels = cls_out_channels + self.reg_offset_out_channels = reg_offset_out_channels + self.reg_cls_out_channels = reg_cls_out_channels + self.num_cls_fcs = num_cls_fcs + self.num_reg_fcs = num_reg_fcs + self.reg_class_agnostic = reg_class_agnostic + assert self.reg_class_agnostic + self.norm_cfg = norm_cfg + + self.bbox_coder = build_bbox_coder(bbox_coder) + self.loss_cls = build_loss(loss_cls) + self.loss_bbox_cls = build_loss(loss_bbox_cls) + self.loss_bbox_reg = build_loss(loss_bbox_reg) + + self.cls_fcs = self._add_fc_branch(self.num_cls_fcs, + self.cls_in_channels, + self.roi_feat_size, + self.cls_out_channels) + + self.side_num = int(np.ceil(self.num_buckets / 2)) + + if self.reg_feat_up_ratio > 1: + self.upsample_x = nn.ConvTranspose1d( + reg_in_channels, + reg_in_channels, + self.reg_feat_up_ratio, + stride=self.reg_feat_up_ratio) + self.upsample_y = nn.ConvTranspose1d( + reg_in_channels, + reg_in_channels, + self.reg_feat_up_ratio, + stride=self.reg_feat_up_ratio) + + self.reg_pre_convs = nn.ModuleList() + for i in range(self.reg_pre_num): + reg_pre_conv = ConvModule( + reg_in_channels, + reg_in_channels, + kernel_size=reg_pre_kernel, + padding=reg_pre_kernel // 2, + norm_cfg=norm_cfg, + act_cfg=dict(type='ReLU')) + self.reg_pre_convs.append(reg_pre_conv) + + self.reg_post_conv_xs = nn.ModuleList() + for i in range(self.reg_post_num): + reg_post_conv_x = ConvModule( + reg_in_channels, + reg_in_channels, + kernel_size=(1, reg_post_kernel), + padding=(0, reg_post_kernel // 2), + norm_cfg=norm_cfg, + act_cfg=dict(type='ReLU')) + self.reg_post_conv_xs.append(reg_post_conv_x) + self.reg_post_conv_ys = nn.ModuleList() + for i in range(self.reg_post_num): + reg_post_conv_y = ConvModule( + reg_in_channels, + reg_in_channels, + kernel_size=(reg_post_kernel, 1), + padding=(reg_post_kernel // 2, 0), + norm_cfg=norm_cfg, + act_cfg=dict(type='ReLU')) + self.reg_post_conv_ys.append(reg_post_conv_y) + + self.reg_conv_att_x = nn.Conv2d(reg_in_channels, 1, 1) + self.reg_conv_att_y = nn.Conv2d(reg_in_channels, 1, 1) + + self.fc_cls = nn.Linear(self.cls_out_channels, self.num_classes + 1) + self.relu = nn.ReLU(inplace=True) + + self.reg_cls_fcs = self._add_fc_branch(self.num_reg_fcs, + self.reg_in_channels, 1, + self.reg_cls_out_channels) + self.reg_offset_fcs = self._add_fc_branch(self.num_reg_fcs, + self.reg_in_channels, 1, + self.reg_offset_out_channels) + self.fc_reg_cls = nn.Linear(self.reg_cls_out_channels, 1) + self.fc_reg_offset = nn.Linear(self.reg_offset_out_channels, 1) + + def _add_fc_branch(self, num_branch_fcs, in_channels, roi_feat_size, + fc_out_channels): + in_channels = in_channels * roi_feat_size * roi_feat_size + branch_fcs = nn.ModuleList() + for i in range(num_branch_fcs): + fc_in_channels = (in_channels if i == 0 else fc_out_channels) + branch_fcs.append(nn.Linear(fc_in_channels, fc_out_channels)) + return branch_fcs + + def init_weights(self): + for module_list in [ + self.reg_cls_fcs, self.reg_offset_fcs, self.cls_fcs + ]: + for m in module_list.modules(): + if isinstance(m, nn.Linear): + xavier_init(m, distribution='uniform') + if self.reg_feat_up_ratio > 1: + kaiming_init(self.upsample_x, distribution='normal') + kaiming_init(self.upsample_y, distribution='normal') + + normal_init(self.reg_conv_att_x, 0, 0.01) + normal_init(self.reg_conv_att_y, 0, 0.01) + normal_init(self.fc_reg_offset, 0, 0.001) + normal_init(self.fc_reg_cls, 0, 0.01) + normal_init(self.fc_cls, 0, 0.01) + + def cls_forward(self, cls_x): + cls_x = cls_x.view(cls_x.size(0), -1) + for fc in self.cls_fcs: + cls_x = self.relu(fc(cls_x)) + cls_score = self.fc_cls(cls_x) + return cls_score + + def attention_pool(self, reg_x): + """Extract direction-specific features fx and fy with attention + methanism.""" + reg_fx = reg_x + reg_fy = reg_x + reg_fx_att = self.reg_conv_att_x(reg_fx).sigmoid() + reg_fy_att = self.reg_conv_att_y(reg_fy).sigmoid() + reg_fx_att = reg_fx_att / reg_fx_att.sum(dim=2).unsqueeze(2) + reg_fy_att = reg_fy_att / reg_fy_att.sum(dim=3).unsqueeze(3) + reg_fx = (reg_fx * reg_fx_att).sum(dim=2) + reg_fy = (reg_fy * reg_fy_att).sum(dim=3) + return reg_fx, reg_fy + + def side_aware_feature_extractor(self, reg_x): + """Refine and extract side-aware features without split them.""" + for reg_pre_conv in self.reg_pre_convs: + reg_x = reg_pre_conv(reg_x) + reg_fx, reg_fy = self.attention_pool(reg_x) + + if self.reg_post_num > 0: + reg_fx = reg_fx.unsqueeze(2) + reg_fy = reg_fy.unsqueeze(3) + for i in range(self.reg_post_num): + reg_fx = self.reg_post_conv_xs[i](reg_fx) + reg_fy = self.reg_post_conv_ys[i](reg_fy) + reg_fx = reg_fx.squeeze(2) + reg_fy = reg_fy.squeeze(3) + if self.reg_feat_up_ratio > 1: + reg_fx = self.relu(self.upsample_x(reg_fx)) + reg_fy = self.relu(self.upsample_y(reg_fy)) + reg_fx = torch.transpose(reg_fx, 1, 2) + reg_fy = torch.transpose(reg_fy, 1, 2) + return reg_fx.contiguous(), reg_fy.contiguous() + + def reg_pred(self, x, offset_fcs, cls_fcs): + """Predict bucketing estimation (cls_pred) and fine regression (offset + pred) with side-aware features.""" + x_offset = x.view(-1, self.reg_in_channels) + x_cls = x.view(-1, self.reg_in_channels) + + for fc in offset_fcs: + x_offset = self.relu(fc(x_offset)) + for fc in cls_fcs: + x_cls = self.relu(fc(x_cls)) + offset_pred = self.fc_reg_offset(x_offset) + cls_pred = self.fc_reg_cls(x_cls) + + offset_pred = offset_pred.view(x.size(0), -1) + cls_pred = cls_pred.view(x.size(0), -1) + + return offset_pred, cls_pred + + def side_aware_split(self, feat): + """Split side-aware features aligned with orders of bucketing + targets.""" + l_end = int(np.ceil(self.up_reg_feat_size / 2)) + r_start = int(np.floor(self.up_reg_feat_size / 2)) + feat_fl = feat[:, :l_end] + feat_fr = feat[:, r_start:].flip(dims=(1, )) + feat_fl = feat_fl.contiguous() + feat_fr = feat_fr.contiguous() + feat = torch.cat([feat_fl, feat_fr], dim=-1) + return feat + + def bbox_pred_split(self, bbox_pred, num_proposals_per_img): + """Split batch bbox prediction back to each image.""" + bucket_cls_preds, bucket_offset_preds = bbox_pred + bucket_cls_preds = bucket_cls_preds.split(num_proposals_per_img, 0) + bucket_offset_preds = bucket_offset_preds.split( + num_proposals_per_img, 0) + bbox_pred = tuple(zip(bucket_cls_preds, bucket_offset_preds)) + return bbox_pred + + def reg_forward(self, reg_x): + outs = self.side_aware_feature_extractor(reg_x) + edge_offset_preds = [] + edge_cls_preds = [] + reg_fx = outs[0] + reg_fy = outs[1] + offset_pred_x, cls_pred_x = self.reg_pred(reg_fx, self.reg_offset_fcs, + self.reg_cls_fcs) + offset_pred_y, cls_pred_y = self.reg_pred(reg_fy, self.reg_offset_fcs, + self.reg_cls_fcs) + offset_pred_x = self.side_aware_split(offset_pred_x) + offset_pred_y = self.side_aware_split(offset_pred_y) + cls_pred_x = self.side_aware_split(cls_pred_x) + cls_pred_y = self.side_aware_split(cls_pred_y) + edge_offset_preds = torch.cat([offset_pred_x, offset_pred_y], dim=-1) + edge_cls_preds = torch.cat([cls_pred_x, cls_pred_y], dim=-1) + + return (edge_cls_preds, edge_offset_preds) + + def forward(self, x): + + bbox_pred = self.reg_forward(x) + cls_score = self.cls_forward(x) + + return cls_score, bbox_pred + + def get_targets(self, sampling_results, gt_bboxes, gt_labels, + rcnn_train_cfg): + pos_proposals = [res.pos_bboxes for res in sampling_results] + neg_proposals = [res.neg_bboxes for res in sampling_results] + pos_gt_bboxes = [res.pos_gt_bboxes for res in sampling_results] + pos_gt_labels = [res.pos_gt_labels for res in sampling_results] + cls_reg_targets = self.bucket_target(pos_proposals, neg_proposals, + pos_gt_bboxes, pos_gt_labels, + rcnn_train_cfg) + (labels, label_weights, bucket_cls_targets, bucket_cls_weights, + bucket_offset_targets, bucket_offset_weights) = cls_reg_targets + return (labels, label_weights, (bucket_cls_targets, + bucket_offset_targets), + (bucket_cls_weights, bucket_offset_weights)) + + def bucket_target(self, + pos_proposals_list, + neg_proposals_list, + pos_gt_bboxes_list, + pos_gt_labels_list, + rcnn_train_cfg, + concat=True): + (labels, label_weights, bucket_cls_targets, bucket_cls_weights, + bucket_offset_targets, bucket_offset_weights) = multi_apply( + self._bucket_target_single, + pos_proposals_list, + neg_proposals_list, + pos_gt_bboxes_list, + pos_gt_labels_list, + cfg=rcnn_train_cfg) + + if concat: + labels = torch.cat(labels, 0) + label_weights = torch.cat(label_weights, 0) + bucket_cls_targets = torch.cat(bucket_cls_targets, 0) + bucket_cls_weights = torch.cat(bucket_cls_weights, 0) + bucket_offset_targets = torch.cat(bucket_offset_targets, 0) + bucket_offset_weights = torch.cat(bucket_offset_weights, 0) + return (labels, label_weights, bucket_cls_targets, bucket_cls_weights, + bucket_offset_targets, bucket_offset_weights) + + def _bucket_target_single(self, pos_proposals, neg_proposals, + pos_gt_bboxes, pos_gt_labels, cfg): + """Compute bucketing estimation targets and fine regression targets for + a single image. + + Args: + pos_proposals (Tensor): positive proposals of a single image, + Shape (n_pos, 4) + neg_proposals (Tensor): negative proposals of a single image, + Shape (n_neg, 4). + pos_gt_bboxes (Tensor): gt bboxes assigned to positive proposals + of a single image, Shape (n_pos, 4). + pos_gt_labels (Tensor): gt labels assigned to positive proposals + of a single image, Shape (n_pos, ). + cfg (dict): Config of calculating targets + + Returns: + tuple: + + - labels (Tensor): Labels in a single image. \ + Shape (n,). + - label_weights (Tensor): Label weights in a single image.\ + Shape (n,) + - bucket_cls_targets (Tensor): Bucket cls targets in \ + a single image. Shape (n, num_buckets*2). + - bucket_cls_weights (Tensor): Bucket cls weights in \ + a single image. Shape (n, num_buckets*2). + - bucket_offset_targets (Tensor): Bucket offset targets \ + in a single image. Shape (n, num_buckets*2). + - bucket_offset_targets (Tensor): Bucket offset weights \ + in a single image. Shape (n, num_buckets*2). + """ + num_pos = pos_proposals.size(0) + num_neg = neg_proposals.size(0) + num_samples = num_pos + num_neg + labels = pos_gt_bboxes.new_full((num_samples, ), + self.num_classes, + dtype=torch.long) + label_weights = pos_proposals.new_zeros(num_samples) + bucket_cls_targets = pos_proposals.new_zeros(num_samples, + 4 * self.side_num) + bucket_cls_weights = pos_proposals.new_zeros(num_samples, + 4 * self.side_num) + bucket_offset_targets = pos_proposals.new_zeros( + num_samples, 4 * self.side_num) + bucket_offset_weights = pos_proposals.new_zeros( + num_samples, 4 * self.side_num) + if num_pos > 0: + labels[:num_pos] = pos_gt_labels + label_weights[:num_pos] = 1.0 + (pos_bucket_offset_targets, pos_bucket_offset_weights, + pos_bucket_cls_targets, + pos_bucket_cls_weights) = self.bbox_coder.encode( + pos_proposals, pos_gt_bboxes) + bucket_cls_targets[:num_pos, :] = pos_bucket_cls_targets + bucket_cls_weights[:num_pos, :] = pos_bucket_cls_weights + bucket_offset_targets[:num_pos, :] = pos_bucket_offset_targets + bucket_offset_weights[:num_pos, :] = pos_bucket_offset_weights + if num_neg > 0: + label_weights[-num_neg:] = 1.0 + return (labels, label_weights, bucket_cls_targets, bucket_cls_weights, + bucket_offset_targets, bucket_offset_weights) + + def loss(self, + cls_score, + bbox_pred, + rois, + labels, + label_weights, + bbox_targets, + bbox_weights, + reduction_override=None): + losses = dict() + if cls_score is not None: + avg_factor = max(torch.sum(label_weights > 0).float().item(), 1.) + losses['loss_cls'] = self.loss_cls( + cls_score, + labels, + label_weights, + avg_factor=avg_factor, + reduction_override=reduction_override) + losses['acc'] = accuracy(cls_score, labels) + + if bbox_pred is not None: + bucket_cls_preds, bucket_offset_preds = bbox_pred + bucket_cls_targets, bucket_offset_targets = bbox_targets + bucket_cls_weights, bucket_offset_weights = bbox_weights + # edge cls + bucket_cls_preds = bucket_cls_preds.view(-1, self.side_num) + bucket_cls_targets = bucket_cls_targets.view(-1, self.side_num) + bucket_cls_weights = bucket_cls_weights.view(-1, self.side_num) + losses['loss_bbox_cls'] = self.loss_bbox_cls( + bucket_cls_preds, + bucket_cls_targets, + bucket_cls_weights, + avg_factor=bucket_cls_targets.size(0), + reduction_override=reduction_override) + + losses['loss_bbox_reg'] = self.loss_bbox_reg( + bucket_offset_preds, + bucket_offset_targets, + bucket_offset_weights, + avg_factor=bucket_offset_targets.size(0), + reduction_override=reduction_override) + + return losses + + @force_fp32(apply_to=('cls_score', 'bbox_pred')) + def get_bboxes(self, + rois, + cls_score, + bbox_pred, + img_shape, + scale_factor, + rescale=False, + cfg=None): + if isinstance(cls_score, list): + cls_score = sum(cls_score) / float(len(cls_score)) + scores = F.softmax(cls_score, dim=1) if cls_score is not None else None + + if bbox_pred is not None: + bboxes, confids = self.bbox_coder.decode(rois[:, 1:], bbox_pred, + img_shape) + else: + bboxes = rois[:, 1:].clone() + confids = None + if img_shape is not None: + bboxes[:, [0, 2]].clamp_(min=0, max=img_shape[1] - 1) + bboxes[:, [1, 3]].clamp_(min=0, max=img_shape[0] - 1) + + if rescale and bboxes.size(0) > 0: + if isinstance(scale_factor, float): + bboxes /= scale_factor + else: + bboxes /= torch.from_numpy(scale_factor).to(bboxes.device) + + if cfg is None: + return bboxes, scores + else: + det_bboxes, det_labels = multiclass_nms( + bboxes, + scores, + cfg.score_thr, + cfg.nms, + cfg.max_per_img, + score_factors=confids) + + return det_bboxes, det_labels + + @force_fp32(apply_to=('bbox_preds', )) + def refine_bboxes(self, rois, labels, bbox_preds, pos_is_gts, img_metas): + """Refine bboxes during training. + + Args: + rois (Tensor): Shape (n*bs, 5), where n is image number per GPU, + and bs is the sampled RoIs per image. + labels (Tensor): Shape (n*bs, ). + bbox_preds (list[Tensor]): Shape [(n*bs, num_buckets*2), \ + (n*bs, num_buckets*2)]. + pos_is_gts (list[Tensor]): Flags indicating if each positive bbox + is a gt bbox. + img_metas (list[dict]): Meta info of each image. + + Returns: + list[Tensor]: Refined bboxes of each image in a mini-batch. + """ + img_ids = rois[:, 0].long().unique(sorted=True) + assert img_ids.numel() == len(img_metas) + + bboxes_list = [] + for i in range(len(img_metas)): + inds = torch.nonzero( + rois[:, 0] == i, as_tuple=False).squeeze(dim=1) + num_rois = inds.numel() + + bboxes_ = rois[inds, 1:] + label_ = labels[inds] + edge_cls_preds, edge_offset_preds = bbox_preds + edge_cls_preds_ = edge_cls_preds[inds] + edge_offset_preds_ = edge_offset_preds[inds] + bbox_pred_ = [edge_cls_preds_, edge_offset_preds_] + img_meta_ = img_metas[i] + pos_is_gts_ = pos_is_gts[i] + + bboxes = self.regress_by_class(bboxes_, label_, bbox_pred_, + img_meta_) + # filter gt bboxes + pos_keep = 1 - pos_is_gts_ + keep_inds = pos_is_gts_.new_ones(num_rois) + keep_inds[:len(pos_is_gts_)] = pos_keep + + bboxes_list.append(bboxes[keep_inds.type(torch.bool)]) + + return bboxes_list + + @force_fp32(apply_to=('bbox_pred', )) + def regress_by_class(self, rois, label, bbox_pred, img_meta): + """Regress the bbox for the predicted class. Used in Cascade R-CNN. + + Args: + rois (Tensor): shape (n, 4) or (n, 5) + label (Tensor): shape (n, ) + bbox_pred (list[Tensor]): shape [(n, num_buckets *2), \ + (n, num_buckets *2)] + img_meta (dict): Image meta info. + + Returns: + Tensor: Regressed bboxes, the same shape as input rois. + """ + assert rois.size(1) == 4 or rois.size(1) == 5 + + if rois.size(1) == 4: + new_rois, _ = self.bbox_coder.decode(rois, bbox_pred, + img_meta['img_shape']) + else: + bboxes, _ = self.bbox_coder.decode(rois[:, 1:], bbox_pred, + img_meta['img_shape']) + new_rois = torch.cat((rois[:, [0]], bboxes), dim=1) + + return new_rois diff --git a/detection/mmdet/models/roi_heads/bbox_heads/scnet_bbox_head.py b/detection/mmdet/models/roi_heads/bbox_heads/scnet_bbox_head.py new file mode 100644 index 0000000..35758f4 --- /dev/null +++ b/detection/mmdet/models/roi_heads/bbox_heads/scnet_bbox_head.py @@ -0,0 +1,76 @@ +from mmdet.models.builder import HEADS +from .convfc_bbox_head import ConvFCBBoxHead + + +@HEADS.register_module() +class SCNetBBoxHead(ConvFCBBoxHead): + """BBox head for `SCNet `_. + + This inherits ``ConvFCBBoxHead`` with modified forward() function, allow us + to get intermediate shared feature. + """ + + def _forward_shared(self, x): + """Forward function for shared part.""" + if self.num_shared_convs > 0: + for conv in self.shared_convs: + x = conv(x) + + if self.num_shared_fcs > 0: + if self.with_avg_pool: + x = self.avg_pool(x) + + x = x.flatten(1) + + for fc in self.shared_fcs: + x = self.relu(fc(x)) + + return x + + def _forward_cls_reg(self, x): + """Forward function for classification and regression parts.""" + x_cls = x + x_reg = x + + for conv in self.cls_convs: + x_cls = conv(x_cls) + if x_cls.dim() > 2: + if self.with_avg_pool: + x_cls = self.avg_pool(x_cls) + x_cls = x_cls.flatten(1) + for fc in self.cls_fcs: + x_cls = self.relu(fc(x_cls)) + + for conv in self.reg_convs: + x_reg = conv(x_reg) + if x_reg.dim() > 2: + if self.with_avg_pool: + x_reg = self.avg_pool(x_reg) + x_reg = x_reg.flatten(1) + for fc in self.reg_fcs: + x_reg = self.relu(fc(x_reg)) + + cls_score = self.fc_cls(x_cls) if self.with_cls else None + bbox_pred = self.fc_reg(x_reg) if self.with_reg else None + + return cls_score, bbox_pred + + def forward(self, x, return_shared_feat=False): + """Forward function. + + Args: + x (Tensor): input features + return_shared_feat (bool): If True, return cls-reg-shared feature. + + Return: + out (tuple[Tensor]): contain ``cls_score`` and ``bbox_pred``, + if ``return_shared_feat`` is True, append ``x_shared`` to the + returned tuple. + """ + x_shared = self._forward_shared(x) + out = self._forward_cls_reg(x_shared) + + if return_shared_feat: + out += (x_shared, ) + + return out diff --git a/detection/mmdet/models/roi_heads/cascade_roi_head.py b/detection/mmdet/models/roi_heads/cascade_roi_head.py new file mode 100644 index 0000000..45b6f36 --- /dev/null +++ b/detection/mmdet/models/roi_heads/cascade_roi_head.py @@ -0,0 +1,507 @@ +import torch +import torch.nn as nn + +from mmdet.core import (bbox2result, bbox2roi, bbox_mapping, build_assigner, + build_sampler, merge_aug_bboxes, merge_aug_masks, + multiclass_nms) +from ..builder import HEADS, build_head, build_roi_extractor +from .base_roi_head import BaseRoIHead +from .test_mixins import BBoxTestMixin, MaskTestMixin + + +@HEADS.register_module() +class CascadeRoIHead(BaseRoIHead, BBoxTestMixin, MaskTestMixin): + """Cascade roi head including one bbox head and one mask head. + + https://arxiv.org/abs/1712.00726 + """ + + def __init__(self, + num_stages, + stage_loss_weights, + bbox_roi_extractor=None, + bbox_head=None, + mask_roi_extractor=None, + mask_head=None, + shared_head=None, + train_cfg=None, + test_cfg=None): + assert bbox_roi_extractor is not None + assert bbox_head is not None + assert shared_head is None, \ + 'Shared head is not supported in Cascade RCNN anymore' + self.num_stages = num_stages + self.stage_loss_weights = stage_loss_weights + super(CascadeRoIHead, self).__init__( + bbox_roi_extractor=bbox_roi_extractor, + bbox_head=bbox_head, + mask_roi_extractor=mask_roi_extractor, + mask_head=mask_head, + shared_head=shared_head, + train_cfg=train_cfg, + test_cfg=test_cfg) + + def init_bbox_head(self, bbox_roi_extractor, bbox_head): + """Initialize box head and box roi extractor. + + Args: + bbox_roi_extractor (dict): Config of box roi extractor. + bbox_head (dict): Config of box in box head. + """ + self.bbox_roi_extractor = nn.ModuleList() + self.bbox_head = nn.ModuleList() + if not isinstance(bbox_roi_extractor, list): + bbox_roi_extractor = [ + bbox_roi_extractor for _ in range(self.num_stages) + ] + if not isinstance(bbox_head, list): + bbox_head = [bbox_head for _ in range(self.num_stages)] + assert len(bbox_roi_extractor) == len(bbox_head) == self.num_stages + for roi_extractor, head in zip(bbox_roi_extractor, bbox_head): + self.bbox_roi_extractor.append(build_roi_extractor(roi_extractor)) + self.bbox_head.append(build_head(head)) + + def init_mask_head(self, mask_roi_extractor, mask_head): + """Initialize mask head and mask roi extractor. + + Args: + mask_roi_extractor (dict): Config of mask roi extractor. + mask_head (dict): Config of mask in mask head. + """ + self.mask_head = nn.ModuleList() + if not isinstance(mask_head, list): + mask_head = [mask_head for _ in range(self.num_stages)] + assert len(mask_head) == self.num_stages + for head in mask_head: + self.mask_head.append(build_head(head)) + if mask_roi_extractor is not None: + self.share_roi_extractor = False + self.mask_roi_extractor = nn.ModuleList() + if not isinstance(mask_roi_extractor, list): + mask_roi_extractor = [ + mask_roi_extractor for _ in range(self.num_stages) + ] + assert len(mask_roi_extractor) == self.num_stages + for roi_extractor in mask_roi_extractor: + self.mask_roi_extractor.append( + build_roi_extractor(roi_extractor)) + else: + self.share_roi_extractor = True + self.mask_roi_extractor = self.bbox_roi_extractor + + def init_assigner_sampler(self): + """Initialize assigner and sampler for each stage.""" + self.bbox_assigner = [] + self.bbox_sampler = [] + if self.train_cfg is not None: + for idx, rcnn_train_cfg in enumerate(self.train_cfg): + self.bbox_assigner.append( + build_assigner(rcnn_train_cfg.assigner)) + self.current_stage = idx + self.bbox_sampler.append( + build_sampler(rcnn_train_cfg.sampler, context=self)) + + def init_weights(self, pretrained): + """Initialize the weights in head. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + if self.with_shared_head: + self.shared_head.init_weights(pretrained=pretrained) + for i in range(self.num_stages): + if self.with_bbox: + self.bbox_roi_extractor[i].init_weights() + self.bbox_head[i].init_weights() + if self.with_mask: + if not self.share_roi_extractor: + self.mask_roi_extractor[i].init_weights() + self.mask_head[i].init_weights() + + def forward_dummy(self, x, proposals): + """Dummy forward function.""" + # bbox head + outs = () + rois = bbox2roi([proposals]) + if self.with_bbox: + for i in range(self.num_stages): + bbox_results = self._bbox_forward(i, x, rois) + outs = outs + (bbox_results['cls_score'], + bbox_results['bbox_pred']) + # mask heads + if self.with_mask: + mask_rois = rois[:100] + for i in range(self.num_stages): + mask_results = self._mask_forward(i, x, mask_rois) + outs = outs + (mask_results['mask_pred'], ) + return outs + + def _bbox_forward(self, stage, x, rois): + """Box head forward function used in both training and testing.""" + bbox_roi_extractor = self.bbox_roi_extractor[stage] + bbox_head = self.bbox_head[stage] + bbox_feats = bbox_roi_extractor(x[:bbox_roi_extractor.num_inputs], + rois) + # do not support caffe_c4 model anymore + cls_score, bbox_pred = bbox_head(bbox_feats) + + bbox_results = dict( + cls_score=cls_score, bbox_pred=bbox_pred, bbox_feats=bbox_feats) + return bbox_results + + def _bbox_forward_train(self, stage, x, sampling_results, gt_bboxes, + gt_labels, rcnn_train_cfg): + """Run forward function and calculate loss for box head in training.""" + rois = bbox2roi([res.bboxes for res in sampling_results]) + bbox_results = self._bbox_forward(stage, x, rois) + bbox_targets = self.bbox_head[stage].get_targets( + sampling_results, gt_bboxes, gt_labels, rcnn_train_cfg) + loss_bbox = self.bbox_head[stage].loss(bbox_results['cls_score'], + bbox_results['bbox_pred'], rois, + *bbox_targets) + + bbox_results.update( + loss_bbox=loss_bbox, rois=rois, bbox_targets=bbox_targets) + return bbox_results + + def _mask_forward(self, stage, x, rois): + """Mask head forward function used in both training and testing.""" + mask_roi_extractor = self.mask_roi_extractor[stage] + mask_head = self.mask_head[stage] + mask_feats = mask_roi_extractor(x[:mask_roi_extractor.num_inputs], + rois) + # do not support caffe_c4 model anymore + mask_pred = mask_head(mask_feats) + + mask_results = dict(mask_pred=mask_pred) + return mask_results + + def _mask_forward_train(self, + stage, + x, + sampling_results, + gt_masks, + rcnn_train_cfg, + bbox_feats=None): + """Run forward function and calculate loss for mask head in + training.""" + pos_rois = bbox2roi([res.pos_bboxes for res in sampling_results]) + mask_results = self._mask_forward(stage, x, pos_rois) + + mask_targets = self.mask_head[stage].get_targets( + sampling_results, gt_masks, rcnn_train_cfg) + pos_labels = torch.cat([res.pos_gt_labels for res in sampling_results]) + loss_mask = self.mask_head[stage].loss(mask_results['mask_pred'], + mask_targets, pos_labels) + + mask_results.update(loss_mask=loss_mask) + return mask_results + + def forward_train(self, + x, + img_metas, + proposal_list, + gt_bboxes, + gt_labels, + gt_bboxes_ignore=None, + gt_masks=None): + """ + Args: + x (list[Tensor]): list of multi-level img features. + img_metas (list[dict]): list of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmdet/datasets/pipelines/formatting.py:Collect`. + proposals (list[Tensors]): list of region proposals. + gt_bboxes (list[Tensor]): Ground truth bboxes for each image with + shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + gt_labels (list[Tensor]): class indices corresponding to each box + gt_bboxes_ignore (None | list[Tensor]): specify which bounding + boxes can be ignored when computing the loss. + gt_masks (None | Tensor) : true segmentation masks for each box + used if the architecture supports a segmentation task. + + Returns: + dict[str, Tensor]: a dictionary of loss components + """ + losses = dict() + for i in range(self.num_stages): + self.current_stage = i + rcnn_train_cfg = self.train_cfg[i] + lw = self.stage_loss_weights[i] + + # assign gts and sample proposals + sampling_results = [] + if self.with_bbox or self.with_mask: + bbox_assigner = self.bbox_assigner[i] + bbox_sampler = self.bbox_sampler[i] + num_imgs = len(img_metas) + if gt_bboxes_ignore is None: + gt_bboxes_ignore = [None for _ in range(num_imgs)] + + for j in range(num_imgs): + assign_result = bbox_assigner.assign( + proposal_list[j], gt_bboxes[j], gt_bboxes_ignore[j], + gt_labels[j]) + sampling_result = bbox_sampler.sample( + assign_result, + proposal_list[j], + gt_bboxes[j], + gt_labels[j], + feats=[lvl_feat[j][None] for lvl_feat in x]) + sampling_results.append(sampling_result) + + # bbox head forward and loss + bbox_results = self._bbox_forward_train(i, x, sampling_results, + gt_bboxes, gt_labels, + rcnn_train_cfg) + + for name, value in bbox_results['loss_bbox'].items(): + losses[f's{i}.{name}'] = ( + value * lw if 'loss' in name else value) + + # mask head forward and loss + if self.with_mask: + mask_results = self._mask_forward_train( + i, x, sampling_results, gt_masks, rcnn_train_cfg, + bbox_results['bbox_feats']) + for name, value in mask_results['loss_mask'].items(): + losses[f's{i}.{name}'] = ( + value * lw if 'loss' in name else value) + + # refine bboxes + if i < self.num_stages - 1: + pos_is_gts = [res.pos_is_gt for res in sampling_results] + # bbox_targets is a tuple + roi_labels = bbox_results['bbox_targets'][0] + with torch.no_grad(): + roi_labels = torch.where( + roi_labels == self.bbox_head[i].num_classes, + bbox_results['cls_score'][:, :-1].argmax(1), + roi_labels) + proposal_list = self.bbox_head[i].refine_bboxes( + bbox_results['rois'], roi_labels, + bbox_results['bbox_pred'], pos_is_gts, img_metas) + + return losses + + def simple_test(self, x, proposal_list, img_metas, rescale=False): + """Test without augmentation.""" + assert self.with_bbox, 'Bbox head must be implemented.' + num_imgs = len(proposal_list) + img_shapes = tuple(meta['img_shape'] for meta in img_metas) + ori_shapes = tuple(meta['ori_shape'] for meta in img_metas) + scale_factors = tuple(meta['scale_factor'] for meta in img_metas) + + # "ms" in variable names means multi-stage + ms_bbox_result = {} + ms_segm_result = {} + ms_scores = [] + rcnn_test_cfg = self.test_cfg + + rois = bbox2roi(proposal_list) + for i in range(self.num_stages): + bbox_results = self._bbox_forward(i, x, rois) + + # split batch bbox prediction back to each image + cls_score = bbox_results['cls_score'] + bbox_pred = bbox_results['bbox_pred'] + num_proposals_per_img = tuple( + len(proposals) for proposals in proposal_list) + rois = rois.split(num_proposals_per_img, 0) + cls_score = cls_score.split(num_proposals_per_img, 0) + if isinstance(bbox_pred, torch.Tensor): + bbox_pred = bbox_pred.split(num_proposals_per_img, 0) + else: + bbox_pred = self.bbox_head[i].bbox_pred_split( + bbox_pred, num_proposals_per_img) + ms_scores.append(cls_score) + + if i < self.num_stages - 1: + bbox_label = [s[:, :-1].argmax(dim=1) for s in cls_score] + rois = torch.cat([ + self.bbox_head[i].regress_by_class(rois[j], bbox_label[j], + bbox_pred[j], + img_metas[j]) + for j in range(num_imgs) + ]) + + # average scores of each image by stages + cls_score = [ + sum([score[i] for score in ms_scores]) / float(len(ms_scores)) + for i in range(num_imgs) + ] + + # apply bbox post-processing to each image individually + det_bboxes = [] + det_labels = [] + for i in range(num_imgs): + det_bbox, det_label = self.bbox_head[-1].get_bboxes( + rois[i], + cls_score[i], + bbox_pred[i], + img_shapes[i], + scale_factors[i], + rescale=rescale, + cfg=rcnn_test_cfg) + det_bboxes.append(det_bbox) + det_labels.append(det_label) + + if torch.onnx.is_in_onnx_export(): + return det_bboxes, det_labels + bbox_results = [ + bbox2result(det_bboxes[i], det_labels[i], + self.bbox_head[-1].num_classes) + for i in range(num_imgs) + ] + ms_bbox_result['ensemble'] = bbox_results + + if self.with_mask: + if all(det_bbox.shape[0] == 0 for det_bbox in det_bboxes): + mask_classes = self.mask_head[-1].num_classes + segm_results = [[[] for _ in range(mask_classes)] + for _ in range(num_imgs)] + else: + if rescale and not isinstance(scale_factors[0], float): + scale_factors = [ + torch.from_numpy(scale_factor).to(det_bboxes[0].device) + for scale_factor in scale_factors + ] + _bboxes = [ + det_bboxes[i][:, :4] * + scale_factors[i] if rescale else det_bboxes[i][:, :4] + for i in range(len(det_bboxes)) + ] + mask_rois = bbox2roi(_bboxes) + num_mask_rois_per_img = tuple( + _bbox.size(0) for _bbox in _bboxes) + aug_masks = [] + for i in range(self.num_stages): + mask_results = self._mask_forward(i, x, mask_rois) + mask_pred = mask_results['mask_pred'] + # split batch mask prediction back to each image + mask_pred = mask_pred.split(num_mask_rois_per_img, 0) + aug_masks.append( + [m.sigmoid().cpu().numpy() for m in mask_pred]) + + # apply mask post-processing to each image individually + segm_results = [] + for i in range(num_imgs): + if det_bboxes[i].shape[0] == 0: + segm_results.append( + [[] + for _ in range(self.mask_head[-1].num_classes)]) + else: + aug_mask = [mask[i] for mask in aug_masks] + merged_masks = merge_aug_masks( + aug_mask, [[img_metas[i]]] * self.num_stages, + rcnn_test_cfg) + segm_result = self.mask_head[-1].get_seg_masks( + merged_masks, _bboxes[i], det_labels[i], + rcnn_test_cfg, ori_shapes[i], scale_factors[i], + rescale) + segm_results.append(segm_result) + ms_segm_result['ensemble'] = segm_results + + if self.with_mask: + results = list( + zip(ms_bbox_result['ensemble'], ms_segm_result['ensemble'])) + else: + results = ms_bbox_result['ensemble'] + + return results + + def aug_test(self, features, proposal_list, img_metas, rescale=False): + """Test with augmentations. + + If rescale is False, then returned bboxes and masks will fit the scale + of imgs[0]. + """ + rcnn_test_cfg = self.test_cfg + aug_bboxes = [] + aug_scores = [] + for x, img_meta in zip(features, img_metas): + # only one image in the batch + img_shape = img_meta[0]['img_shape'] + scale_factor = img_meta[0]['scale_factor'] + flip = img_meta[0]['flip'] + flip_direction = img_meta[0]['flip_direction'] + + proposals = bbox_mapping(proposal_list[0][:, :4], img_shape, + scale_factor, flip, flip_direction) + # "ms" in variable names means multi-stage + ms_scores = [] + + rois = bbox2roi([proposals]) + for i in range(self.num_stages): + bbox_results = self._bbox_forward(i, x, rois) + ms_scores.append(bbox_results['cls_score']) + + if i < self.num_stages - 1: + bbox_label = bbox_results['cls_score'][:, :-1].argmax( + dim=1) + rois = self.bbox_head[i].regress_by_class( + rois, bbox_label, bbox_results['bbox_pred'], + img_meta[0]) + + cls_score = sum(ms_scores) / float(len(ms_scores)) + bboxes, scores = self.bbox_head[-1].get_bboxes( + rois, + cls_score, + bbox_results['bbox_pred'], + img_shape, + scale_factor, + rescale=False, + cfg=None) + aug_bboxes.append(bboxes) + aug_scores.append(scores) + + # after merging, bboxes will be rescaled to the original image size + merged_bboxes, merged_scores = merge_aug_bboxes( + aug_bboxes, aug_scores, img_metas, rcnn_test_cfg) + det_bboxes, det_labels = multiclass_nms(merged_bboxes, merged_scores, + rcnn_test_cfg.score_thr, + rcnn_test_cfg.nms, + rcnn_test_cfg.max_per_img) + + bbox_result = bbox2result(det_bboxes, det_labels, + self.bbox_head[-1].num_classes) + + if self.with_mask: + if det_bboxes.shape[0] == 0: + segm_result = [[[] + for _ in range(self.mask_head[-1].num_classes)] + ] + else: + aug_masks = [] + aug_img_metas = [] + for x, img_meta in zip(features, img_metas): + img_shape = img_meta[0]['img_shape'] + scale_factor = img_meta[0]['scale_factor'] + flip = img_meta[0]['flip'] + flip_direction = img_meta[0]['flip_direction'] + _bboxes = bbox_mapping(det_bboxes[:, :4], img_shape, + scale_factor, flip, flip_direction) + mask_rois = bbox2roi([_bboxes]) + for i in range(self.num_stages): + mask_results = self._mask_forward(i, x, mask_rois) + aug_masks.append( + mask_results['mask_pred'].sigmoid().cpu().numpy()) + aug_img_metas.append(img_meta) + merged_masks = merge_aug_masks(aug_masks, aug_img_metas, + self.test_cfg) + + ori_shape = img_metas[0][0]['ori_shape'] + segm_result = self.mask_head[-1].get_seg_masks( + merged_masks, + det_bboxes, + det_labels, + rcnn_test_cfg, + ori_shape, + scale_factor=1.0, + rescale=False) + return [(bbox_result, segm_result)] + else: + return [bbox_result] diff --git a/detection/mmdet/models/roi_heads/double_roi_head.py b/detection/mmdet/models/roi_heads/double_roi_head.py new file mode 100644 index 0000000..a1aa6c8 --- /dev/null +++ b/detection/mmdet/models/roi_heads/double_roi_head.py @@ -0,0 +1,33 @@ +from ..builder import HEADS +from .standard_roi_head import StandardRoIHead + + +@HEADS.register_module() +class DoubleHeadRoIHead(StandardRoIHead): + """RoI head for Double Head RCNN. + + https://arxiv.org/abs/1904.06493 + """ + + def __init__(self, reg_roi_scale_factor, **kwargs): + super(DoubleHeadRoIHead, self).__init__(**kwargs) + self.reg_roi_scale_factor = reg_roi_scale_factor + + def _bbox_forward(self, x, rois): + """Box head forward function used in both training and testing time.""" + bbox_cls_feats = self.bbox_roi_extractor( + x[:self.bbox_roi_extractor.num_inputs], rois) + bbox_reg_feats = self.bbox_roi_extractor( + x[:self.bbox_roi_extractor.num_inputs], + rois, + roi_scale_factor=self.reg_roi_scale_factor) + if self.with_shared_head: + bbox_cls_feats = self.shared_head(bbox_cls_feats) + bbox_reg_feats = self.shared_head(bbox_reg_feats) + cls_score, bbox_pred = self.bbox_head(bbox_cls_feats, bbox_reg_feats) + + bbox_results = dict( + cls_score=cls_score, + bbox_pred=bbox_pred, + bbox_feats=bbox_cls_feats) + return bbox_results diff --git a/detection/mmdet/models/roi_heads/dynamic_roi_head.py b/detection/mmdet/models/roi_heads/dynamic_roi_head.py new file mode 100644 index 0000000..89427a9 --- /dev/null +++ b/detection/mmdet/models/roi_heads/dynamic_roi_head.py @@ -0,0 +1,154 @@ +import numpy as np +import torch + +from mmdet.core import bbox2roi +from mmdet.models.losses import SmoothL1Loss +from ..builder import HEADS +from .standard_roi_head import StandardRoIHead + +EPS = 1e-15 + + +@HEADS.register_module() +class DynamicRoIHead(StandardRoIHead): + """RoI head for `Dynamic R-CNN `_.""" + + def __init__(self, **kwargs): + super(DynamicRoIHead, self).__init__(**kwargs) + assert isinstance(self.bbox_head.loss_bbox, SmoothL1Loss) + # the IoU history of the past `update_iter_interval` iterations + self.iou_history = [] + # the beta history of the past `update_iter_interval` iterations + self.beta_history = [] + + def forward_train(self, + x, + img_metas, + proposal_list, + gt_bboxes, + gt_labels, + gt_bboxes_ignore=None, + gt_masks=None): + """Forward function for training. + + Args: + x (list[Tensor]): list of multi-level img features. + + img_metas (list[dict]): list of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmdet/datasets/pipelines/formatting.py:Collect`. + + proposals (list[Tensors]): list of region proposals. + + gt_bboxes (list[Tensor]): each item are the truth boxes for each + image in [tl_x, tl_y, br_x, br_y] format. + + gt_labels (list[Tensor]): class indices corresponding to each box + + gt_bboxes_ignore (None | list[Tensor]): specify which bounding + boxes can be ignored when computing the loss. + + gt_masks (None | Tensor) : true segmentation masks for each box + used if the architecture supports a segmentation task. + + Returns: + dict[str, Tensor]: a dictionary of loss components + """ + # assign gts and sample proposals + if self.with_bbox or self.with_mask: + num_imgs = len(img_metas) + if gt_bboxes_ignore is None: + gt_bboxes_ignore = [None for _ in range(num_imgs)] + sampling_results = [] + cur_iou = [] + for i in range(num_imgs): + assign_result = self.bbox_assigner.assign( + proposal_list[i], gt_bboxes[i], gt_bboxes_ignore[i], + gt_labels[i]) + sampling_result = self.bbox_sampler.sample( + assign_result, + proposal_list[i], + gt_bboxes[i], + gt_labels[i], + feats=[lvl_feat[i][None] for lvl_feat in x]) + # record the `iou_topk`-th largest IoU in an image + iou_topk = min(self.train_cfg.dynamic_rcnn.iou_topk, + len(assign_result.max_overlaps)) + ious, _ = torch.topk(assign_result.max_overlaps, iou_topk) + cur_iou.append(ious[-1].item()) + sampling_results.append(sampling_result) + # average the current IoUs over images + cur_iou = np.mean(cur_iou) + self.iou_history.append(cur_iou) + + losses = dict() + # bbox head forward and loss + if self.with_bbox: + bbox_results = self._bbox_forward_train(x, sampling_results, + gt_bboxes, gt_labels, + img_metas) + losses.update(bbox_results['loss_bbox']) + + # mask head forward and loss + if self.with_mask: + mask_results = self._mask_forward_train(x, sampling_results, + bbox_results['bbox_feats'], + gt_masks, img_metas) + losses.update(mask_results['loss_mask']) + + # update IoU threshold and SmoothL1 beta + update_iter_interval = self.train_cfg.dynamic_rcnn.update_iter_interval + if len(self.iou_history) % update_iter_interval == 0: + new_iou_thr, new_beta = self.update_hyperparameters() + + return losses + + def _bbox_forward_train(self, x, sampling_results, gt_bboxes, gt_labels, + img_metas): + num_imgs = len(img_metas) + rois = bbox2roi([res.bboxes for res in sampling_results]) + bbox_results = self._bbox_forward(x, rois) + + bbox_targets = self.bbox_head.get_targets(sampling_results, gt_bboxes, + gt_labels, self.train_cfg) + # record the `beta_topk`-th smallest target + # `bbox_targets[2]` and `bbox_targets[3]` stand for bbox_targets + # and bbox_weights, respectively + pos_inds = bbox_targets[3][:, 0].nonzero().squeeze(1) + num_pos = len(pos_inds) + cur_target = bbox_targets[2][pos_inds, :2].abs().mean(dim=1) + beta_topk = min(self.train_cfg.dynamic_rcnn.beta_topk * num_imgs, + num_pos) + cur_target = torch.kthvalue(cur_target, beta_topk)[0].item() + self.beta_history.append(cur_target) + loss_bbox = self.bbox_head.loss(bbox_results['cls_score'], + bbox_results['bbox_pred'], rois, + *bbox_targets) + + bbox_results.update(loss_bbox=loss_bbox) + return bbox_results + + def update_hyperparameters(self): + """Update hyperparameters like IoU thresholds for assigner and beta for + SmoothL1 loss based on the training statistics. + + Returns: + tuple[float]: the updated ``iou_thr`` and ``beta``. + """ + new_iou_thr = max(self.train_cfg.dynamic_rcnn.initial_iou, + np.mean(self.iou_history)) + self.iou_history = [] + self.bbox_assigner.pos_iou_thr = new_iou_thr + self.bbox_assigner.neg_iou_thr = new_iou_thr + self.bbox_assigner.min_pos_iou = new_iou_thr + if (np.median(self.beta_history) < EPS): + # avoid 0 or too small value for new_beta + new_beta = self.bbox_head.loss_bbox.beta + else: + new_beta = min(self.train_cfg.dynamic_rcnn.initial_beta, + np.median(self.beta_history)) + self.beta_history = [] + self.bbox_head.loss_bbox.beta = new_beta + return new_iou_thr, new_beta diff --git a/detection/mmdet/models/roi_heads/grid_roi_head.py b/detection/mmdet/models/roi_heads/grid_roi_head.py new file mode 100644 index 0000000..4c52c79 --- /dev/null +++ b/detection/mmdet/models/roi_heads/grid_roi_head.py @@ -0,0 +1,176 @@ +import torch + +from mmdet.core import bbox2result, bbox2roi +from ..builder import HEADS, build_head, build_roi_extractor +from .standard_roi_head import StandardRoIHead + + +@HEADS.register_module() +class GridRoIHead(StandardRoIHead): + """Grid roi head for Grid R-CNN. + + https://arxiv.org/abs/1811.12030 + """ + + def __init__(self, grid_roi_extractor, grid_head, **kwargs): + assert grid_head is not None + super(GridRoIHead, self).__init__(**kwargs) + if grid_roi_extractor is not None: + self.grid_roi_extractor = build_roi_extractor(grid_roi_extractor) + self.share_roi_extractor = False + else: + self.share_roi_extractor = True + self.grid_roi_extractor = self.bbox_roi_extractor + self.grid_head = build_head(grid_head) + + def init_weights(self, pretrained): + """Initialize the weights in head. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + super(GridRoIHead, self).init_weights(pretrained) + self.grid_head.init_weights() + if not self.share_roi_extractor: + self.grid_roi_extractor.init_weights() + + def _random_jitter(self, sampling_results, img_metas, amplitude=0.15): + """Ramdom jitter positive proposals for training.""" + for sampling_result, img_meta in zip(sampling_results, img_metas): + bboxes = sampling_result.pos_bboxes + random_offsets = bboxes.new_empty(bboxes.shape[0], 4).uniform_( + -amplitude, amplitude) + # before jittering + cxcy = (bboxes[:, 2:4] + bboxes[:, :2]) / 2 + wh = (bboxes[:, 2:4] - bboxes[:, :2]).abs() + # after jittering + new_cxcy = cxcy + wh * random_offsets[:, :2] + new_wh = wh * (1 + random_offsets[:, 2:]) + # xywh to xyxy + new_x1y1 = (new_cxcy - new_wh / 2) + new_x2y2 = (new_cxcy + new_wh / 2) + new_bboxes = torch.cat([new_x1y1, new_x2y2], dim=1) + # clip bboxes + max_shape = img_meta['img_shape'] + if max_shape is not None: + new_bboxes[:, 0::2].clamp_(min=0, max=max_shape[1] - 1) + new_bboxes[:, 1::2].clamp_(min=0, max=max_shape[0] - 1) + + sampling_result.pos_bboxes = new_bboxes + return sampling_results + + def forward_dummy(self, x, proposals): + """Dummy forward function.""" + # bbox head + outs = () + rois = bbox2roi([proposals]) + if self.with_bbox: + bbox_results = self._bbox_forward(x, rois) + outs = outs + (bbox_results['cls_score'], + bbox_results['bbox_pred']) + + # grid head + grid_rois = rois[:100] + grid_feats = self.grid_roi_extractor( + x[:self.grid_roi_extractor.num_inputs], grid_rois) + if self.with_shared_head: + grid_feats = self.shared_head(grid_feats) + grid_pred = self.grid_head(grid_feats) + outs = outs + (grid_pred, ) + + # mask head + if self.with_mask: + mask_rois = rois[:100] + mask_results = self._mask_forward(x, mask_rois) + outs = outs + (mask_results['mask_pred'], ) + return outs + + def _bbox_forward_train(self, x, sampling_results, gt_bboxes, gt_labels, + img_metas): + """Run forward function and calculate loss for box head in training.""" + bbox_results = super(GridRoIHead, + self)._bbox_forward_train(x, sampling_results, + gt_bboxes, gt_labels, + img_metas) + + # Grid head forward and loss + sampling_results = self._random_jitter(sampling_results, img_metas) + pos_rois = bbox2roi([res.pos_bboxes for res in sampling_results]) + + # GN in head does not support zero shape input + if pos_rois.shape[0] == 0: + return bbox_results + + grid_feats = self.grid_roi_extractor( + x[:self.grid_roi_extractor.num_inputs], pos_rois) + if self.with_shared_head: + grid_feats = self.shared_head(grid_feats) + # Accelerate training + max_sample_num_grid = self.train_cfg.get('max_num_grid', 192) + sample_idx = torch.randperm( + grid_feats.shape[0])[:min(grid_feats.shape[0], max_sample_num_grid + )] + grid_feats = grid_feats[sample_idx] + + grid_pred = self.grid_head(grid_feats) + + grid_targets = self.grid_head.get_targets(sampling_results, + self.train_cfg) + grid_targets = grid_targets[sample_idx] + + loss_grid = self.grid_head.loss(grid_pred, grid_targets) + + bbox_results['loss_bbox'].update(loss_grid) + return bbox_results + + def simple_test(self, + x, + proposal_list, + img_metas, + proposals=None, + rescale=False): + """Test without augmentation.""" + assert self.with_bbox, 'Bbox head must be implemented.' + + det_bboxes, det_labels = self.simple_test_bboxes( + x, img_metas, proposal_list, self.test_cfg, rescale=False) + # pack rois into bboxes + grid_rois = bbox2roi([det_bbox[:, :4] for det_bbox in det_bboxes]) + if grid_rois.shape[0] != 0: + grid_feats = self.grid_roi_extractor( + x[:len(self.grid_roi_extractor.featmap_strides)], grid_rois) + self.grid_head.test_mode = True + grid_pred = self.grid_head(grid_feats) + # split batch grid head prediction back to each image + num_roi_per_img = tuple(len(det_bbox) for det_bbox in det_bboxes) + grid_pred = { + k: v.split(num_roi_per_img, 0) + for k, v in grid_pred.items() + } + + # apply bbox post-processing to each image individually + bbox_results = [] + num_imgs = len(det_bboxes) + for i in range(num_imgs): + if det_bboxes[i].shape[0] == 0: + bbox_results.append(grid_rois.new_tensor([])) + else: + det_bbox = self.grid_head.get_bboxes( + det_bboxes[i], grid_pred['fused'][i], [img_metas[i]]) + if rescale: + det_bbox[:, :4] /= img_metas[i]['scale_factor'] + bbox_results.append( + bbox2result(det_bbox, det_labels[i], + self.bbox_head.num_classes)) + else: + bbox_results = [ + grid_rois.new_tensor([]) for _ in range(len(det_bboxes)) + ] + + if not self.with_mask: + return bbox_results + else: + segm_results = self.simple_test_mask( + x, img_metas, det_bboxes, det_labels, rescale=rescale) + return list(zip(bbox_results, segm_results)) diff --git a/detection/mmdet/models/roi_heads/htc_roi_head.py b/detection/mmdet/models/roi_heads/htc_roi_head.py new file mode 100644 index 0000000..5b5c2ec --- /dev/null +++ b/detection/mmdet/models/roi_heads/htc_roi_head.py @@ -0,0 +1,589 @@ +import torch +import torch.nn.functional as F + +from mmdet.core import (bbox2result, bbox2roi, bbox_mapping, merge_aug_bboxes, + merge_aug_masks, multiclass_nms) +from ..builder import HEADS, build_head, build_roi_extractor +from .cascade_roi_head import CascadeRoIHead + + +@HEADS.register_module() +class HybridTaskCascadeRoIHead(CascadeRoIHead): + """Hybrid task cascade roi head including one bbox head and one mask head. + + https://arxiv.org/abs/1901.07518 + """ + + def __init__(self, + num_stages, + stage_loss_weights, + semantic_roi_extractor=None, + semantic_head=None, + semantic_fusion=('bbox', 'mask'), + interleaved=True, + mask_info_flow=True, + **kwargs): + super(HybridTaskCascadeRoIHead, + self).__init__(num_stages, stage_loss_weights, **kwargs) + assert self.with_bbox and self.with_mask + assert not self.with_shared_head # shared head is not supported + + if semantic_head is not None: + self.semantic_roi_extractor = build_roi_extractor( + semantic_roi_extractor) + self.semantic_head = build_head(semantic_head) + + self.semantic_fusion = semantic_fusion + self.interleaved = interleaved + self.mask_info_flow = mask_info_flow + + def init_weights(self, pretrained): + """Initialize the weights in head. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + super(HybridTaskCascadeRoIHead, self).init_weights(pretrained) + if self.with_semantic: + self.semantic_head.init_weights() + + @property + def with_semantic(self): + """bool: whether the head has semantic head""" + if hasattr(self, 'semantic_head') and self.semantic_head is not None: + return True + else: + return False + + def forward_dummy(self, x, proposals): + """Dummy forward function.""" + outs = () + # semantic head + if self.with_semantic: + _, semantic_feat = self.semantic_head(x) + else: + semantic_feat = None + # bbox heads + rois = bbox2roi([proposals]) + for i in range(self.num_stages): + bbox_results = self._bbox_forward( + i, x, rois, semantic_feat=semantic_feat) + outs = outs + (bbox_results['cls_score'], + bbox_results['bbox_pred']) + # mask heads + if self.with_mask: + mask_rois = rois[:100] + mask_roi_extractor = self.mask_roi_extractor[-1] + mask_feats = mask_roi_extractor( + x[:len(mask_roi_extractor.featmap_strides)], mask_rois) + if self.with_semantic and 'mask' in self.semantic_fusion: + mask_semantic_feat = self.semantic_roi_extractor( + [semantic_feat], mask_rois) + mask_feats += mask_semantic_feat + last_feat = None + for i in range(self.num_stages): + mask_head = self.mask_head[i] + if self.mask_info_flow: + mask_pred, last_feat = mask_head(mask_feats, last_feat) + else: + mask_pred = mask_head(mask_feats) + outs = outs + (mask_pred, ) + return outs + + def _bbox_forward_train(self, + stage, + x, + sampling_results, + gt_bboxes, + gt_labels, + rcnn_train_cfg, + semantic_feat=None): + """Run forward function and calculate loss for box head in training.""" + bbox_head = self.bbox_head[stage] + rois = bbox2roi([res.bboxes for res in sampling_results]) + bbox_results = self._bbox_forward( + stage, x, rois, semantic_feat=semantic_feat) + + bbox_targets = bbox_head.get_targets(sampling_results, gt_bboxes, + gt_labels, rcnn_train_cfg) + loss_bbox = bbox_head.loss(bbox_results['cls_score'], + bbox_results['bbox_pred'], rois, + *bbox_targets) + + bbox_results.update( + loss_bbox=loss_bbox, + rois=rois, + bbox_targets=bbox_targets, + ) + return bbox_results + + def _mask_forward_train(self, + stage, + x, + sampling_results, + gt_masks, + rcnn_train_cfg, + semantic_feat=None): + """Run forward function and calculate loss for mask head in + training.""" + mask_roi_extractor = self.mask_roi_extractor[stage] + mask_head = self.mask_head[stage] + pos_rois = bbox2roi([res.pos_bboxes for res in sampling_results]) + mask_feats = mask_roi_extractor(x[:mask_roi_extractor.num_inputs], + pos_rois) + + # semantic feature fusion + # element-wise sum for original features and pooled semantic features + if self.with_semantic and 'mask' in self.semantic_fusion: + mask_semantic_feat = self.semantic_roi_extractor([semantic_feat], + pos_rois) + if mask_semantic_feat.shape[-2:] != mask_feats.shape[-2:]: + mask_semantic_feat = F.adaptive_avg_pool2d( + mask_semantic_feat, mask_feats.shape[-2:]) + mask_feats += mask_semantic_feat + + # mask information flow + # forward all previous mask heads to obtain last_feat, and fuse it + # with the normal mask feature + if self.mask_info_flow: + last_feat = None + for i in range(stage): + last_feat = self.mask_head[i]( + mask_feats, last_feat, return_logits=False) + mask_pred = mask_head(mask_feats, last_feat, return_feat=False) + else: + mask_pred = mask_head(mask_feats, return_feat=False) + + mask_targets = mask_head.get_targets(sampling_results, gt_masks, + rcnn_train_cfg) + pos_labels = torch.cat([res.pos_gt_labels for res in sampling_results]) + loss_mask = mask_head.loss(mask_pred, mask_targets, pos_labels) + + mask_results = dict(loss_mask=loss_mask) + return mask_results + + def _bbox_forward(self, stage, x, rois, semantic_feat=None): + """Box head forward function used in both training and testing.""" + bbox_roi_extractor = self.bbox_roi_extractor[stage] + bbox_head = self.bbox_head[stage] + bbox_feats = bbox_roi_extractor( + x[:len(bbox_roi_extractor.featmap_strides)], rois) + if self.with_semantic and 'bbox' in self.semantic_fusion: + bbox_semantic_feat = self.semantic_roi_extractor([semantic_feat], + rois) + if bbox_semantic_feat.shape[-2:] != bbox_feats.shape[-2:]: + bbox_semantic_feat = F.adaptive_avg_pool2d( + bbox_semantic_feat, bbox_feats.shape[-2:]) + bbox_feats += bbox_semantic_feat + cls_score, bbox_pred = bbox_head(bbox_feats) + + bbox_results = dict(cls_score=cls_score, bbox_pred=bbox_pred) + return bbox_results + + def _mask_forward_test(self, stage, x, bboxes, semantic_feat=None): + """Mask head forward function for testing.""" + mask_roi_extractor = self.mask_roi_extractor[stage] + mask_head = self.mask_head[stage] + mask_rois = bbox2roi([bboxes]) + mask_feats = mask_roi_extractor( + x[:len(mask_roi_extractor.featmap_strides)], mask_rois) + if self.with_semantic and 'mask' in self.semantic_fusion: + mask_semantic_feat = self.semantic_roi_extractor([semantic_feat], + mask_rois) + if mask_semantic_feat.shape[-2:] != mask_feats.shape[-2:]: + mask_semantic_feat = F.adaptive_avg_pool2d( + mask_semantic_feat, mask_feats.shape[-2:]) + mask_feats += mask_semantic_feat + if self.mask_info_flow: + last_feat = None + last_pred = None + for i in range(stage): + mask_pred, last_feat = self.mask_head[i](mask_feats, last_feat) + if last_pred is not None: + mask_pred = mask_pred + last_pred + last_pred = mask_pred + mask_pred = mask_head(mask_feats, last_feat, return_feat=False) + if last_pred is not None: + mask_pred = mask_pred + last_pred + else: + mask_pred = mask_head(mask_feats) + return mask_pred + + def forward_train(self, + x, + img_metas, + proposal_list, + gt_bboxes, + gt_labels, + gt_bboxes_ignore=None, + gt_masks=None, + gt_semantic_seg=None): + """ + Args: + x (list[Tensor]): list of multi-level img features. + + img_metas (list[dict]): list of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmdet/datasets/pipelines/formatting.py:Collect`. + + proposal_list (list[Tensors]): list of region proposals. + + gt_bboxes (list[Tensor]): Ground truth bboxes for each image with + shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + + gt_labels (list[Tensor]): class indices corresponding to each box + + gt_bboxes_ignore (None, list[Tensor]): specify which bounding + boxes can be ignored when computing the loss. + + gt_masks (None, Tensor) : true segmentation masks for each box + used if the architecture supports a segmentation task. + + gt_semantic_seg (None, list[Tensor]): semantic segmentation masks + used if the architecture supports semantic segmentation task. + + Returns: + dict[str, Tensor]: a dictionary of loss components + """ + # semantic segmentation part + # 2 outputs: segmentation prediction and embedded features + losses = dict() + if self.with_semantic: + semantic_pred, semantic_feat = self.semantic_head(x) + loss_seg = self.semantic_head.loss(semantic_pred, gt_semantic_seg) + losses['loss_semantic_seg'] = loss_seg + else: + semantic_feat = None + + for i in range(self.num_stages): + self.current_stage = i + rcnn_train_cfg = self.train_cfg[i] + lw = self.stage_loss_weights[i] + + # assign gts and sample proposals + sampling_results = [] + bbox_assigner = self.bbox_assigner[i] + bbox_sampler = self.bbox_sampler[i] + num_imgs = len(img_metas) + if gt_bboxes_ignore is None: + gt_bboxes_ignore = [None for _ in range(num_imgs)] + + for j in range(num_imgs): + assign_result = bbox_assigner.assign(proposal_list[j], + gt_bboxes[j], + gt_bboxes_ignore[j], + gt_labels[j]) + sampling_result = bbox_sampler.sample( + assign_result, + proposal_list[j], + gt_bboxes[j], + gt_labels[j], + feats=[lvl_feat[j][None] for lvl_feat in x]) + sampling_results.append(sampling_result) + + # bbox head forward and loss + bbox_results = \ + self._bbox_forward_train( + i, x, sampling_results, gt_bboxes, gt_labels, + rcnn_train_cfg, semantic_feat) + roi_labels = bbox_results['bbox_targets'][0] + + for name, value in bbox_results['loss_bbox'].items(): + losses[f's{i}.{name}'] = ( + value * lw if 'loss' in name else value) + + # mask head forward and loss + if self.with_mask: + # interleaved execution: use regressed bboxes by the box branch + # to train the mask branch + if self.interleaved: + pos_is_gts = [res.pos_is_gt for res in sampling_results] + with torch.no_grad(): + proposal_list = self.bbox_head[i].refine_bboxes( + bbox_results['rois'], roi_labels, + bbox_results['bbox_pred'], pos_is_gts, img_metas) + # re-assign and sample 512 RoIs from 512 RoIs + sampling_results = [] + for j in range(num_imgs): + assign_result = bbox_assigner.assign( + proposal_list[j], gt_bboxes[j], + gt_bboxes_ignore[j], gt_labels[j]) + sampling_result = bbox_sampler.sample( + assign_result, + proposal_list[j], + gt_bboxes[j], + gt_labels[j], + feats=[lvl_feat[j][None] for lvl_feat in x]) + sampling_results.append(sampling_result) + mask_results = self._mask_forward_train( + i, x, sampling_results, gt_masks, rcnn_train_cfg, + semantic_feat) + for name, value in mask_results['loss_mask'].items(): + losses[f's{i}.{name}'] = ( + value * lw if 'loss' in name else value) + + # refine bboxes (same as Cascade R-CNN) + if i < self.num_stages - 1 and not self.interleaved: + pos_is_gts = [res.pos_is_gt for res in sampling_results] + with torch.no_grad(): + proposal_list = self.bbox_head[i].refine_bboxes( + bbox_results['rois'], roi_labels, + bbox_results['bbox_pred'], pos_is_gts, img_metas) + + return losses + + def simple_test(self, x, proposal_list, img_metas, rescale=False): + """Test without augmentation.""" + if self.with_semantic: + _, semantic_feat = self.semantic_head(x) + else: + semantic_feat = None + + num_imgs = len(proposal_list) + img_shapes = tuple(meta['img_shape'] for meta in img_metas) + ori_shapes = tuple(meta['ori_shape'] for meta in img_metas) + scale_factors = tuple(meta['scale_factor'] for meta in img_metas) + + # "ms" in variable names means multi-stage + ms_bbox_result = {} + ms_segm_result = {} + ms_scores = [] + rcnn_test_cfg = self.test_cfg + + rois = bbox2roi(proposal_list) + for i in range(self.num_stages): + bbox_head = self.bbox_head[i] + bbox_results = self._bbox_forward( + i, x, rois, semantic_feat=semantic_feat) + # split batch bbox prediction back to each image + cls_score = bbox_results['cls_score'] + bbox_pred = bbox_results['bbox_pred'] + num_proposals_per_img = tuple(len(p) for p in proposal_list) + rois = rois.split(num_proposals_per_img, 0) + cls_score = cls_score.split(num_proposals_per_img, 0) + bbox_pred = bbox_pred.split(num_proposals_per_img, 0) + ms_scores.append(cls_score) + + if i < self.num_stages - 1: + bbox_label = [s[:, :-1].argmax(dim=1) for s in cls_score] + rois = torch.cat([ + bbox_head.regress_by_class(rois[i], bbox_label[i], + bbox_pred[i], img_metas[i]) + for i in range(num_imgs) + ]) + + # average scores of each image by stages + cls_score = [ + sum([score[i] for score in ms_scores]) / float(len(ms_scores)) + for i in range(num_imgs) + ] + + # apply bbox post-processing to each image individually + det_bboxes = [] + det_labels = [] + for i in range(num_imgs): + det_bbox, det_label = self.bbox_head[-1].get_bboxes( + rois[i], + cls_score[i], + bbox_pred[i], + img_shapes[i], + scale_factors[i], + rescale=rescale, + cfg=rcnn_test_cfg) + det_bboxes.append(det_bbox) + det_labels.append(det_label) + bbox_result = [ + bbox2result(det_bboxes[i], det_labels[i], + self.bbox_head[-1].num_classes) + for i in range(num_imgs) + ] + ms_bbox_result['ensemble'] = bbox_result + + if self.with_mask: + if all(det_bbox.shape[0] == 0 for det_bbox in det_bboxes): + mask_classes = self.mask_head[-1].num_classes + segm_results = [[[] for _ in range(mask_classes)] + for _ in range(num_imgs)] + else: + if rescale and not isinstance(scale_factors[0], float): + scale_factors = [ + torch.from_numpy(scale_factor).to(det_bboxes[0].device) + for scale_factor in scale_factors + ] + _bboxes = [ + det_bboxes[i][:, :4] * + scale_factors[i] if rescale else det_bboxes[i] + for i in range(num_imgs) + ] + mask_rois = bbox2roi(_bboxes) + aug_masks = [] + mask_roi_extractor = self.mask_roi_extractor[-1] + mask_feats = mask_roi_extractor( + x[:len(mask_roi_extractor.featmap_strides)], mask_rois) + if self.with_semantic and 'mask' in self.semantic_fusion: + mask_semantic_feat = self.semantic_roi_extractor( + [semantic_feat], mask_rois) + mask_feats += mask_semantic_feat + last_feat = None + + num_bbox_per_img = tuple(len(_bbox) for _bbox in _bboxes) + for i in range(self.num_stages): + mask_head = self.mask_head[i] + if self.mask_info_flow: + mask_pred, last_feat = mask_head(mask_feats, last_feat) + else: + mask_pred = mask_head(mask_feats) + + # split batch mask prediction back to each image + mask_pred = mask_pred.split(num_bbox_per_img, 0) + aug_masks.append( + [mask.sigmoid().cpu().numpy() for mask in mask_pred]) + + # apply mask post-processing to each image individually + segm_results = [] + for i in range(num_imgs): + if det_bboxes[i].shape[0] == 0: + segm_results.append( + [[] + for _ in range(self.mask_head[-1].num_classes)]) + else: + aug_mask = [mask[i] for mask in aug_masks] + merged_mask = merge_aug_masks( + aug_mask, [[img_metas[i]]] * self.num_stages, + rcnn_test_cfg) + segm_result = self.mask_head[-1].get_seg_masks( + merged_mask, _bboxes[i], det_labels[i], + rcnn_test_cfg, ori_shapes[i], scale_factors[i], + rescale) + segm_results.append(segm_result) + ms_segm_result['ensemble'] = segm_results + + if self.with_mask: + results = list( + zip(ms_bbox_result['ensemble'], ms_segm_result['ensemble'])) + else: + results = ms_bbox_result['ensemble'] + + return results + + def aug_test(self, img_feats, proposal_list, img_metas, rescale=False): + """Test with augmentations. + + If rescale is False, then returned bboxes and masks will fit the scale + of imgs[0]. + """ + if self.with_semantic: + semantic_feats = [ + self.semantic_head(feat)[1] for feat in img_feats + ] + else: + semantic_feats = [None] * len(img_metas) + + rcnn_test_cfg = self.test_cfg + aug_bboxes = [] + aug_scores = [] + for x, img_meta, semantic in zip(img_feats, img_metas, semantic_feats): + # only one image in the batch + img_shape = img_meta[0]['img_shape'] + scale_factor = img_meta[0]['scale_factor'] + flip = img_meta[0]['flip'] + flip_direction = img_meta[0]['flip_direction'] + + proposals = bbox_mapping(proposal_list[0][:, :4], img_shape, + scale_factor, flip, flip_direction) + # "ms" in variable names means multi-stage + ms_scores = [] + + rois = bbox2roi([proposals]) + for i in range(self.num_stages): + bbox_head = self.bbox_head[i] + bbox_results = self._bbox_forward( + i, x, rois, semantic_feat=semantic) + ms_scores.append(bbox_results['cls_score']) + + if i < self.num_stages - 1: + bbox_label = bbox_results['cls_score'].argmax(dim=1) + rois = bbox_head.regress_by_class( + rois, bbox_label, bbox_results['bbox_pred'], + img_meta[0]) + + cls_score = sum(ms_scores) / float(len(ms_scores)) + bboxes, scores = self.bbox_head[-1].get_bboxes( + rois, + cls_score, + bbox_results['bbox_pred'], + img_shape, + scale_factor, + rescale=False, + cfg=None) + aug_bboxes.append(bboxes) + aug_scores.append(scores) + + # after merging, bboxes will be rescaled to the original image size + merged_bboxes, merged_scores = merge_aug_bboxes( + aug_bboxes, aug_scores, img_metas, rcnn_test_cfg) + det_bboxes, det_labels = multiclass_nms(merged_bboxes, merged_scores, + rcnn_test_cfg.score_thr, + rcnn_test_cfg.nms, + rcnn_test_cfg.max_per_img) + + bbox_result = bbox2result(det_bboxes, det_labels, + self.bbox_head[-1].num_classes) + + if self.with_mask: + if det_bboxes.shape[0] == 0: + segm_result = [[[] + for _ in range(self.mask_head[-1].num_classes)] + ] + else: + aug_masks = [] + aug_img_metas = [] + for x, img_meta, semantic in zip(img_feats, img_metas, + semantic_feats): + img_shape = img_meta[0]['img_shape'] + scale_factor = img_meta[0]['scale_factor'] + flip = img_meta[0]['flip'] + flip_direction = img_meta[0]['flip_direction'] + _bboxes = bbox_mapping(det_bboxes[:, :4], img_shape, + scale_factor, flip, flip_direction) + mask_rois = bbox2roi([_bboxes]) + mask_feats = self.mask_roi_extractor[-1]( + x[:len(self.mask_roi_extractor[-1].featmap_strides)], + mask_rois) + if self.with_semantic: + semantic_feat = semantic + mask_semantic_feat = self.semantic_roi_extractor( + [semantic_feat], mask_rois) + if mask_semantic_feat.shape[-2:] != mask_feats.shape[ + -2:]: + mask_semantic_feat = F.adaptive_avg_pool2d( + mask_semantic_feat, mask_feats.shape[-2:]) + mask_feats += mask_semantic_feat + last_feat = None + for i in range(self.num_stages): + mask_head = self.mask_head[i] + if self.mask_info_flow: + mask_pred, last_feat = mask_head( + mask_feats, last_feat) + else: + mask_pred = mask_head(mask_feats) + aug_masks.append(mask_pred.sigmoid().cpu().numpy()) + aug_img_metas.append(img_meta) + merged_masks = merge_aug_masks(aug_masks, aug_img_metas, + self.test_cfg) + + ori_shape = img_metas[0][0]['ori_shape'] + segm_result = self.mask_head[-1].get_seg_masks( + merged_masks, + det_bboxes, + det_labels, + rcnn_test_cfg, + ori_shape, + scale_factor=1.0, + rescale=False) + return [(bbox_result, segm_result)] + else: + return [bbox_result] diff --git a/detection/mmdet/models/roi_heads/mask_heads/__init__.py b/detection/mmdet/models/roi_heads/mask_heads/__init__.py new file mode 100644 index 0000000..abfbe26 --- /dev/null +++ b/detection/mmdet/models/roi_heads/mask_heads/__init__.py @@ -0,0 +1,17 @@ +from .coarse_mask_head import CoarseMaskHead +from .fcn_mask_head import FCNMaskHead +from .feature_relay_head import FeatureRelayHead +from .fused_semantic_head import FusedSemanticHead +from .global_context_head import GlobalContextHead +from .grid_head import GridHead +from .htc_mask_head import HTCMaskHead +from .mask_point_head import MaskPointHead +from .maskiou_head import MaskIoUHead +from .scnet_mask_head import SCNetMaskHead +from .scnet_semantic_head import SCNetSemanticHead + +__all__ = [ + 'FCNMaskHead', 'HTCMaskHead', 'FusedSemanticHead', 'GridHead', + 'MaskIoUHead', 'CoarseMaskHead', 'MaskPointHead', 'SCNetMaskHead', + 'SCNetSemanticHead', 'GlobalContextHead', 'FeatureRelayHead' +] diff --git a/detection/mmdet/models/roi_heads/mask_heads/coarse_mask_head.py b/detection/mmdet/models/roi_heads/mask_heads/coarse_mask_head.py new file mode 100644 index 0000000..d665dff --- /dev/null +++ b/detection/mmdet/models/roi_heads/mask_heads/coarse_mask_head.py @@ -0,0 +1,91 @@ +import torch.nn as nn +from mmcv.cnn import ConvModule, Linear, constant_init, xavier_init +from mmcv.runner import auto_fp16 + +from mmdet.models.builder import HEADS +from .fcn_mask_head import FCNMaskHead + + +@HEADS.register_module() +class CoarseMaskHead(FCNMaskHead): + """Coarse mask head used in PointRend. + + Compared with standard ``FCNMaskHead``, ``CoarseMaskHead`` will downsample + the input feature map instead of upsample it. + + Args: + num_convs (int): Number of conv layers in the head. Default: 0. + num_fcs (int): Number of fc layers in the head. Default: 2. + fc_out_channels (int): Number of output channels of fc layer. + Default: 1024. + downsample_factor (int): The factor that feature map is downsampled by. + Default: 2. + """ + + def __init__(self, + num_convs=0, + num_fcs=2, + fc_out_channels=1024, + downsample_factor=2, + *arg, + **kwarg): + super(CoarseMaskHead, self).__init__( + *arg, num_convs=num_convs, upsample_cfg=dict(type=None), **kwarg) + self.num_fcs = num_fcs + assert self.num_fcs > 0 + self.fc_out_channels = fc_out_channels + self.downsample_factor = downsample_factor + assert self.downsample_factor >= 1 + # remove conv_logit + delattr(self, 'conv_logits') + + if downsample_factor > 1: + downsample_in_channels = ( + self.conv_out_channels + if self.num_convs > 0 else self.in_channels) + self.downsample_conv = ConvModule( + downsample_in_channels, + self.conv_out_channels, + kernel_size=downsample_factor, + stride=downsample_factor, + padding=0, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg) + else: + self.downsample_conv = None + + self.output_size = (self.roi_feat_size[0] // downsample_factor, + self.roi_feat_size[1] // downsample_factor) + self.output_area = self.output_size[0] * self.output_size[1] + + last_layer_dim = self.conv_out_channels * self.output_area + + self.fcs = nn.ModuleList() + for i in range(num_fcs): + fc_in_channels = ( + last_layer_dim if i == 0 else self.fc_out_channels) + self.fcs.append(Linear(fc_in_channels, self.fc_out_channels)) + last_layer_dim = self.fc_out_channels + output_channels = self.num_classes * self.output_area + self.fc_logits = Linear(last_layer_dim, output_channels) + + def init_weights(self): + for m in self.fcs.modules(): + if isinstance(m, nn.Linear): + xavier_init(m) + constant_init(self.fc_logits, 0.001) + + @auto_fp16() + def forward(self, x): + for conv in self.convs: + x = conv(x) + + if self.downsample_conv is not None: + x = self.downsample_conv(x) + + x = x.flatten(1) + for fc in self.fcs: + x = self.relu(fc(x)) + mask_pred = self.fc_logits(x).view( + x.size(0), self.num_classes, *self.output_size) + return mask_pred diff --git a/detection/mmdet/models/roi_heads/mask_heads/fcn_mask_head.py b/detection/mmdet/models/roi_heads/mask_heads/fcn_mask_head.py new file mode 100644 index 0000000..be6772f --- /dev/null +++ b/detection/mmdet/models/roi_heads/mask_heads/fcn_mask_head.py @@ -0,0 +1,377 @@ +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import Conv2d, ConvModule, build_upsample_layer +from mmcv.ops.carafe import CARAFEPack +from mmcv.runner import auto_fp16, force_fp32 +from torch.nn.modules.utils import _pair + +from mmdet.core import mask_target +from mmdet.models.builder import HEADS, build_loss + +BYTES_PER_FLOAT = 4 +# TODO: This memory limit may be too much or too little. It would be better to +# determine it based on available resources. +GPU_MEM_LIMIT = 1024**3 # 1 GB memory limit + + +@HEADS.register_module() +class FCNMaskHead(nn.Module): + + def __init__(self, + num_convs=4, + roi_feat_size=14, + in_channels=256, + conv_kernel_size=3, + conv_out_channels=256, + num_classes=80, + class_agnostic=False, + upsample_cfg=dict(type='deconv', scale_factor=2), + conv_cfg=None, + norm_cfg=None, + loss_mask=dict( + type='CrossEntropyLoss', use_mask=True, loss_weight=1.0)): + super(FCNMaskHead, self).__init__() + self.upsample_cfg = upsample_cfg.copy() + if self.upsample_cfg['type'] not in [ + None, 'deconv', 'nearest', 'bilinear', 'carafe' + ]: + raise ValueError( + f'Invalid upsample method {self.upsample_cfg["type"]}, ' + 'accepted methods are "deconv", "nearest", "bilinear", ' + '"carafe"') + self.num_convs = num_convs + # WARN: roi_feat_size is reserved and not used + self.roi_feat_size = _pair(roi_feat_size) + self.in_channels = in_channels + self.conv_kernel_size = conv_kernel_size + self.conv_out_channels = conv_out_channels + self.upsample_method = self.upsample_cfg.get('type') + self.scale_factor = self.upsample_cfg.pop('scale_factor', None) + self.num_classes = num_classes + self.class_agnostic = class_agnostic + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.fp16_enabled = False + self.loss_mask = build_loss(loss_mask) + + self.convs = nn.ModuleList() + for i in range(self.num_convs): + in_channels = ( + self.in_channels if i == 0 else self.conv_out_channels) + padding = (self.conv_kernel_size - 1) // 2 + self.convs.append( + ConvModule( + in_channels, + self.conv_out_channels, + self.conv_kernel_size, + padding=padding, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg)) + upsample_in_channels = ( + self.conv_out_channels if self.num_convs > 0 else in_channels) + upsample_cfg_ = self.upsample_cfg.copy() + if self.upsample_method is None: + self.upsample = None + elif self.upsample_method == 'deconv': + upsample_cfg_.update( + in_channels=upsample_in_channels, + out_channels=self.conv_out_channels, + kernel_size=self.scale_factor, + stride=self.scale_factor) + self.upsample = build_upsample_layer(upsample_cfg_) + elif self.upsample_method == 'carafe': + upsample_cfg_.update( + channels=upsample_in_channels, scale_factor=self.scale_factor) + self.upsample = build_upsample_layer(upsample_cfg_) + else: + # suppress warnings + align_corners = (None + if self.upsample_method == 'nearest' else False) + upsample_cfg_.update( + scale_factor=self.scale_factor, + mode=self.upsample_method, + align_corners=align_corners) + self.upsample = build_upsample_layer(upsample_cfg_) + + out_channels = 1 if self.class_agnostic else self.num_classes + logits_in_channel = ( + self.conv_out_channels + if self.upsample_method == 'deconv' else upsample_in_channels) + self.conv_logits = Conv2d(logits_in_channel, out_channels, 1) + self.relu = nn.ReLU(inplace=True) + self.debug_imgs = None + + def init_weights(self): + for m in [self.upsample, self.conv_logits]: + if m is None: + continue + elif isinstance(m, CARAFEPack): + m.init_weights() + else: + nn.init.kaiming_normal_( + m.weight, mode='fan_out', nonlinearity='relu') + nn.init.constant_(m.bias, 0) + + @auto_fp16() + def forward(self, x): + for conv in self.convs: + x = conv(x) + if self.upsample is not None: + x = self.upsample(x) + if self.upsample_method == 'deconv': + x = self.relu(x) + mask_pred = self.conv_logits(x) + return mask_pred + + def get_targets(self, sampling_results, gt_masks, rcnn_train_cfg): + pos_proposals = [res.pos_bboxes for res in sampling_results] + pos_assigned_gt_inds = [ + res.pos_assigned_gt_inds for res in sampling_results + ] + mask_targets = mask_target(pos_proposals, pos_assigned_gt_inds, + gt_masks, rcnn_train_cfg) + return mask_targets + + @force_fp32(apply_to=('mask_pred', )) + def loss(self, mask_pred, mask_targets, labels): + """ + Example: + >>> from mmdet.models.roi_heads.mask_heads.fcn_mask_head import * # NOQA + >>> N = 7 # N = number of extracted ROIs + >>> C, H, W = 11, 32, 32 + >>> # Create example instance of FCN Mask Head. + >>> # There are lots of variations depending on the configuration + >>> self = FCNMaskHead(num_classes=C, num_convs=1) + >>> inputs = torch.rand(N, self.in_channels, H, W) + >>> mask_pred = self.forward(inputs) + >>> sf = self.scale_factor + >>> labels = torch.randint(0, C, size=(N,)) + >>> # With the default properties the mask targets should indicate + >>> # a (potentially soft) single-class label + >>> mask_targets = torch.rand(N, H * sf, W * sf) + >>> loss = self.loss(mask_pred, mask_targets, labels) + >>> print('loss = {!r}'.format(loss)) + """ + loss = dict() + if mask_pred.size(0) == 0: + loss_mask = mask_pred.sum() + else: + if self.class_agnostic: + loss_mask = self.loss_mask(mask_pred, mask_targets, + torch.zeros_like(labels)) + else: + loss_mask = self.loss_mask(mask_pred, mask_targets, labels) + loss['loss_mask'] = loss_mask + return loss + + def get_seg_masks(self, mask_pred, det_bboxes, det_labels, rcnn_test_cfg, + ori_shape, scale_factor, rescale): + """Get segmentation masks from mask_pred and bboxes. + + Args: + mask_pred (Tensor or ndarray): shape (n, #class, h, w). + For single-scale testing, mask_pred is the direct output of + model, whose type is Tensor, while for multi-scale testing, + it will be converted to numpy array outside of this method. + det_bboxes (Tensor): shape (n, 4/5) + det_labels (Tensor): shape (n, ) + rcnn_test_cfg (dict): rcnn testing config + ori_shape (Tuple): original image height and width, shape (2,) + scale_factor(float | Tensor): If ``rescale is True``, box + coordinates are divided by this scale factor to fit + ``ori_shape``. + rescale (bool): If True, the resulting masks will be rescaled to + ``ori_shape``. + + Returns: + list[list]: encoded masks. The c-th item in the outer list + corresponds to the c-th class. Given the c-th outer list, the + i-th item in that inner list is the mask for the i-th box with + class label c. + + Example: + >>> import mmcv + >>> from mmdet.models.roi_heads.mask_heads.fcn_mask_head import * # NOQA + >>> N = 7 # N = number of extracted ROIs + >>> C, H, W = 11, 32, 32 + >>> # Create example instance of FCN Mask Head. + >>> self = FCNMaskHead(num_classes=C, num_convs=0) + >>> inputs = torch.rand(N, self.in_channels, H, W) + >>> mask_pred = self.forward(inputs) + >>> # Each input is associated with some bounding box + >>> det_bboxes = torch.Tensor([[1, 1, 42, 42 ]] * N) + >>> det_labels = torch.randint(0, C, size=(N,)) + >>> rcnn_test_cfg = mmcv.Config({'mask_thr_binary': 0, }) + >>> ori_shape = (H * 4, W * 4) + >>> scale_factor = torch.FloatTensor((1, 1)) + >>> rescale = False + >>> # Encoded masks are a list for each category. + >>> encoded_masks = self.get_seg_masks( + >>> mask_pred, det_bboxes, det_labels, rcnn_test_cfg, ori_shape, + >>> scale_factor, rescale + >>> ) + >>> assert len(encoded_masks) == C + >>> assert sum(list(map(len, encoded_masks))) == N + """ + if isinstance(mask_pred, torch.Tensor): + mask_pred = mask_pred.sigmoid() + else: + mask_pred = det_bboxes.new_tensor(mask_pred) + + device = mask_pred.device + cls_segms = [[] for _ in range(self.num_classes) + ] # BG is not included in num_classes + bboxes = det_bboxes[:, :4] + labels = det_labels + + if rescale: + img_h, img_w = ori_shape[:2] + else: + if isinstance(scale_factor, float): + img_h = np.round(ori_shape[0] * scale_factor).astype(np.int32) + img_w = np.round(ori_shape[1] * scale_factor).astype(np.int32) + else: + w_scale, h_scale = scale_factor[0], scale_factor[1] + img_h = np.round(ori_shape[0] * h_scale.item()).astype( + np.int32) + img_w = np.round(ori_shape[1] * w_scale.item()).astype( + np.int32) + scale_factor = 1.0 + + if not isinstance(scale_factor, (float, torch.Tensor)): + scale_factor = bboxes.new_tensor(scale_factor) + bboxes = bboxes / scale_factor + + if torch.onnx.is_in_onnx_export(): + # TODO: Remove after F.grid_sample is supported. + from torchvision.models.detection.roi_heads \ + import paste_masks_in_image + masks = paste_masks_in_image(mask_pred, bboxes, ori_shape[:2]) + thr = rcnn_test_cfg.get('mask_thr_binary', 0) + if thr > 0: + masks = masks >= thr + return masks + + N = len(mask_pred) + # The actual implementation split the input into chunks, + # and paste them chunk by chunk. + if device.type == 'cpu': + # CPU is most efficient when they are pasted one by one with + # skip_empty=True, so that it performs minimal number of + # operations. + num_chunks = N + else: + # GPU benefits from parallelism for larger chunks, + # but may have memory issue + num_chunks = int( + np.ceil(N * img_h * img_w * BYTES_PER_FLOAT / GPU_MEM_LIMIT)) + assert (num_chunks <= + N), 'Default GPU_MEM_LIMIT is too small; try increasing it' + chunks = torch.chunk(torch.arange(N, device=device), num_chunks) + + threshold = rcnn_test_cfg.mask_thr_binary + im_mask = torch.zeros( + N, + img_h, + img_w, + device=device, + dtype=torch.bool if threshold >= 0 else torch.uint8) + + if not self.class_agnostic: + mask_pred = mask_pred[range(N), labels][:, None] + + for inds in chunks: + masks_chunk, spatial_inds = _do_paste_mask( + mask_pred[inds], + bboxes[inds], + img_h, + img_w, + skip_empty=device.type == 'cpu') + + if threshold >= 0: + masks_chunk = (masks_chunk >= threshold).to(dtype=torch.bool) + else: + # for visualization and debugging + masks_chunk = (masks_chunk * 255).to(dtype=torch.uint8) + + im_mask[(inds, ) + spatial_inds] = masks_chunk + + for i in range(N): + cls_segms[labels[i]].append(im_mask[i].detach().cpu().numpy()) + return cls_segms + + +def _do_paste_mask(masks, boxes, img_h, img_w, skip_empty=True): + """Paste instance masks according to boxes. + + This implementation is modified from + https://github.com/facebookresearch/detectron2/ + + Args: + masks (Tensor): N, 1, H, W + boxes (Tensor): N, 4 + img_h (int): Height of the image to be pasted. + img_w (int): Width of the image to be pasted. + skip_empty (bool): Only paste masks within the region that + tightly bound all boxes, and returns the results this region only. + An important optimization for CPU. + + Returns: + tuple: (Tensor, tuple). The first item is mask tensor, the second one + is the slice object. + If skip_empty == False, the whole image will be pasted. It will + return a mask of shape (N, img_h, img_w) and an empty tuple. + If skip_empty == True, only area around the mask will be pasted. + A mask of shape (N, h', w') and its start and end coordinates + in the original image will be returned. + """ + # On GPU, paste all masks together (up to chunk size) + # by using the entire image to sample the masks + # Compared to pasting them one by one, + # this has more operations but is faster on COCO-scale dataset. + device = masks.device + if skip_empty: + x0_int, y0_int = torch.clamp( + boxes.min(dim=0).values.floor()[:2] - 1, + min=0).to(dtype=torch.int32) + x1_int = torch.clamp( + boxes[:, 2].max().ceil() + 1, max=img_w).to(dtype=torch.int32) + y1_int = torch.clamp( + boxes[:, 3].max().ceil() + 1, max=img_h).to(dtype=torch.int32) + else: + x0_int, y0_int = 0, 0 + x1_int, y1_int = img_w, img_h + x0, y0, x1, y1 = torch.split(boxes, 1, dim=1) # each is Nx1 + + N = masks.shape[0] + + img_y = torch.arange( + y0_int, y1_int, device=device, dtype=torch.float32) + 0.5 + img_x = torch.arange( + x0_int, x1_int, device=device, dtype=torch.float32) + 0.5 + img_y = (img_y - y0) / (y1 - y0) * 2 - 1 + img_x = (img_x - x0) / (x1 - x0) * 2 - 1 + # img_x, img_y have shapes (N, w), (N, h) + if torch.isinf(img_x).any(): + inds = torch.where(torch.isinf(img_x)) + img_x[inds] = 0 + if torch.isinf(img_y).any(): + inds = torch.where(torch.isinf(img_y)) + img_y[inds] = 0 + + gx = img_x[:, None, :].expand(N, img_y.size(1), img_x.size(1)) + gy = img_y[:, :, None].expand(N, img_y.size(1), img_x.size(1)) + grid = torch.stack([gx, gy], dim=3) + + if torch.onnx.is_in_onnx_export(): + raise RuntimeError( + 'Exporting F.grid_sample from Pytorch to ONNX is not supported.') + img_masks = F.grid_sample( + masks.to(dtype=torch.float32), grid, align_corners=False) + + if skip_empty: + return img_masks[:, 0], (slice(y0_int, y1_int), slice(x0_int, x1_int)) + else: + return img_masks[:, 0], () diff --git a/detection/mmdet/models/roi_heads/mask_heads/feature_relay_head.py b/detection/mmdet/models/roi_heads/mask_heads/feature_relay_head.py new file mode 100644 index 0000000..a1cfb2c --- /dev/null +++ b/detection/mmdet/models/roi_heads/mask_heads/feature_relay_head.py @@ -0,0 +1,55 @@ +import torch.nn as nn +from mmcv.cnn import kaiming_init +from mmcv.runner import auto_fp16 + +from mmdet.models.builder import HEADS + + +@HEADS.register_module() +class FeatureRelayHead(nn.Module): + """Feature Relay Head used in `SCNet `_. + + Args: + in_channels (int, optional): number of input channels. Default: 256. + conv_out_channels (int, optional): number of output channels before + classification layer. Default: 256. + roi_feat_size (int, optional): roi feat size at box head. Default: 7. + scale_factor (int, optional): scale factor to match roi feat size + at mask head. Default: 2. + """ + + def __init__(self, + in_channels=1024, + out_conv_channels=256, + roi_feat_size=7, + scale_factor=2): + super(FeatureRelayHead, self).__init__() + assert isinstance(roi_feat_size, int) + + self.in_channels = in_channels + self.out_conv_channels = out_conv_channels + self.roi_feat_size = roi_feat_size + self.out_channels = (roi_feat_size**2) * out_conv_channels + self.scale_factor = scale_factor + self.fp16_enabled = False + + self.fc = nn.Linear(self.in_channels, self.out_channels) + self.upsample = nn.Upsample( + scale_factor=scale_factor, mode='bilinear', align_corners=True) + + def init_weights(self): + """Init weights for the head.""" + kaiming_init(self.fc) + + @auto_fp16() + def forward(self, x): + """Forward function.""" + N, in_C = x.shape + if N > 0: + out_C = self.out_conv_channels + out_HW = self.roi_feat_size + x = self.fc(x) + x = x.reshape(N, out_C, out_HW, out_HW) + x = self.upsample(x) + return x + return None diff --git a/detection/mmdet/models/roi_heads/mask_heads/fused_semantic_head.py b/detection/mmdet/models/roi_heads/mask_heads/fused_semantic_head.py new file mode 100644 index 0000000..2aa6033 --- /dev/null +++ b/detection/mmdet/models/roi_heads/mask_heads/fused_semantic_head.py @@ -0,0 +1,107 @@ +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import ConvModule, kaiming_init +from mmcv.runner import auto_fp16, force_fp32 + +from mmdet.models.builder import HEADS + + +@HEADS.register_module() +class FusedSemanticHead(nn.Module): + r"""Multi-level fused semantic segmentation head. + + .. code-block:: none + + in_1 -> 1x1 conv --- + | + in_2 -> 1x1 conv -- | + || + in_3 -> 1x1 conv - || + ||| /-> 1x1 conv (mask prediction) + in_4 -> 1x1 conv -----> 3x3 convs (*4) + | \-> 1x1 conv (feature) + in_5 -> 1x1 conv --- + """ # noqa: W605 + + def __init__(self, + num_ins, + fusion_level, + num_convs=4, + in_channels=256, + conv_out_channels=256, + num_classes=183, + ignore_label=255, + loss_weight=0.2, + conv_cfg=None, + norm_cfg=None): + super(FusedSemanticHead, self).__init__() + self.num_ins = num_ins + self.fusion_level = fusion_level + self.num_convs = num_convs + self.in_channels = in_channels + self.conv_out_channels = conv_out_channels + self.num_classes = num_classes + self.ignore_label = ignore_label + self.loss_weight = loss_weight + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.fp16_enabled = False + + self.lateral_convs = nn.ModuleList() + for i in range(self.num_ins): + self.lateral_convs.append( + ConvModule( + self.in_channels, + self.in_channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + inplace=False)) + + self.convs = nn.ModuleList() + for i in range(self.num_convs): + in_channels = self.in_channels if i == 0 else conv_out_channels + self.convs.append( + ConvModule( + in_channels, + conv_out_channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg)) + self.conv_embedding = ConvModule( + conv_out_channels, + conv_out_channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg) + self.conv_logits = nn.Conv2d(conv_out_channels, self.num_classes, 1) + + self.criterion = nn.CrossEntropyLoss(ignore_index=ignore_label) + + def init_weights(self): + kaiming_init(self.conv_logits) + + @auto_fp16() + def forward(self, feats): + x = self.lateral_convs[self.fusion_level](feats[self.fusion_level]) + fused_size = tuple(x.shape[-2:]) + for i, feat in enumerate(feats): + if i != self.fusion_level: + feat = F.interpolate( + feat, size=fused_size, mode='bilinear', align_corners=True) + x += self.lateral_convs[i](feat) + + for i in range(self.num_convs): + x = self.convs[i](x) + + mask_pred = self.conv_logits(x) + x = self.conv_embedding(x) + return mask_pred, x + + @force_fp32(apply_to=('mask_pred', )) + def loss(self, mask_pred, labels): + labels = labels.squeeze(1).long() + loss_semantic_seg = self.criterion(mask_pred, labels) + loss_semantic_seg *= self.loss_weight + return loss_semantic_seg diff --git a/detection/mmdet/models/roi_heads/mask_heads/global_context_head.py b/detection/mmdet/models/roi_heads/mask_heads/global_context_head.py new file mode 100644 index 0000000..d8e8cbc --- /dev/null +++ b/detection/mmdet/models/roi_heads/mask_heads/global_context_head.py @@ -0,0 +1,102 @@ +import torch.nn as nn +from mmcv.cnn import ConvModule +from mmcv.runner import auto_fp16, force_fp32 + +from mmdet.models.builder import HEADS +from mmdet.models.utils import ResLayer, SimplifiedBasicBlock + + +@HEADS.register_module() +class GlobalContextHead(nn.Module): + """Global context head used in `SCNet `_. + + Args: + num_convs (int, optional): number of convolutional layer in GlbCtxHead. + Default: 4. + in_channels (int, optional): number of input channels. Default: 256. + conv_out_channels (int, optional): number of output channels before + classification layer. Default: 256. + num_classes (int, optional): number of classes. Default: 80. + loss_weight (float, optional): global context loss weight. Default: 1. + conv_cfg (dict, optional): config to init conv layer. Default: None. + norm_cfg (dict, optional): config to init norm layer. Default: None. + conv_to_res (bool, optional): if True, 2 convs will be grouped into + 1 `SimplifiedBasicBlock` using a skip connection. Default: False. + """ + + def __init__(self, + num_convs=4, + in_channels=256, + conv_out_channels=256, + num_classes=80, + loss_weight=1.0, + conv_cfg=None, + norm_cfg=None, + conv_to_res=False): + super(GlobalContextHead, self).__init__() + self.num_convs = num_convs + self.in_channels = in_channels + self.conv_out_channels = conv_out_channels + self.num_classes = num_classes + self.loss_weight = loss_weight + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.conv_to_res = conv_to_res + self.fp16_enabled = False + + if self.conv_to_res: + num_res_blocks = num_convs // 2 + self.convs = ResLayer( + SimplifiedBasicBlock, + in_channels, + self.conv_out_channels, + num_res_blocks, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg) + self.num_convs = num_res_blocks + else: + self.convs = nn.ModuleList() + for i in range(self.num_convs): + in_channels = self.in_channels if i == 0 else conv_out_channels + self.convs.append( + ConvModule( + in_channels, + conv_out_channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg)) + + self.pool = nn.AdaptiveAvgPool2d(1) + self.fc = nn.Linear(conv_out_channels, num_classes) + + self.criterion = nn.BCEWithLogitsLoss() + + def init_weights(self): + """Init weights for the head.""" + nn.init.normal_(self.fc.weight, 0, 0.01) + nn.init.constant_(self.fc.bias, 0) + + @auto_fp16() + def forward(self, feats): + """Forward function.""" + x = feats[-1] + for i in range(self.num_convs): + x = self.convs[i](x) + x = self.pool(x) + + # multi-class prediction + mc_pred = x.reshape(x.size(0), -1) + mc_pred = self.fc(mc_pred) + + return mc_pred, x + + @force_fp32(apply_to=('pred', )) + def loss(self, pred, labels): + """Loss function.""" + labels = [lbl.unique() for lbl in labels] + targets = pred.new_zeros(pred.size()) + for i, label in enumerate(labels): + targets[i, label] = 1.0 + loss = self.loss_weight * self.criterion(pred, targets) + return loss diff --git a/detection/mmdet/models/roi_heads/mask_heads/grid_head.py b/detection/mmdet/models/roi_heads/mask_heads/grid_head.py new file mode 100644 index 0000000..83058cb --- /dev/null +++ b/detection/mmdet/models/roi_heads/mask_heads/grid_head.py @@ -0,0 +1,359 @@ +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import ConvModule, kaiming_init, normal_init + +from mmdet.models.builder import HEADS, build_loss + + +@HEADS.register_module() +class GridHead(nn.Module): + + def __init__(self, + grid_points=9, + num_convs=8, + roi_feat_size=14, + in_channels=256, + conv_kernel_size=3, + point_feat_channels=64, + deconv_kernel_size=4, + class_agnostic=False, + loss_grid=dict( + type='CrossEntropyLoss', use_sigmoid=True, + loss_weight=15), + conv_cfg=None, + norm_cfg=dict(type='GN', num_groups=36)): + super(GridHead, self).__init__() + self.grid_points = grid_points + self.num_convs = num_convs + self.roi_feat_size = roi_feat_size + self.in_channels = in_channels + self.conv_kernel_size = conv_kernel_size + self.point_feat_channels = point_feat_channels + self.conv_out_channels = self.point_feat_channels * self.grid_points + self.class_agnostic = class_agnostic + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + if isinstance(norm_cfg, dict) and norm_cfg['type'] == 'GN': + assert self.conv_out_channels % norm_cfg['num_groups'] == 0 + + assert self.grid_points >= 4 + self.grid_size = int(np.sqrt(self.grid_points)) + if self.grid_size * self.grid_size != self.grid_points: + raise ValueError('grid_points must be a square number') + + # the predicted heatmap is half of whole_map_size + if not isinstance(self.roi_feat_size, int): + raise ValueError('Only square RoIs are supporeted in Grid R-CNN') + self.whole_map_size = self.roi_feat_size * 4 + + # compute point-wise sub-regions + self.sub_regions = self.calc_sub_regions() + + self.convs = [] + for i in range(self.num_convs): + in_channels = ( + self.in_channels if i == 0 else self.conv_out_channels) + stride = 2 if i == 0 else 1 + padding = (self.conv_kernel_size - 1) // 2 + self.convs.append( + ConvModule( + in_channels, + self.conv_out_channels, + self.conv_kernel_size, + stride=stride, + padding=padding, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + bias=True)) + self.convs = nn.Sequential(*self.convs) + + self.deconv1 = nn.ConvTranspose2d( + self.conv_out_channels, + self.conv_out_channels, + kernel_size=deconv_kernel_size, + stride=2, + padding=(deconv_kernel_size - 2) // 2, + groups=grid_points) + self.norm1 = nn.GroupNorm(grid_points, self.conv_out_channels) + self.deconv2 = nn.ConvTranspose2d( + self.conv_out_channels, + grid_points, + kernel_size=deconv_kernel_size, + stride=2, + padding=(deconv_kernel_size - 2) // 2, + groups=grid_points) + + # find the 4-neighbor of each grid point + self.neighbor_points = [] + grid_size = self.grid_size + for i in range(grid_size): # i-th column + for j in range(grid_size): # j-th row + neighbors = [] + if i > 0: # left: (i - 1, j) + neighbors.append((i - 1) * grid_size + j) + if j > 0: # up: (i, j - 1) + neighbors.append(i * grid_size + j - 1) + if j < grid_size - 1: # down: (i, j + 1) + neighbors.append(i * grid_size + j + 1) + if i < grid_size - 1: # right: (i + 1, j) + neighbors.append((i + 1) * grid_size + j) + self.neighbor_points.append(tuple(neighbors)) + # total edges in the grid + self.num_edges = sum([len(p) for p in self.neighbor_points]) + + self.forder_trans = nn.ModuleList() # first-order feature transition + self.sorder_trans = nn.ModuleList() # second-order feature transition + for neighbors in self.neighbor_points: + fo_trans = nn.ModuleList() + so_trans = nn.ModuleList() + for _ in range(len(neighbors)): + # each transition module consists of a 5x5 depth-wise conv and + # 1x1 conv. + fo_trans.append( + nn.Sequential( + nn.Conv2d( + self.point_feat_channels, + self.point_feat_channels, + 5, + stride=1, + padding=2, + groups=self.point_feat_channels), + nn.Conv2d(self.point_feat_channels, + self.point_feat_channels, 1))) + so_trans.append( + nn.Sequential( + nn.Conv2d( + self.point_feat_channels, + self.point_feat_channels, + 5, + 1, + 2, + groups=self.point_feat_channels), + nn.Conv2d(self.point_feat_channels, + self.point_feat_channels, 1))) + self.forder_trans.append(fo_trans) + self.sorder_trans.append(so_trans) + + self.loss_grid = build_loss(loss_grid) + + def init_weights(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d) or isinstance(m, nn.Linear): + # TODO: compare mode = "fan_in" or "fan_out" + kaiming_init(m) + for m in self.modules(): + if isinstance(m, nn.ConvTranspose2d): + normal_init(m, std=0.001) + nn.init.constant_(self.deconv2.bias, -np.log(0.99 / 0.01)) + + def forward(self, x): + assert x.shape[-1] == x.shape[-2] == self.roi_feat_size + # RoI feature transformation, downsample 2x + x = self.convs(x) + + c = self.point_feat_channels + # first-order fusion + x_fo = [None for _ in range(self.grid_points)] + for i, points in enumerate(self.neighbor_points): + x_fo[i] = x[:, i * c:(i + 1) * c] + for j, point_idx in enumerate(points): + x_fo[i] = x_fo[i] + self.forder_trans[i][j]( + x[:, point_idx * c:(point_idx + 1) * c]) + + # second-order fusion + x_so = [None for _ in range(self.grid_points)] + for i, points in enumerate(self.neighbor_points): + x_so[i] = x[:, i * c:(i + 1) * c] + for j, point_idx in enumerate(points): + x_so[i] = x_so[i] + self.sorder_trans[i][j](x_fo[point_idx]) + + # predicted heatmap with fused features + x2 = torch.cat(x_so, dim=1) + x2 = self.deconv1(x2) + x2 = F.relu(self.norm1(x2), inplace=True) + heatmap = self.deconv2(x2) + + # predicted heatmap with original features (applicable during training) + if self.training: + x1 = x + x1 = self.deconv1(x1) + x1 = F.relu(self.norm1(x1), inplace=True) + heatmap_unfused = self.deconv2(x1) + else: + heatmap_unfused = heatmap + + return dict(fused=heatmap, unfused=heatmap_unfused) + + def calc_sub_regions(self): + """Compute point specific representation regions. + + See Grid R-CNN Plus (https://arxiv.org/abs/1906.05688) for details. + """ + # to make it consistent with the original implementation, half_size + # is computed as 2 * quarter_size, which is smaller + half_size = self.whole_map_size // 4 * 2 + sub_regions = [] + for i in range(self.grid_points): + x_idx = i // self.grid_size + y_idx = i % self.grid_size + if x_idx == 0: + sub_x1 = 0 + elif x_idx == self.grid_size - 1: + sub_x1 = half_size + else: + ratio = x_idx / (self.grid_size - 1) - 0.25 + sub_x1 = max(int(ratio * self.whole_map_size), 0) + + if y_idx == 0: + sub_y1 = 0 + elif y_idx == self.grid_size - 1: + sub_y1 = half_size + else: + ratio = y_idx / (self.grid_size - 1) - 0.25 + sub_y1 = max(int(ratio * self.whole_map_size), 0) + sub_regions.append( + (sub_x1, sub_y1, sub_x1 + half_size, sub_y1 + half_size)) + return sub_regions + + def get_targets(self, sampling_results, rcnn_train_cfg): + # mix all samples (across images) together. + pos_bboxes = torch.cat([res.pos_bboxes for res in sampling_results], + dim=0).cpu() + pos_gt_bboxes = torch.cat( + [res.pos_gt_bboxes for res in sampling_results], dim=0).cpu() + assert pos_bboxes.shape == pos_gt_bboxes.shape + + # expand pos_bboxes to 2x of original size + x1 = pos_bboxes[:, 0] - (pos_bboxes[:, 2] - pos_bboxes[:, 0]) / 2 + y1 = pos_bboxes[:, 1] - (pos_bboxes[:, 3] - pos_bboxes[:, 1]) / 2 + x2 = pos_bboxes[:, 2] + (pos_bboxes[:, 2] - pos_bboxes[:, 0]) / 2 + y2 = pos_bboxes[:, 3] + (pos_bboxes[:, 3] - pos_bboxes[:, 1]) / 2 + pos_bboxes = torch.stack([x1, y1, x2, y2], dim=-1) + pos_bbox_ws = (pos_bboxes[:, 2] - pos_bboxes[:, 0]).unsqueeze(-1) + pos_bbox_hs = (pos_bboxes[:, 3] - pos_bboxes[:, 1]).unsqueeze(-1) + + num_rois = pos_bboxes.shape[0] + map_size = self.whole_map_size + # this is not the final target shape + targets = torch.zeros((num_rois, self.grid_points, map_size, map_size), + dtype=torch.float) + + # pre-compute interpolation factors for all grid points. + # the first item is the factor of x-dim, and the second is y-dim. + # for a 9-point grid, factors are like (1, 0), (0.5, 0.5), (0, 1) + factors = [] + for j in range(self.grid_points): + x_idx = j // self.grid_size + y_idx = j % self.grid_size + factors.append((1 - x_idx / (self.grid_size - 1), + 1 - y_idx / (self.grid_size - 1))) + + radius = rcnn_train_cfg.pos_radius + radius2 = radius**2 + for i in range(num_rois): + # ignore small bboxes + if (pos_bbox_ws[i] <= self.grid_size + or pos_bbox_hs[i] <= self.grid_size): + continue + # for each grid point, mark a small circle as positive + for j in range(self.grid_points): + factor_x, factor_y = factors[j] + gridpoint_x = factor_x * pos_gt_bboxes[i, 0] + ( + 1 - factor_x) * pos_gt_bboxes[i, 2] + gridpoint_y = factor_y * pos_gt_bboxes[i, 1] + ( + 1 - factor_y) * pos_gt_bboxes[i, 3] + + cx = int((gridpoint_x - pos_bboxes[i, 0]) / pos_bbox_ws[i] * + map_size) + cy = int((gridpoint_y - pos_bboxes[i, 1]) / pos_bbox_hs[i] * + map_size) + + for x in range(cx - radius, cx + radius + 1): + for y in range(cy - radius, cy + radius + 1): + if x >= 0 and x < map_size and y >= 0 and y < map_size: + if (x - cx)**2 + (y - cy)**2 <= radius2: + targets[i, j, y, x] = 1 + # reduce the target heatmap size by a half + # proposed in Grid R-CNN Plus (https://arxiv.org/abs/1906.05688). + sub_targets = [] + for i in range(self.grid_points): + sub_x1, sub_y1, sub_x2, sub_y2 = self.sub_regions[i] + sub_targets.append(targets[:, [i], sub_y1:sub_y2, sub_x1:sub_x2]) + sub_targets = torch.cat(sub_targets, dim=1) + sub_targets = sub_targets.to(sampling_results[0].pos_bboxes.device) + return sub_targets + + def loss(self, grid_pred, grid_targets): + loss_fused = self.loss_grid(grid_pred['fused'], grid_targets) + loss_unfused = self.loss_grid(grid_pred['unfused'], grid_targets) + loss_grid = loss_fused + loss_unfused + return dict(loss_grid=loss_grid) + + def get_bboxes(self, det_bboxes, grid_pred, img_metas): + # TODO: refactoring + assert det_bboxes.shape[0] == grid_pred.shape[0] + det_bboxes = det_bboxes.cpu() + cls_scores = det_bboxes[:, [4]] + det_bboxes = det_bboxes[:, :4] + grid_pred = grid_pred.sigmoid().cpu() + + R, c, h, w = grid_pred.shape + half_size = self.whole_map_size // 4 * 2 + assert h == w == half_size + assert c == self.grid_points + + # find the point with max scores in the half-sized heatmap + grid_pred = grid_pred.view(R * c, h * w) + pred_scores, pred_position = grid_pred.max(dim=1) + xs = pred_position % w + ys = pred_position // w + + # get the position in the whole heatmap instead of half-sized heatmap + for i in range(self.grid_points): + xs[i::self.grid_points] += self.sub_regions[i][0] + ys[i::self.grid_points] += self.sub_regions[i][1] + + # reshape to (num_rois, grid_points) + pred_scores, xs, ys = tuple( + map(lambda x: x.view(R, c), [pred_scores, xs, ys])) + + # get expanded pos_bboxes + widths = (det_bboxes[:, 2] - det_bboxes[:, 0]).unsqueeze(-1) + heights = (det_bboxes[:, 3] - det_bboxes[:, 1]).unsqueeze(-1) + x1 = (det_bboxes[:, 0, None] - widths / 2) + y1 = (det_bboxes[:, 1, None] - heights / 2) + # map the grid point to the absolute coordinates + abs_xs = (xs.float() + 0.5) / w * widths + x1 + abs_ys = (ys.float() + 0.5) / h * heights + y1 + + # get the grid points indices that fall on the bbox boundaries + x1_inds = [i for i in range(self.grid_size)] + y1_inds = [i * self.grid_size for i in range(self.grid_size)] + x2_inds = [ + self.grid_points - self.grid_size + i + for i in range(self.grid_size) + ] + y2_inds = [(i + 1) * self.grid_size - 1 for i in range(self.grid_size)] + + # voting of all grid points on some boundary + bboxes_x1 = (abs_xs[:, x1_inds] * pred_scores[:, x1_inds]).sum( + dim=1, keepdim=True) / ( + pred_scores[:, x1_inds].sum(dim=1, keepdim=True)) + bboxes_y1 = (abs_ys[:, y1_inds] * pred_scores[:, y1_inds]).sum( + dim=1, keepdim=True) / ( + pred_scores[:, y1_inds].sum(dim=1, keepdim=True)) + bboxes_x2 = (abs_xs[:, x2_inds] * pred_scores[:, x2_inds]).sum( + dim=1, keepdim=True) / ( + pred_scores[:, x2_inds].sum(dim=1, keepdim=True)) + bboxes_y2 = (abs_ys[:, y2_inds] * pred_scores[:, y2_inds]).sum( + dim=1, keepdim=True) / ( + pred_scores[:, y2_inds].sum(dim=1, keepdim=True)) + + bbox_res = torch.cat( + [bboxes_x1, bboxes_y1, bboxes_x2, bboxes_y2, cls_scores], dim=1) + bbox_res[:, [0, 2]].clamp_(min=0, max=img_metas[0]['img_shape'][1]) + bbox_res[:, [1, 3]].clamp_(min=0, max=img_metas[0]['img_shape'][0]) + + return bbox_res diff --git a/detection/mmdet/models/roi_heads/mask_heads/htc_mask_head.py b/detection/mmdet/models/roi_heads/mask_heads/htc_mask_head.py new file mode 100644 index 0000000..330b778 --- /dev/null +++ b/detection/mmdet/models/roi_heads/mask_heads/htc_mask_head.py @@ -0,0 +1,43 @@ +from mmcv.cnn import ConvModule + +from mmdet.models.builder import HEADS +from .fcn_mask_head import FCNMaskHead + + +@HEADS.register_module() +class HTCMaskHead(FCNMaskHead): + + def __init__(self, with_conv_res=True, *args, **kwargs): + super(HTCMaskHead, self).__init__(*args, **kwargs) + self.with_conv_res = with_conv_res + if self.with_conv_res: + self.conv_res = ConvModule( + self.conv_out_channels, + self.conv_out_channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg) + + def init_weights(self): + super(HTCMaskHead, self).init_weights() + if self.with_conv_res: + self.conv_res.init_weights() + + def forward(self, x, res_feat=None, return_logits=True, return_feat=True): + if res_feat is not None: + assert self.with_conv_res + res_feat = self.conv_res(res_feat) + x = x + res_feat + for conv in self.convs: + x = conv(x) + res_feat = x + outs = [] + if return_logits: + x = self.upsample(x) + if self.upsample_method == 'deconv': + x = self.relu(x) + mask_pred = self.conv_logits(x) + outs.append(mask_pred) + if return_feat: + outs.append(res_feat) + return outs if len(outs) > 1 else outs[0] diff --git a/detection/mmdet/models/roi_heads/mask_heads/mask_point_head.py b/detection/mmdet/models/roi_heads/mask_heads/mask_point_head.py new file mode 100644 index 0000000..fb92903 --- /dev/null +++ b/detection/mmdet/models/roi_heads/mask_heads/mask_point_head.py @@ -0,0 +1,300 @@ +# Modified from https://github.com/facebookresearch/detectron2/tree/master/projects/PointRend/point_head/point_head.py # noqa + +import torch +import torch.nn as nn +from mmcv.cnn import ConvModule, normal_init +from mmcv.ops import point_sample, rel_roi_point_to_rel_img_point + +from mmdet.models.builder import HEADS, build_loss + + +@HEADS.register_module() +class MaskPointHead(nn.Module): + """A mask point head use in PointRend. + + ``MaskPointHead`` use shared multi-layer perceptron (equivalent to + nn.Conv1d) to predict the logit of input points. The fine-grained feature + and coarse feature will be concatenate together for predication. + + Args: + num_fcs (int): Number of fc layers in the head. Default: 3. + in_channels (int): Number of input channels. Default: 256. + fc_channels (int): Number of fc channels. Default: 256. + num_classes (int): Number of classes for logits. Default: 80. + class_agnostic (bool): Whether use class agnostic classification. + If so, the output channels of logits will be 1. Default: False. + coarse_pred_each_layer (bool): Whether concatenate coarse feature with + the output of each fc layer. Default: True. + conv_cfg (dict | None): Dictionary to construct and config conv layer. + Default: dict(type='Conv1d')) + norm_cfg (dict | None): Dictionary to construct and config norm layer. + Default: None. + loss_point (dict): Dictionary to construct and config loss layer of + point head. Default: dict(type='CrossEntropyLoss', use_mask=True, + loss_weight=1.0). + """ + + def __init__(self, + num_classes, + num_fcs=3, + in_channels=256, + fc_channels=256, + class_agnostic=False, + coarse_pred_each_layer=True, + conv_cfg=dict(type='Conv1d'), + norm_cfg=None, + act_cfg=dict(type='ReLU'), + loss_point=dict( + type='CrossEntropyLoss', use_mask=True, loss_weight=1.0)): + super().__init__() + self.num_fcs = num_fcs + self.in_channels = in_channels + self.fc_channels = fc_channels + self.num_classes = num_classes + self.class_agnostic = class_agnostic + self.coarse_pred_each_layer = coarse_pred_each_layer + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.loss_point = build_loss(loss_point) + + fc_in_channels = in_channels + num_classes + self.fcs = nn.ModuleList() + for _ in range(num_fcs): + fc = ConvModule( + fc_in_channels, + fc_channels, + kernel_size=1, + stride=1, + padding=0, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + self.fcs.append(fc) + fc_in_channels = fc_channels + fc_in_channels += num_classes if self.coarse_pred_each_layer else 0 + + out_channels = 1 if self.class_agnostic else self.num_classes + self.fc_logits = nn.Conv1d( + fc_in_channels, out_channels, kernel_size=1, stride=1, padding=0) + + def init_weights(self): + """Initialize last classification layer of MaskPointHead, conv layers + are already initialized by ConvModule.""" + normal_init(self.fc_logits, std=0.001) + + def forward(self, fine_grained_feats, coarse_feats): + """Classify each point base on fine grained and coarse feats. + + Args: + fine_grained_feats (Tensor): Fine grained feature sampled from FPN, + shape (num_rois, in_channels, num_points). + coarse_feats (Tensor): Coarse feature sampled from CoarseMaskHead, + shape (num_rois, num_classes, num_points). + + Returns: + Tensor: Point classification results, + shape (num_rois, num_class, num_points). + """ + + x = torch.cat([fine_grained_feats, coarse_feats], dim=1) + for fc in self.fcs: + x = fc(x) + if self.coarse_pred_each_layer: + x = torch.cat((x, coarse_feats), dim=1) + return self.fc_logits(x) + + def get_targets(self, rois, rel_roi_points, sampling_results, gt_masks, + cfg): + """Get training targets of MaskPointHead for all images. + + Args: + rois (Tensor): Region of Interest, shape (num_rois, 5). + rel_roi_points: Points coordinates relative to RoI, shape + (num_rois, num_points, 2). + sampling_results (:obj:`SamplingResult`): Sampling result after + sampling and assignment. + gt_masks (Tensor) : Ground truth segmentation masks of + corresponding boxes, shape (num_rois, height, width). + cfg (dict): Training cfg. + + Returns: + Tensor: Point target, shape (num_rois, num_points). + """ + + num_imgs = len(sampling_results) + rois_list = [] + rel_roi_points_list = [] + for batch_ind in range(num_imgs): + inds = (rois[:, 0] == batch_ind) + rois_list.append(rois[inds]) + rel_roi_points_list.append(rel_roi_points[inds]) + pos_assigned_gt_inds_list = [ + res.pos_assigned_gt_inds for res in sampling_results + ] + cfg_list = [cfg for _ in range(num_imgs)] + + point_targets = map(self._get_target_single, rois_list, + rel_roi_points_list, pos_assigned_gt_inds_list, + gt_masks, cfg_list) + point_targets = list(point_targets) + + if len(point_targets) > 0: + point_targets = torch.cat(point_targets) + + return point_targets + + def _get_target_single(self, rois, rel_roi_points, pos_assigned_gt_inds, + gt_masks, cfg): + """Get training target of MaskPointHead for each image.""" + num_pos = rois.size(0) + num_points = cfg.num_points + if num_pos > 0: + gt_masks_th = ( + gt_masks.to_tensor(rois.dtype, rois.device).index_select( + 0, pos_assigned_gt_inds)) + gt_masks_th = gt_masks_th.unsqueeze(1) + rel_img_points = rel_roi_point_to_rel_img_point( + rois, rel_roi_points, gt_masks_th.shape[2:]) + point_targets = point_sample(gt_masks_th, + rel_img_points).squeeze(1) + else: + point_targets = rois.new_zeros((0, num_points)) + return point_targets + + def loss(self, point_pred, point_targets, labels): + """Calculate loss for MaskPointHead. + + Args: + point_pred (Tensor): Point predication result, shape + (num_rois, num_classes, num_points). + point_targets (Tensor): Point targets, shape (num_roi, num_points). + labels (Tensor): Class label of corresponding boxes, + shape (num_rois, ) + + Returns: + dict[str, Tensor]: a dictionary of point loss components + """ + + loss = dict() + if self.class_agnostic: + loss_point = self.loss_point(point_pred, point_targets, + torch.zeros_like(labels)) + else: + loss_point = self.loss_point(point_pred, point_targets, labels) + loss['loss_point'] = loss_point + return loss + + def _get_uncertainty(self, mask_pred, labels): + """Estimate uncertainty based on pred logits. + + We estimate uncertainty as L1 distance between 0.0 and the logits + prediction in 'mask_pred' for the foreground class in `classes`. + + Args: + mask_pred (Tensor): mask predication logits, shape (num_rois, + num_classes, mask_height, mask_width). + + labels (list[Tensor]): Either predicted or ground truth label for + each predicted mask, of length num_rois. + + Returns: + scores (Tensor): Uncertainty scores with the most uncertain + locations having the highest uncertainty score, + shape (num_rois, 1, mask_height, mask_width) + """ + if mask_pred.shape[1] == 1: + gt_class_logits = mask_pred.clone() + else: + inds = torch.arange(mask_pred.shape[0], device=mask_pred.device) + gt_class_logits = mask_pred[inds, labels].unsqueeze(1) + return -torch.abs(gt_class_logits) + + def get_roi_rel_points_train(self, mask_pred, labels, cfg): + """Get ``num_points`` most uncertain points with random points during + train. + + Sample points in [0, 1] x [0, 1] coordinate space based on their + uncertainty. The uncertainties are calculated for each point using + '_get_uncertainty()' function that takes point's logit prediction as + input. + + Args: + mask_pred (Tensor): A tensor of shape (num_rois, num_classes, + mask_height, mask_width) for class-specific or class-agnostic + prediction. + labels (list): The ground truth class for each instance. + cfg (dict): Training config of point head. + + Returns: + point_coords (Tensor): A tensor of shape (num_rois, num_points, 2) + that contains the coordinates sampled points. + """ + num_points = cfg.num_points + oversample_ratio = cfg.oversample_ratio + importance_sample_ratio = cfg.importance_sample_ratio + assert oversample_ratio >= 1 + assert 0 <= importance_sample_ratio <= 1 + batch_size = mask_pred.shape[0] + num_sampled = int(num_points * oversample_ratio) + point_coords = torch.rand( + batch_size, num_sampled, 2, device=mask_pred.device) + point_logits = point_sample(mask_pred, point_coords) + # It is crucial to calculate uncertainty based on the sampled + # prediction value for the points. Calculating uncertainties of the + # coarse predictions first and sampling them for points leads to + # incorrect results. To illustrate this: assume uncertainty func( + # logits)=-abs(logits), a sampled point between two coarse + # predictions with -1 and 1 logits has 0 logits, and therefore 0 + # uncertainty value. However, if we calculate uncertainties for the + # coarse predictions first, both will have -1 uncertainty, + # and sampled point will get -1 uncertainty. + point_uncertainties = self._get_uncertainty(point_logits, labels) + num_uncertain_points = int(importance_sample_ratio * num_points) + num_random_points = num_points - num_uncertain_points + idx = torch.topk( + point_uncertainties[:, 0, :], k=num_uncertain_points, dim=1)[1] + shift = num_sampled * torch.arange( + batch_size, dtype=torch.long, device=mask_pred.device) + idx += shift[:, None] + point_coords = point_coords.view(-1, 2)[idx.view(-1), :].view( + batch_size, num_uncertain_points, 2) + if num_random_points > 0: + rand_roi_coords = torch.rand( + batch_size, num_random_points, 2, device=mask_pred.device) + point_coords = torch.cat((point_coords, rand_roi_coords), dim=1) + return point_coords + + def get_roi_rel_points_test(self, mask_pred, pred_label, cfg): + """Get ``num_points`` most uncertain points during test. + + Args: + mask_pred (Tensor): A tensor of shape (num_rois, num_classes, + mask_height, mask_width) for class-specific or class-agnostic + prediction. + pred_label (list): The predication class for each instance. + cfg (dict): Testing config of point head. + + Returns: + point_indices (Tensor): A tensor of shape (num_rois, num_points) + that contains indices from [0, mask_height x mask_width) of the + most uncertain points. + point_coords (Tensor): A tensor of shape (num_rois, num_points, 2) + that contains [0, 1] x [0, 1] normalized coordinates of the + most uncertain points from the [mask_height, mask_width] grid . + """ + num_points = cfg.subdivision_num_points + uncertainty_map = self._get_uncertainty(mask_pred, pred_label) + num_rois, _, mask_height, mask_width = uncertainty_map.shape + h_step = 1.0 / mask_height + w_step = 1.0 / mask_width + + uncertainty_map = uncertainty_map.view(num_rois, + mask_height * mask_width) + num_points = min(mask_height * mask_width, num_points) + point_indices = uncertainty_map.topk(num_points, dim=1)[1] + point_coords = uncertainty_map.new_zeros(num_rois, num_points, 2) + point_coords[:, :, 0] = w_step / 2.0 + (point_indices % + mask_width).float() * w_step + point_coords[:, :, 1] = h_step / 2.0 + (point_indices // + mask_width).float() * h_step + return point_indices, point_coords diff --git a/detection/mmdet/models/roi_heads/mask_heads/maskiou_head.py b/detection/mmdet/models/roi_heads/mask_heads/maskiou_head.py new file mode 100644 index 0000000..39bcd6a --- /dev/null +++ b/detection/mmdet/models/roi_heads/mask_heads/maskiou_head.py @@ -0,0 +1,186 @@ +import numpy as np +import torch +import torch.nn as nn +from mmcv.cnn import Conv2d, Linear, MaxPool2d, kaiming_init, normal_init +from mmcv.runner import force_fp32 +from torch.nn.modules.utils import _pair + +from mmdet.models.builder import HEADS, build_loss + + +@HEADS.register_module() +class MaskIoUHead(nn.Module): + """Mask IoU Head. + + This head predicts the IoU of predicted masks and corresponding gt masks. + """ + + def __init__(self, + num_convs=4, + num_fcs=2, + roi_feat_size=14, + in_channels=256, + conv_out_channels=256, + fc_out_channels=1024, + num_classes=80, + loss_iou=dict(type='MSELoss', loss_weight=0.5)): + super(MaskIoUHead, self).__init__() + self.in_channels = in_channels + self.conv_out_channels = conv_out_channels + self.fc_out_channels = fc_out_channels + self.num_classes = num_classes + self.fp16_enabled = False + + self.convs = nn.ModuleList() + for i in range(num_convs): + if i == 0: + # concatenation of mask feature and mask prediction + in_channels = self.in_channels + 1 + else: + in_channels = self.conv_out_channels + stride = 2 if i == num_convs - 1 else 1 + self.convs.append( + Conv2d( + in_channels, + self.conv_out_channels, + 3, + stride=stride, + padding=1)) + + roi_feat_size = _pair(roi_feat_size) + pooled_area = (roi_feat_size[0] // 2) * (roi_feat_size[1] // 2) + self.fcs = nn.ModuleList() + for i in range(num_fcs): + in_channels = ( + self.conv_out_channels * + pooled_area if i == 0 else self.fc_out_channels) + self.fcs.append(Linear(in_channels, self.fc_out_channels)) + + self.fc_mask_iou = Linear(self.fc_out_channels, self.num_classes) + self.relu = nn.ReLU() + self.max_pool = MaxPool2d(2, 2) + self.loss_iou = build_loss(loss_iou) + + def init_weights(self): + for conv in self.convs: + kaiming_init(conv) + for fc in self.fcs: + kaiming_init( + fc, + a=1, + mode='fan_in', + nonlinearity='leaky_relu', + distribution='uniform') + normal_init(self.fc_mask_iou, std=0.01) + + def forward(self, mask_feat, mask_pred): + mask_pred = mask_pred.sigmoid() + mask_pred_pooled = self.max_pool(mask_pred.unsqueeze(1)) + + x = torch.cat((mask_feat, mask_pred_pooled), 1) + + for conv in self.convs: + x = self.relu(conv(x)) + x = x.flatten(1) + for fc in self.fcs: + x = self.relu(fc(x)) + mask_iou = self.fc_mask_iou(x) + return mask_iou + + @force_fp32(apply_to=('mask_iou_pred', )) + def loss(self, mask_iou_pred, mask_iou_targets): + pos_inds = mask_iou_targets > 0 + if pos_inds.sum() > 0: + loss_mask_iou = self.loss_iou(mask_iou_pred[pos_inds], + mask_iou_targets[pos_inds]) + else: + loss_mask_iou = mask_iou_pred.sum() * 0 + return dict(loss_mask_iou=loss_mask_iou) + + @force_fp32(apply_to=('mask_pred', )) + def get_targets(self, sampling_results, gt_masks, mask_pred, mask_targets, + rcnn_train_cfg): + """Compute target of mask IoU. + + Mask IoU target is the IoU of the predicted mask (inside a bbox) and + the gt mask of corresponding gt mask (the whole instance). + The intersection area is computed inside the bbox, and the gt mask area + is computed with two steps, firstly we compute the gt area inside the + bbox, then divide it by the area ratio of gt area inside the bbox and + the gt area of the whole instance. + + Args: + sampling_results (list[:obj:`SamplingResult`]): sampling results. + gt_masks (BitmapMask | PolygonMask): Gt masks (the whole instance) + of each image, with the same shape of the input image. + mask_pred (Tensor): Predicted masks of each positive proposal, + shape (num_pos, h, w). + mask_targets (Tensor): Gt mask of each positive proposal, + binary map of the shape (num_pos, h, w). + rcnn_train_cfg (dict): Training config for R-CNN part. + + Returns: + Tensor: mask iou target (length == num positive). + """ + pos_proposals = [res.pos_bboxes for res in sampling_results] + pos_assigned_gt_inds = [ + res.pos_assigned_gt_inds for res in sampling_results + ] + + # compute the area ratio of gt areas inside the proposals and + # the whole instance + area_ratios = map(self._get_area_ratio, pos_proposals, + pos_assigned_gt_inds, gt_masks) + area_ratios = torch.cat(list(area_ratios)) + assert mask_targets.size(0) == area_ratios.size(0) + + mask_pred = (mask_pred > rcnn_train_cfg.mask_thr_binary).float() + mask_pred_areas = mask_pred.sum((-1, -2)) + + # mask_pred and mask_targets are binary maps + overlap_areas = (mask_pred * mask_targets).sum((-1, -2)) + + # compute the mask area of the whole instance + gt_full_areas = mask_targets.sum((-1, -2)) / (area_ratios + 1e-7) + + mask_iou_targets = overlap_areas / ( + mask_pred_areas + gt_full_areas - overlap_areas) + return mask_iou_targets + + def _get_area_ratio(self, pos_proposals, pos_assigned_gt_inds, gt_masks): + """Compute area ratio of the gt mask inside the proposal and the gt + mask of the corresponding instance.""" + num_pos = pos_proposals.size(0) + if num_pos > 0: + area_ratios = [] + proposals_np = pos_proposals.cpu().numpy() + pos_assigned_gt_inds = pos_assigned_gt_inds.cpu().numpy() + # compute mask areas of gt instances (batch processing for speedup) + gt_instance_mask_area = gt_masks.areas + for i in range(num_pos): + gt_mask = gt_masks[pos_assigned_gt_inds[i]] + + # crop the gt mask inside the proposal + bbox = proposals_np[i, :].astype(np.int32) + gt_mask_in_proposal = gt_mask.crop(bbox) + + ratio = gt_mask_in_proposal.areas[0] / ( + gt_instance_mask_area[pos_assigned_gt_inds[i]] + 1e-7) + area_ratios.append(ratio) + area_ratios = torch.from_numpy(np.stack(area_ratios)).float().to( + pos_proposals.device) + else: + area_ratios = pos_proposals.new_zeros((0, )) + return area_ratios + + @force_fp32(apply_to=('mask_iou_pred', )) + def get_mask_scores(self, mask_iou_pred, det_bboxes, det_labels): + """Get the mask scores. + + mask_score = bbox_score * mask_iou + """ + inds = range(det_labels.size(0)) + mask_scores = mask_iou_pred[inds, det_labels] * det_bboxes[inds, -1] + mask_scores = mask_scores.cpu().numpy() + det_labels = det_labels.cpu().numpy() + return [mask_scores[det_labels == i] for i in range(self.num_classes)] diff --git a/detection/mmdet/models/roi_heads/mask_heads/scnet_mask_head.py b/detection/mmdet/models/roi_heads/mask_heads/scnet_mask_head.py new file mode 100644 index 0000000..983a2d9 --- /dev/null +++ b/detection/mmdet/models/roi_heads/mask_heads/scnet_mask_head.py @@ -0,0 +1,27 @@ +from mmdet.models.builder import HEADS +from mmdet.models.utils import ResLayer, SimplifiedBasicBlock +from .fcn_mask_head import FCNMaskHead + + +@HEADS.register_module() +class SCNetMaskHead(FCNMaskHead): + """Mask head for `SCNet `_. + + Args: + conv_to_res (bool, optional): if True, change the conv layers to + ``SimplifiedBasicBlock``. + """ + + def __init__(self, conv_to_res=True, **kwargs): + super(SCNetMaskHead, self).__init__(**kwargs) + self.conv_to_res = conv_to_res + if conv_to_res: + assert self.conv_kernel_size == 3 + self.num_res_blocks = self.num_convs // 2 + self.convs = ResLayer( + SimplifiedBasicBlock, + self.in_channels, + self.conv_out_channels, + self.num_res_blocks, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg) diff --git a/detection/mmdet/models/roi_heads/mask_heads/scnet_semantic_head.py b/detection/mmdet/models/roi_heads/mask_heads/scnet_semantic_head.py new file mode 100644 index 0000000..df85a01 --- /dev/null +++ b/detection/mmdet/models/roi_heads/mask_heads/scnet_semantic_head.py @@ -0,0 +1,27 @@ +from mmdet.models.builder import HEADS +from mmdet.models.utils import ResLayer, SimplifiedBasicBlock +from .fused_semantic_head import FusedSemanticHead + + +@HEADS.register_module() +class SCNetSemanticHead(FusedSemanticHead): + """Mask head for `SCNet `_. + + Args: + conv_to_res (bool, optional): if True, change the conv layers to + ``SimplifiedBasicBlock``. + """ + + def __init__(self, conv_to_res=True, **kwargs): + super(SCNetSemanticHead, self).__init__(**kwargs) + self.conv_to_res = conv_to_res + if self.conv_to_res: + num_res_blocks = self.num_convs // 2 + self.convs = ResLayer( + SimplifiedBasicBlock, + self.in_channels, + self.conv_out_channels, + num_res_blocks, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg) + self.num_convs = num_res_blocks diff --git a/detection/mmdet/models/roi_heads/mask_scoring_roi_head.py b/detection/mmdet/models/roi_heads/mask_scoring_roi_head.py new file mode 100644 index 0000000..c6e55c7 --- /dev/null +++ b/detection/mmdet/models/roi_heads/mask_scoring_roi_head.py @@ -0,0 +1,122 @@ +import torch + +from mmdet.core import bbox2roi +from ..builder import HEADS, build_head +from .standard_roi_head import StandardRoIHead + + +@HEADS.register_module() +class MaskScoringRoIHead(StandardRoIHead): + """Mask Scoring RoIHead for Mask Scoring RCNN. + + https://arxiv.org/abs/1903.00241 + """ + + def __init__(self, mask_iou_head, **kwargs): + assert mask_iou_head is not None + super(MaskScoringRoIHead, self).__init__(**kwargs) + self.mask_iou_head = build_head(mask_iou_head) + + def init_weights(self, pretrained): + """Initialize the weights in head. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + super(MaskScoringRoIHead, self).init_weights(pretrained) + self.mask_iou_head.init_weights() + + def _mask_forward_train(self, x, sampling_results, bbox_feats, gt_masks, + img_metas): + """Run forward function and calculate loss for Mask head in + training.""" + pos_labels = torch.cat([res.pos_gt_labels for res in sampling_results]) + mask_results = super(MaskScoringRoIHead, + self)._mask_forward_train(x, sampling_results, + bbox_feats, gt_masks, + img_metas) + if mask_results['loss_mask'] is None: + return mask_results + + # mask iou head forward and loss + pos_mask_pred = mask_results['mask_pred'][ + range(mask_results['mask_pred'].size(0)), pos_labels] + mask_iou_pred = self.mask_iou_head(mask_results['mask_feats'], + pos_mask_pred) + pos_mask_iou_pred = mask_iou_pred[range(mask_iou_pred.size(0)), + pos_labels] + + mask_iou_targets = self.mask_iou_head.get_targets( + sampling_results, gt_masks, pos_mask_pred, + mask_results['mask_targets'], self.train_cfg) + loss_mask_iou = self.mask_iou_head.loss(pos_mask_iou_pred, + mask_iou_targets) + mask_results['loss_mask'].update(loss_mask_iou) + return mask_results + + def simple_test_mask(self, + x, + img_metas, + det_bboxes, + det_labels, + rescale=False): + """Obtain mask prediction without augmentation.""" + # image shapes of images in the batch + ori_shapes = tuple(meta['ori_shape'] for meta in img_metas) + scale_factors = tuple(meta['scale_factor'] for meta in img_metas) + + num_imgs = len(det_bboxes) + if all(det_bbox.shape[0] == 0 for det_bbox in det_bboxes): + num_classes = self.mask_head.num_classes + segm_results = [[[] for _ in range(num_classes)] + for _ in range(num_imgs)] + mask_scores = [[[] for _ in range(num_classes)] + for _ in range(num_imgs)] + else: + # if det_bboxes is rescaled to the original image size, we need to + # rescale it back to the testing scale to obtain RoIs. + if rescale and not isinstance(scale_factors[0], float): + scale_factors = [ + torch.from_numpy(scale_factor).to(det_bboxes[0].device) + for scale_factor in scale_factors + ] + _bboxes = [ + det_bboxes[i][:, :4] * + scale_factors[i] if rescale else det_bboxes[i] + for i in range(num_imgs) + ] + mask_rois = bbox2roi(_bboxes) + mask_results = self._mask_forward(x, mask_rois) + concat_det_labels = torch.cat(det_labels) + # get mask scores with mask iou head + mask_feats = mask_results['mask_feats'] + mask_pred = mask_results['mask_pred'] + mask_iou_pred = self.mask_iou_head( + mask_feats, mask_pred[range(concat_det_labels.size(0)), + concat_det_labels]) + # split batch mask prediction back to each image + num_bboxes_per_img = tuple(len(_bbox) for _bbox in _bboxes) + mask_preds = mask_pred.split(num_bboxes_per_img, 0) + mask_iou_preds = mask_iou_pred.split(num_bboxes_per_img, 0) + + # apply mask post-processing to each image individually + segm_results = [] + mask_scores = [] + for i in range(num_imgs): + if det_bboxes[i].shape[0] == 0: + segm_results.append( + [[] for _ in range(self.mask_head.num_classes)]) + mask_scores.append( + [[] for _ in range(self.mask_head.num_classes)]) + else: + segm_result = self.mask_head.get_seg_masks( + mask_preds[i], _bboxes[i], det_labels[i], + self.test_cfg, ori_shapes[i], scale_factors[i], + rescale) + # get mask scores with mask iou head + mask_score = self.mask_iou_head.get_mask_scores( + mask_iou_preds[i], det_bboxes[i], det_labels[i]) + segm_results.append(segm_result) + mask_scores.append(mask_score) + return list(zip(segm_results, mask_scores)) diff --git a/detection/mmdet/models/roi_heads/pisa_roi_head.py b/detection/mmdet/models/roi_heads/pisa_roi_head.py new file mode 100644 index 0000000..e011136 --- /dev/null +++ b/detection/mmdet/models/roi_heads/pisa_roi_head.py @@ -0,0 +1,159 @@ +from mmdet.core import bbox2roi +from ..builder import HEADS +from ..losses.pisa_loss import carl_loss, isr_p +from .standard_roi_head import StandardRoIHead + + +@HEADS.register_module() +class PISARoIHead(StandardRoIHead): + r"""The RoI head for `Prime Sample Attention in Object Detection + `_.""" + + def forward_train(self, + x, + img_metas, + proposal_list, + gt_bboxes, + gt_labels, + gt_bboxes_ignore=None, + gt_masks=None): + """Forward function for training. + + Args: + x (list[Tensor]): List of multi-level img features. + img_metas (list[dict]): List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmdet/datasets/pipelines/formatting.py:Collect`. + proposals (list[Tensors]): List of region proposals. + gt_bboxes (list[Tensor]): Each item are the truth boxes for each + image in [tl_x, tl_y, br_x, br_y] format. + gt_labels (list[Tensor]): Class indices corresponding to each box + gt_bboxes_ignore (list[Tensor], optional): Specify which bounding + boxes can be ignored when computing the loss. + gt_masks (None | Tensor) : True segmentation masks for each box + used if the architecture supports a segmentation task. + + Returns: + dict[str, Tensor]: a dictionary of loss components + """ + # assign gts and sample proposals + if self.with_bbox or self.with_mask: + num_imgs = len(img_metas) + if gt_bboxes_ignore is None: + gt_bboxes_ignore = [None for _ in range(num_imgs)] + sampling_results = [] + neg_label_weights = [] + for i in range(num_imgs): + assign_result = self.bbox_assigner.assign( + proposal_list[i], gt_bboxes[i], gt_bboxes_ignore[i], + gt_labels[i]) + sampling_result = self.bbox_sampler.sample( + assign_result, + proposal_list[i], + gt_bboxes[i], + gt_labels[i], + feats=[lvl_feat[i][None] for lvl_feat in x]) + # neg label weight is obtained by sampling when using ISR-N + neg_label_weight = None + if isinstance(sampling_result, tuple): + sampling_result, neg_label_weight = sampling_result + sampling_results.append(sampling_result) + neg_label_weights.append(neg_label_weight) + + losses = dict() + # bbox head forward and loss + if self.with_bbox: + bbox_results = self._bbox_forward_train( + x, + sampling_results, + gt_bboxes, + gt_labels, + img_metas, + neg_label_weights=neg_label_weights) + losses.update(bbox_results['loss_bbox']) + + # mask head forward and loss + if self.with_mask: + mask_results = self._mask_forward_train(x, sampling_results, + bbox_results['bbox_feats'], + gt_masks, img_metas) + losses.update(mask_results['loss_mask']) + + return losses + + def _bbox_forward(self, x, rois): + """Box forward function used in both training and testing.""" + # TODO: a more flexible way to decide which feature maps to use + bbox_feats = self.bbox_roi_extractor( + x[:self.bbox_roi_extractor.num_inputs], rois) + if self.with_shared_head: + bbox_feats = self.shared_head(bbox_feats) + cls_score, bbox_pred = self.bbox_head(bbox_feats) + + bbox_results = dict( + cls_score=cls_score, bbox_pred=bbox_pred, bbox_feats=bbox_feats) + return bbox_results + + def _bbox_forward_train(self, + x, + sampling_results, + gt_bboxes, + gt_labels, + img_metas, + neg_label_weights=None): + """Run forward function and calculate loss for box head in training.""" + rois = bbox2roi([res.bboxes for res in sampling_results]) + + bbox_results = self._bbox_forward(x, rois) + + bbox_targets = self.bbox_head.get_targets(sampling_results, gt_bboxes, + gt_labels, self.train_cfg) + + # neg_label_weights obtained by sampler is image-wise, mapping back to + # the corresponding location in label weights + if neg_label_weights[0] is not None: + label_weights = bbox_targets[1] + cur_num_rois = 0 + for i in range(len(sampling_results)): + num_pos = sampling_results[i].pos_inds.size(0) + num_neg = sampling_results[i].neg_inds.size(0) + label_weights[cur_num_rois + num_pos:cur_num_rois + num_pos + + num_neg] = neg_label_weights[i] + cur_num_rois += num_pos + num_neg + + cls_score = bbox_results['cls_score'] + bbox_pred = bbox_results['bbox_pred'] + + # Apply ISR-P + isr_cfg = self.train_cfg.get('isr', None) + if isr_cfg is not None: + bbox_targets = isr_p( + cls_score, + bbox_pred, + bbox_targets, + rois, + sampling_results, + self.bbox_head.loss_cls, + self.bbox_head.bbox_coder, + **isr_cfg, + num_class=self.bbox_head.num_classes) + loss_bbox = self.bbox_head.loss(cls_score, bbox_pred, rois, + *bbox_targets) + + # Add CARL Loss + carl_cfg = self.train_cfg.get('carl', None) + if carl_cfg is not None: + loss_carl = carl_loss( + cls_score, + bbox_targets[0], + bbox_pred, + bbox_targets[2], + self.bbox_head.loss_bbox, + **carl_cfg, + num_class=self.bbox_head.num_classes) + loss_bbox.update(loss_carl) + + bbox_results.update(loss_bbox=loss_bbox) + return bbox_results diff --git a/detection/mmdet/models/roi_heads/point_rend_roi_head.py b/detection/mmdet/models/roi_heads/point_rend_roi_head.py new file mode 100644 index 0000000..478cdf5 --- /dev/null +++ b/detection/mmdet/models/roi_heads/point_rend_roi_head.py @@ -0,0 +1,218 @@ +# Modified from https://github.com/facebookresearch/detectron2/tree/master/projects/PointRend # noqa + +import torch +import torch.nn.functional as F +from mmcv.ops import point_sample, rel_roi_point_to_rel_img_point + +from mmdet.core import bbox2roi, bbox_mapping, merge_aug_masks +from .. import builder +from ..builder import HEADS +from .standard_roi_head import StandardRoIHead + + +@HEADS.register_module() +class PointRendRoIHead(StandardRoIHead): + """`PointRend `_.""" + + def __init__(self, point_head, *args, **kwargs): + super().__init__(*args, **kwargs) + assert self.with_bbox and self.with_mask + self.init_point_head(point_head) + + def init_point_head(self, point_head): + """Initialize ``point_head``""" + self.point_head = builder.build_head(point_head) + + def init_weights(self, pretrained): + """Initialize the weights in head. + + Args: + pretrained (str, optional): Path to pre-trained weights. + """ + super().init_weights(pretrained) + self.point_head.init_weights() + + def _mask_forward_train(self, x, sampling_results, bbox_feats, gt_masks, + img_metas): + """Run forward function and calculate loss for mask head and point head + in training.""" + mask_results = super()._mask_forward_train(x, sampling_results, + bbox_feats, gt_masks, + img_metas) + if mask_results['loss_mask'] is not None: + loss_point = self._mask_point_forward_train( + x, sampling_results, mask_results['mask_pred'], gt_masks, + img_metas) + mask_results['loss_mask'].update(loss_point) + + return mask_results + + def _mask_point_forward_train(self, x, sampling_results, mask_pred, + gt_masks, img_metas): + """Run forward function and calculate loss for point head in + training.""" + pos_labels = torch.cat([res.pos_gt_labels for res in sampling_results]) + rel_roi_points = self.point_head.get_roi_rel_points_train( + mask_pred, pos_labels, cfg=self.train_cfg) + rois = bbox2roi([res.pos_bboxes for res in sampling_results]) + + fine_grained_point_feats = self._get_fine_grained_point_feats( + x, rois, rel_roi_points, img_metas) + coarse_point_feats = point_sample(mask_pred, rel_roi_points) + mask_point_pred = self.point_head(fine_grained_point_feats, + coarse_point_feats) + mask_point_target = self.point_head.get_targets( + rois, rel_roi_points, sampling_results, gt_masks, self.train_cfg) + loss_mask_point = self.point_head.loss(mask_point_pred, + mask_point_target, pos_labels) + + return loss_mask_point + + def _get_fine_grained_point_feats(self, x, rois, rel_roi_points, + img_metas): + """Sample fine grained feats from each level feature map and + concatenate them together.""" + num_imgs = len(img_metas) + fine_grained_feats = [] + for idx in range(self.mask_roi_extractor.num_inputs): + feats = x[idx] + spatial_scale = 1. / float( + self.mask_roi_extractor.featmap_strides[idx]) + point_feats = [] + for batch_ind in range(num_imgs): + # unravel batch dim + feat = feats[batch_ind].unsqueeze(0) + inds = (rois[:, 0].long() == batch_ind) + if inds.any(): + rel_img_points = rel_roi_point_to_rel_img_point( + rois[inds], rel_roi_points[inds], feat.shape[2:], + spatial_scale).unsqueeze(0) + point_feat = point_sample(feat, rel_img_points) + point_feat = point_feat.squeeze(0).transpose(0, 1) + point_feats.append(point_feat) + fine_grained_feats.append(torch.cat(point_feats, dim=0)) + return torch.cat(fine_grained_feats, dim=1) + + def _mask_point_forward_test(self, x, rois, label_pred, mask_pred, + img_metas): + """Mask refining process with point head in testing.""" + refined_mask_pred = mask_pred.clone() + for subdivision_step in range(self.test_cfg.subdivision_steps): + refined_mask_pred = F.interpolate( + refined_mask_pred, + scale_factor=self.test_cfg.scale_factor, + mode='bilinear', + align_corners=False) + # If `subdivision_num_points` is larger or equal to the + # resolution of the next step, then we can skip this step + num_rois, channels, mask_height, mask_width = \ + refined_mask_pred.shape + if (self.test_cfg.subdivision_num_points >= + self.test_cfg.scale_factor**2 * mask_height * mask_width + and + subdivision_step < self.test_cfg.subdivision_steps - 1): + continue + point_indices, rel_roi_points = \ + self.point_head.get_roi_rel_points_test( + refined_mask_pred, label_pred, cfg=self.test_cfg) + fine_grained_point_feats = self._get_fine_grained_point_feats( + x, rois, rel_roi_points, img_metas) + coarse_point_feats = point_sample(mask_pred, rel_roi_points) + mask_point_pred = self.point_head(fine_grained_point_feats, + coarse_point_feats) + + point_indices = point_indices.unsqueeze(1).expand(-1, channels, -1) + refined_mask_pred = refined_mask_pred.reshape( + num_rois, channels, mask_height * mask_width) + refined_mask_pred = refined_mask_pred.scatter_( + 2, point_indices, mask_point_pred) + refined_mask_pred = refined_mask_pred.view(num_rois, channels, + mask_height, mask_width) + + return refined_mask_pred + + def simple_test_mask(self, + x, + img_metas, + det_bboxes, + det_labels, + rescale=False): + """Obtain mask prediction without augmentation.""" + ori_shapes = tuple(meta['ori_shape'] for meta in img_metas) + scale_factors = tuple(meta['scale_factor'] for meta in img_metas) + num_imgs = len(det_bboxes) + if all(det_bbox.shape[0] == 0 for det_bbox in det_bboxes): + segm_results = [[[] for _ in range(self.mask_head.num_classes)] + for _ in range(num_imgs)] + else: + # if det_bboxes is rescaled to the original image size, we need to + # rescale it back to the testing scale to obtain RoIs. + if rescale and not isinstance(scale_factors[0], float): + scale_factors = [ + torch.from_numpy(scale_factor).to(det_bboxes[0].device) + for scale_factor in scale_factors + ] + _bboxes = [ + det_bboxes[i][:, :4] * + scale_factors[i] if rescale else det_bboxes[i][:, :4] + for i in range(len(det_bboxes)) + ] + mask_rois = bbox2roi(_bboxes) + mask_results = self._mask_forward(x, mask_rois) + # split batch mask prediction back to each image + mask_pred = mask_results['mask_pred'] + num_mask_roi_per_img = [len(det_bbox) for det_bbox in det_bboxes] + mask_preds = mask_pred.split(num_mask_roi_per_img, 0) + mask_rois = mask_rois.split(num_mask_roi_per_img, 0) + + # apply mask post-processing to each image individually + segm_results = [] + for i in range(num_imgs): + if det_bboxes[i].shape[0] == 0: + segm_results.append( + [[] for _ in range(self.mask_head.num_classes)]) + else: + x_i = [xx[[i]] for xx in x] + mask_rois_i = mask_rois[i] + mask_rois_i[:, 0] = 0 # TODO: remove this hack + mask_pred_i = self._mask_point_forward_test( + x_i, mask_rois_i, det_labels[i], mask_preds[i], + [img_metas]) + segm_result = self.mask_head.get_seg_masks( + mask_pred_i, _bboxes[i], det_labels[i], self.test_cfg, + ori_shapes[i], scale_factors[i], rescale) + segm_results.append(segm_result) + return segm_results + + def aug_test_mask(self, feats, img_metas, det_bboxes, det_labels): + """Test for mask head with test time augmentation.""" + if det_bboxes.shape[0] == 0: + segm_result = [[] for _ in range(self.mask_head.num_classes)] + else: + aug_masks = [] + for x, img_meta in zip(feats, img_metas): + img_shape = img_meta[0]['img_shape'] + scale_factor = img_meta[0]['scale_factor'] + flip = img_meta[0]['flip'] + _bboxes = bbox_mapping(det_bboxes[:, :4], img_shape, + scale_factor, flip) + mask_rois = bbox2roi([_bboxes]) + mask_results = self._mask_forward(x, mask_rois) + mask_results['mask_pred'] = self._mask_point_forward_test( + x, mask_rois, det_labels, mask_results['mask_pred'], + img_metas) + # convert to numpy array to save memory + aug_masks.append( + mask_results['mask_pred'].sigmoid().cpu().numpy()) + merged_masks = merge_aug_masks(aug_masks, img_metas, self.test_cfg) + + ori_shape = img_metas[0][0]['ori_shape'] + segm_result = self.mask_head.get_seg_masks( + merged_masks, + det_bboxes, + det_labels, + self.test_cfg, + ori_shape, + scale_factor=1.0, + rescale=False) + return segm_result diff --git a/detection/mmdet/models/roi_heads/roi_extractors/__init__.py b/detection/mmdet/models/roi_heads/roi_extractors/__init__.py new file mode 100644 index 0000000..a6ec0ec --- /dev/null +++ b/detection/mmdet/models/roi_heads/roi_extractors/__init__.py @@ -0,0 +1,7 @@ +from .generic_roi_extractor import GenericRoIExtractor +from .single_level_roi_extractor import SingleRoIExtractor + +__all__ = [ + 'SingleRoIExtractor', + 'GenericRoIExtractor', +] diff --git a/detection/mmdet/models/roi_heads/roi_extractors/base_roi_extractor.py b/detection/mmdet/models/roi_heads/roi_extractors/base_roi_extractor.py new file mode 100644 index 0000000..8479325 --- /dev/null +++ b/detection/mmdet/models/roi_heads/roi_extractors/base_roi_extractor.py @@ -0,0 +1,83 @@ +from abc import ABCMeta, abstractmethod + +import torch +import torch.nn as nn +from mmcv import ops + + +class BaseRoIExtractor(nn.Module, metaclass=ABCMeta): + """Base class for RoI extractor. + + Args: + roi_layer (dict): Specify RoI layer type and arguments. + out_channels (int): Output channels of RoI layers. + featmap_strides (List[int]): Strides of input feature maps. + """ + + def __init__(self, roi_layer, out_channels, featmap_strides): + super(BaseRoIExtractor, self).__init__() + self.roi_layers = self.build_roi_layers(roi_layer, featmap_strides) + self.out_channels = out_channels + self.featmap_strides = featmap_strides + self.fp16_enabled = False + + @property + def num_inputs(self): + """int: Number of input feature maps.""" + return len(self.featmap_strides) + + def init_weights(self): + pass + + def build_roi_layers(self, layer_cfg, featmap_strides): + """Build RoI operator to extract feature from each level feature map. + + Args: + layer_cfg (dict): Dictionary to construct and config RoI layer + operation. Options are modules under ``mmcv/ops`` such as + ``RoIAlign``. + featmap_strides (List[int]): The stride of input feature map w.r.t + to the original image size, which would be used to scale RoI + coordinate (original image coordinate system) to feature + coordinate system. + + Returns: + nn.ModuleList: The RoI extractor modules for each level feature + map. + """ + + cfg = layer_cfg.copy() + layer_type = cfg.pop('type') + assert hasattr(ops, layer_type) + layer_cls = getattr(ops, layer_type) + roi_layers = nn.ModuleList( + [layer_cls(spatial_scale=1 / s, **cfg) for s in featmap_strides]) + return roi_layers + + def roi_rescale(self, rois, scale_factor): + """Scale RoI coordinates by scale factor. + + Args: + rois (torch.Tensor): RoI (Region of Interest), shape (n, 5) + scale_factor (float): Scale factor that RoI will be multiplied by. + + Returns: + torch.Tensor: Scaled RoI. + """ + + cx = (rois[:, 1] + rois[:, 3]) * 0.5 + cy = (rois[:, 2] + rois[:, 4]) * 0.5 + w = rois[:, 3] - rois[:, 1] + h = rois[:, 4] - rois[:, 2] + new_w = w * scale_factor + new_h = h * scale_factor + x1 = cx - new_w * 0.5 + x2 = cx + new_w * 0.5 + y1 = cy - new_h * 0.5 + y2 = cy + new_h * 0.5 + new_rois = torch.stack((rois[:, 0], x1, y1, x2, y2), dim=-1) + return new_rois + + @abstractmethod + def forward(self, feats, rois, roi_scale_factor=None): + pass diff --git a/detection/mmdet/models/roi_heads/roi_extractors/generic_roi_extractor.py b/detection/mmdet/models/roi_heads/roi_extractors/generic_roi_extractor.py new file mode 100644 index 0000000..80c25bb --- /dev/null +++ b/detection/mmdet/models/roi_heads/roi_extractors/generic_roi_extractor.py @@ -0,0 +1,83 @@ +from mmcv.cnn.bricks import build_plugin_layer +from mmcv.runner import force_fp32 + +from mmdet.models.builder import ROI_EXTRACTORS +from .base_roi_extractor import BaseRoIExtractor + + +@ROI_EXTRACTORS.register_module() +class GenericRoIExtractor(BaseRoIExtractor): + """Extract RoI features from all level feature maps levels. + + This is the implementation of `A novel Region of Interest Extraction Layer + for Instance Segmentation `_. + + Args: + aggregation (str): The method to aggregate multiple feature maps. + Options are 'sum', 'concat'. Default: 'sum'. + pre_cfg (dict | None): Specify pre-processing modules. Default: None. + post_cfg (dict | None): Specify post-processing modules. Default: None. + kwargs (keyword arguments): Arguments that are the same + as :class:`BaseRoIExtractor`. + """ + + def __init__(self, + aggregation='sum', + pre_cfg=None, + post_cfg=None, + **kwargs): + super(GenericRoIExtractor, self).__init__(**kwargs) + + assert aggregation in ['sum', 'concat'] + + self.aggregation = aggregation + self.with_post = post_cfg is not None + self.with_pre = pre_cfg is not None + # build pre/post processing modules + if self.with_post: + self.post_module = build_plugin_layer(post_cfg, '_post_module')[1] + if self.with_pre: + self.pre_module = build_plugin_layer(pre_cfg, '_pre_module')[1] + + @force_fp32(apply_to=('feats', ), out_fp16=True) + def forward(self, feats, rois, roi_scale_factor=None): + """Forward function.""" + if len(feats) == 1: + return self.roi_layers[0](feats[0], rois) + + out_size = self.roi_layers[0].output_size + num_levels = len(feats) + roi_feats = feats[0].new_zeros( + rois.size(0), self.out_channels, *out_size) + + # some times rois is an empty tensor + if roi_feats.shape[0] == 0: + return roi_feats + + if roi_scale_factor is not None: + rois = self.roi_rescale(rois, roi_scale_factor) + + # mark the starting channels for concat mode + start_channels = 0 + for i in range(num_levels): + roi_feats_t = self.roi_layers[i](feats[i], rois) + end_channels = start_channels + roi_feats_t.size(1) + if self.with_pre: + # apply pre-processing to a RoI extracted from each layer + roi_feats_t = self.pre_module(roi_feats_t) + if self.aggregation == 'sum': + # and sum them all + roi_feats += roi_feats_t + else: + # and concat them along channel dimension + roi_feats[:, start_channels:end_channels] = roi_feats_t + # update channels starting position + start_channels = end_channels + # check if concat channels match at the end + if self.aggregation == 'concat': + assert start_channels == self.out_channels + + if self.with_post: + # apply post-processing before return the result + roi_feats = self.post_module(roi_feats) + return roi_feats diff --git a/detection/mmdet/models/roi_heads/roi_extractors/single_level_roi_extractor.py b/detection/mmdet/models/roi_heads/roi_extractors/single_level_roi_extractor.py new file mode 100644 index 0000000..cfc838f --- /dev/null +++ b/detection/mmdet/models/roi_heads/roi_extractors/single_level_roi_extractor.py @@ -0,0 +1,108 @@ +import torch +from mmcv.runner import force_fp32 + +from mmdet.models.builder import ROI_EXTRACTORS +from .base_roi_extractor import BaseRoIExtractor + + +@ROI_EXTRACTORS.register_module() +class SingleRoIExtractor(BaseRoIExtractor): + """Extract RoI features from a single level feature map. + + If there are multiple input feature levels, each RoI is mapped to a level + according to its scale. The mapping rule is proposed in + `FPN `_. + + Args: + roi_layer (dict): Specify RoI layer type and arguments. + out_channels (int): Output channels of RoI layers. + featmap_strides (List[int]): Strides of input feature maps. + finest_scale (int): Scale threshold of mapping to level 0. Default: 56. + """ + + def __init__(self, + roi_layer, + out_channels, + featmap_strides, + finest_scale=56): + super(SingleRoIExtractor, self).__init__(roi_layer, out_channels, + featmap_strides) + self.finest_scale = finest_scale + + def map_roi_levels(self, rois, num_levels): + """Map rois to corresponding feature levels by scales. + + - scale < finest_scale * 2: level 0 + - finest_scale * 2 <= scale < finest_scale * 4: level 1 + - finest_scale * 4 <= scale < finest_scale * 8: level 2 + - scale >= finest_scale * 8: level 3 + + Args: + rois (Tensor): Input RoIs, shape (k, 5). + num_levels (int): Total level number. + + Returns: + Tensor: Level index (0-based) of each RoI, shape (k, ) + """ + scale = torch.sqrt( + (rois[:, 3] - rois[:, 1]) * (rois[:, 4] - rois[:, 2])) + target_lvls = torch.floor(torch.log2(scale / self.finest_scale + 1e-6)) + target_lvls = target_lvls.clamp(min=0, max=num_levels - 1).long() + return target_lvls + + @force_fp32(apply_to=('feats', ), out_fp16=True) + def forward(self, feats, rois, roi_scale_factor=None): + """Forward function.""" + out_size = self.roi_layers[0].output_size + num_levels = len(feats) + expand_dims = (-1, self.out_channels * out_size[0] * out_size[1]) + if torch.onnx.is_in_onnx_export(): + # Work around to export mask-rcnn to onnx + roi_feats = rois[:, :1].clone().detach() + roi_feats = roi_feats.expand(*expand_dims) + roi_feats = roi_feats.reshape(-1, self.out_channels, *out_size) + roi_feats = roi_feats * 0 + else: + roi_feats = feats[0].new_zeros( + rois.size(0), self.out_channels, *out_size) + # TODO: remove this when parrots supports + if torch.__version__ == 'parrots': + roi_feats.requires_grad = True + + if num_levels == 1: + if len(rois) == 0: + return roi_feats + return self.roi_layers[0](feats[0], rois) + + target_lvls = self.map_roi_levels(rois, num_levels) + + if roi_scale_factor is not None: + rois = self.roi_rescale(rois, roi_scale_factor) + + for i in range(num_levels): + mask = target_lvls == i + if torch.onnx.is_in_onnx_export(): + # To keep all roi_align nodes exported to onnx + # and skip nonzero op + mask = mask.float().unsqueeze(-1).expand(*expand_dims).reshape( + roi_feats.shape) + roi_feats_t = self.roi_layers[i](feats[i], rois) + roi_feats_t *= mask + roi_feats += roi_feats_t + continue + inds = mask.nonzero(as_tuple=False).squeeze(1) + if inds.numel() > 0: + rois_ = rois[inds] + roi_feats_t = self.roi_layers[i](feats[i], rois_) + roi_feats[inds] = roi_feats_t + else: + # Sometimes some pyramid levels will not be used for RoI + # feature extraction and this will cause an incomplete + # computation graph in one GPU, which is different from those + # in other GPUs and will cause a hanging error. + # Therefore, we add it to ensure each feature pyramid is + # included in the computation graph to avoid runtime bugs. + roi_feats += sum( + x.view(-1)[0] + for x in self.parameters()) * 0. + feats[i].sum() * 0. + return roi_feats diff --git a/detection/mmdet/models/roi_heads/scnet_roi_head.py b/detection/mmdet/models/roi_heads/scnet_roi_head.py new file mode 100644 index 0000000..85aaa2f --- /dev/null +++ b/detection/mmdet/models/roi_heads/scnet_roi_head.py @@ -0,0 +1,582 @@ +import torch +import torch.nn.functional as F + +from mmdet.core import (bbox2result, bbox2roi, bbox_mapping, merge_aug_bboxes, + merge_aug_masks, multiclass_nms) +from ..builder import HEADS, build_head, build_roi_extractor +from .cascade_roi_head import CascadeRoIHead + + +@HEADS.register_module() +class SCNetRoIHead(CascadeRoIHead): + """RoIHead for `SCNet `_. + + Args: + num_stages (int): number of cascade stages. + stage_loss_weights (list): loss weight of cascade stages. + semantic_roi_extractor (dict): config to init semantic roi extractor. + semantic_head (dict): config to init semantic head. + feat_relay_head (dict): config to init feature_relay_head. + glbctx_head (dict): config to init global context head. + """ + + def __init__(self, + num_stages, + stage_loss_weights, + semantic_roi_extractor=None, + semantic_head=None, + feat_relay_head=None, + glbctx_head=None, + **kwargs): + super(SCNetRoIHead, self).__init__(num_stages, stage_loss_weights, + **kwargs) + assert self.with_bbox and self.with_mask + assert not self.with_shared_head # shared head is not supported + + if semantic_head is not None: + self.semantic_roi_extractor = build_roi_extractor( + semantic_roi_extractor) + self.semantic_head = build_head(semantic_head) + + if feat_relay_head is not None: + self.feat_relay_head = build_head(feat_relay_head) + + if glbctx_head is not None: + self.glbctx_head = build_head(glbctx_head) + + def init_mask_head(self, mask_roi_extractor, mask_head): + """Initialize ``mask_head``""" + if mask_roi_extractor is not None: + self.mask_roi_extractor = build_roi_extractor(mask_roi_extractor) + self.mask_head = build_head(mask_head) + + def init_weights(self, pretrained): + """Initialize the weights in head. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + for i in range(self.num_stages): + if self.with_bbox: + self.bbox_roi_extractor[i].init_weights() + self.bbox_head[i].init_weights() + if self.with_mask: + self.mask_roi_extractor.init_weights() + self.mask_head.init_weights() + if self.with_semantic: + self.semantic_head.init_weights() + if self.with_glbctx: + self.glbctx_head.init_weights() + if self.with_feat_relay: + self.feat_relay_head.init_weights() + + @property + def with_semantic(self): + """bool: whether the head has semantic head""" + return hasattr(self, + 'semantic_head') and self.semantic_head is not None + + @property + def with_feat_relay(self): + """bool: whether the head has feature relay head""" + return (hasattr(self, 'feat_relay_head') + and self.feat_relay_head is not None) + + @property + def with_glbctx(self): + """bool: whether the head has global context head""" + return hasattr(self, 'glbctx_head') and self.glbctx_head is not None + + def _fuse_glbctx(self, roi_feats, glbctx_feat, rois): + """Fuse global context feats with roi feats.""" + assert roi_feats.size(0) == rois.size(0) + img_inds = torch.unique(rois[:, 0].cpu(), sorted=True).long() + fused_feats = torch.zeros_like(roi_feats) + for img_id in img_inds: + inds = (rois[:, 0] == img_id.item()) + fused_feats[inds] = roi_feats[inds] + glbctx_feat[img_id] + return fused_feats + + def _slice_pos_feats(self, feats, sampling_results): + """Get features from pos rois.""" + num_rois = [res.bboxes.size(0) for res in sampling_results] + num_pos_rois = [res.pos_bboxes.size(0) for res in sampling_results] + inds = torch.zeros(sum(num_rois), dtype=torch.bool) + start = 0 + for i in range(len(num_rois)): + start = 0 if i == 0 else start + num_rois[i - 1] + stop = start + num_pos_rois[i] + inds[start:stop] = 1 + sliced_feats = feats[inds] + return sliced_feats + + def _bbox_forward(self, + stage, + x, + rois, + semantic_feat=None, + glbctx_feat=None): + """Box head forward function used in both training and testing.""" + bbox_roi_extractor = self.bbox_roi_extractor[stage] + bbox_head = self.bbox_head[stage] + bbox_feats = bbox_roi_extractor( + x[:len(bbox_roi_extractor.featmap_strides)], rois) + if self.with_semantic and semantic_feat is not None: + bbox_semantic_feat = self.semantic_roi_extractor([semantic_feat], + rois) + if bbox_semantic_feat.shape[-2:] != bbox_feats.shape[-2:]: + bbox_semantic_feat = F.adaptive_avg_pool2d( + bbox_semantic_feat, bbox_feats.shape[-2:]) + bbox_feats += bbox_semantic_feat + if self.with_glbctx and glbctx_feat is not None: + bbox_feats = self._fuse_glbctx(bbox_feats, glbctx_feat, rois) + cls_score, bbox_pred, relayed_feat = bbox_head( + bbox_feats, return_shared_feat=True) + + bbox_results = dict( + cls_score=cls_score, + bbox_pred=bbox_pred, + relayed_feat=relayed_feat) + return bbox_results + + def _mask_forward(self, + x, + rois, + semantic_feat=None, + glbctx_feat=None, + relayed_feat=None): + """Mask head forward function used in both training and testing.""" + mask_feats = self.mask_roi_extractor( + x[:self.mask_roi_extractor.num_inputs], rois) + if self.with_semantic and semantic_feat is not None: + mask_semantic_feat = self.semantic_roi_extractor([semantic_feat], + rois) + if mask_semantic_feat.shape[-2:] != mask_feats.shape[-2:]: + mask_semantic_feat = F.adaptive_avg_pool2d( + mask_semantic_feat, mask_feats.shape[-2:]) + mask_feats += mask_semantic_feat + if self.with_glbctx and glbctx_feat is not None: + mask_feats = self._fuse_glbctx(mask_feats, glbctx_feat, rois) + if self.with_feat_relay and relayed_feat is not None: + mask_feats = mask_feats + relayed_feat + mask_pred = self.mask_head(mask_feats) + mask_results = dict(mask_pred=mask_pred) + + return mask_results + + def _bbox_forward_train(self, + stage, + x, + sampling_results, + gt_bboxes, + gt_labels, + rcnn_train_cfg, + semantic_feat=None, + glbctx_feat=None): + """Run forward function and calculate loss for box head in training.""" + bbox_head = self.bbox_head[stage] + rois = bbox2roi([res.bboxes for res in sampling_results]) + bbox_results = self._bbox_forward( + stage, + x, + rois, + semantic_feat=semantic_feat, + glbctx_feat=glbctx_feat) + + bbox_targets = bbox_head.get_targets(sampling_results, gt_bboxes, + gt_labels, rcnn_train_cfg) + loss_bbox = bbox_head.loss(bbox_results['cls_score'], + bbox_results['bbox_pred'], rois, + *bbox_targets) + + bbox_results.update( + loss_bbox=loss_bbox, rois=rois, bbox_targets=bbox_targets) + return bbox_results + + def _mask_forward_train(self, + x, + sampling_results, + gt_masks, + rcnn_train_cfg, + semantic_feat=None, + glbctx_feat=None, + relayed_feat=None): + """Run forward function and calculate loss for mask head in + training.""" + pos_rois = bbox2roi([res.pos_bboxes for res in sampling_results]) + mask_results = self._mask_forward( + x, + pos_rois, + semantic_feat=semantic_feat, + glbctx_feat=glbctx_feat, + relayed_feat=relayed_feat) + + mask_targets = self.mask_head.get_targets(sampling_results, gt_masks, + rcnn_train_cfg) + pos_labels = torch.cat([res.pos_gt_labels for res in sampling_results]) + loss_mask = self.mask_head.loss(mask_results['mask_pred'], + mask_targets, pos_labels) + + mask_results = loss_mask + return mask_results + + def forward_train(self, + x, + img_metas, + proposal_list, + gt_bboxes, + gt_labels, + gt_bboxes_ignore=None, + gt_masks=None, + gt_semantic_seg=None): + """ + Args: + x (list[Tensor]): list of multi-level img features. + + img_metas (list[dict]): list of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmdet/datasets/pipelines/formatting.py:Collect`. + + proposal_list (list[Tensors]): list of region proposals. + + gt_bboxes (list[Tensor]): Ground truth bboxes for each image with + shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + + gt_labels (list[Tensor]): class indices corresponding to each box + + gt_bboxes_ignore (None, list[Tensor]): specify which bounding + boxes can be ignored when computing the loss. + + gt_masks (None, Tensor) : true segmentation masks for each box + used if the architecture supports a segmentation task. + + gt_semantic_seg (None, list[Tensor]): semantic segmentation masks + used if the architecture supports semantic segmentation task. + + Returns: + dict[str, Tensor]: a dictionary of loss components + """ + losses = dict() + + # semantic segmentation branch + if self.with_semantic: + semantic_pred, semantic_feat = self.semantic_head(x) + loss_seg = self.semantic_head.loss(semantic_pred, gt_semantic_seg) + losses['loss_semantic_seg'] = loss_seg + else: + semantic_feat = None + + # global context branch + if self.with_glbctx: + mc_pred, glbctx_feat = self.glbctx_head(x) + loss_glbctx = self.glbctx_head.loss(mc_pred, gt_labels) + losses['loss_glbctx'] = loss_glbctx + else: + glbctx_feat = None + + for i in range(self.num_stages): + self.current_stage = i + rcnn_train_cfg = self.train_cfg[i] + lw = self.stage_loss_weights[i] + + # assign gts and sample proposals + sampling_results = [] + bbox_assigner = self.bbox_assigner[i] + bbox_sampler = self.bbox_sampler[i] + num_imgs = len(img_metas) + if gt_bboxes_ignore is None: + gt_bboxes_ignore = [None for _ in range(num_imgs)] + + for j in range(num_imgs): + assign_result = bbox_assigner.assign(proposal_list[j], + gt_bboxes[j], + gt_bboxes_ignore[j], + gt_labels[j]) + sampling_result = bbox_sampler.sample( + assign_result, + proposal_list[j], + gt_bboxes[j], + gt_labels[j], + feats=[lvl_feat[j][None] for lvl_feat in x]) + sampling_results.append(sampling_result) + + bbox_results = \ + self._bbox_forward_train( + i, x, sampling_results, gt_bboxes, gt_labels, + rcnn_train_cfg, semantic_feat, glbctx_feat) + roi_labels = bbox_results['bbox_targets'][0] + + for name, value in bbox_results['loss_bbox'].items(): + losses[f's{i}.{name}'] = ( + value * lw if 'loss' in name else value) + + # refine boxes + if i < self.num_stages - 1: + pos_is_gts = [res.pos_is_gt for res in sampling_results] + with torch.no_grad(): + proposal_list = self.bbox_head[i].refine_bboxes( + bbox_results['rois'], roi_labels, + bbox_results['bbox_pred'], pos_is_gts, img_metas) + + if self.with_feat_relay: + relayed_feat = self._slice_pos_feats(bbox_results['relayed_feat'], + sampling_results) + relayed_feat = self.feat_relay_head(relayed_feat) + else: + relayed_feat = None + + mask_results = self._mask_forward_train(x, sampling_results, gt_masks, + rcnn_train_cfg, semantic_feat, + glbctx_feat, relayed_feat) + mask_lw = sum(self.stage_loss_weights) + losses['loss_mask'] = mask_lw * mask_results['loss_mask'] + + return losses + + def simple_test(self, x, proposal_list, img_metas, rescale=False): + """Test without augmentation.""" + if self.with_semantic: + _, semantic_feat = self.semantic_head(x) + else: + semantic_feat = None + + if self.with_glbctx: + mc_pred, glbctx_feat = self.glbctx_head(x) + else: + glbctx_feat = None + + num_imgs = len(proposal_list) + img_shapes = tuple(meta['img_shape'] for meta in img_metas) + ori_shapes = tuple(meta['ori_shape'] for meta in img_metas) + scale_factors = tuple(meta['scale_factor'] for meta in img_metas) + + # "ms" in variable names means multi-stage + ms_scores = [] + rcnn_test_cfg = self.test_cfg + + rois = bbox2roi(proposal_list) + for i in range(self.num_stages): + bbox_head = self.bbox_head[i] + bbox_results = self._bbox_forward( + i, + x, + rois, + semantic_feat=semantic_feat, + glbctx_feat=glbctx_feat) + # split batch bbox prediction back to each image + cls_score = bbox_results['cls_score'] + bbox_pred = bbox_results['bbox_pred'] + num_proposals_per_img = tuple(len(p) for p in proposal_list) + rois = rois.split(num_proposals_per_img, 0) + cls_score = cls_score.split(num_proposals_per_img, 0) + bbox_pred = bbox_pred.split(num_proposals_per_img, 0) + ms_scores.append(cls_score) + + if i < self.num_stages - 1: + bbox_label = [s[:, :-1].argmax(dim=1) for s in cls_score] + rois = torch.cat([ + bbox_head.regress_by_class(rois[i], bbox_label[i], + bbox_pred[i], img_metas[i]) + for i in range(num_imgs) + ]) + + # average scores of each image by stages + cls_score = [ + sum([score[i] for score in ms_scores]) / float(len(ms_scores)) + for i in range(num_imgs) + ] + + # apply bbox post-processing to each image individually + det_bboxes = [] + det_labels = [] + for i in range(num_imgs): + det_bbox, det_label = self.bbox_head[-1].get_bboxes( + rois[i], + cls_score[i], + bbox_pred[i], + img_shapes[i], + scale_factors[i], + rescale=rescale, + cfg=rcnn_test_cfg) + det_bboxes.append(det_bbox) + det_labels.append(det_label) + det_bbox_results = [ + bbox2result(det_bboxes[i], det_labels[i], + self.bbox_head[-1].num_classes) + for i in range(num_imgs) + ] + + if self.with_mask: + if all(det_bbox.shape[0] == 0 for det_bbox in det_bboxes): + mask_classes = self.mask_head.num_classes + det_segm_results = [[[] for _ in range(mask_classes)] + for _ in range(num_imgs)] + else: + if rescale and not isinstance(scale_factors[0], float): + scale_factors = [ + torch.from_numpy(scale_factor).to(det_bboxes[0].device) + for scale_factor in scale_factors + ] + _bboxes = [ + det_bboxes[i][:, :4] * + scale_factors[i] if rescale else det_bboxes[i] + for i in range(num_imgs) + ] + mask_rois = bbox2roi(_bboxes) + + # get relay feature on mask_rois + bbox_results = self._bbox_forward( + -1, + x, + mask_rois, + semantic_feat=semantic_feat, + glbctx_feat=glbctx_feat) + relayed_feat = bbox_results['relayed_feat'] + relayed_feat = self.feat_relay_head(relayed_feat) + + mask_results = self._mask_forward( + x, + mask_rois, + semantic_feat=semantic_feat, + glbctx_feat=glbctx_feat, + relayed_feat=relayed_feat) + mask_pred = mask_results['mask_pred'] + + # split batch mask prediction back to each image + num_bbox_per_img = tuple(len(_bbox) for _bbox in _bboxes) + mask_preds = mask_pred.split(num_bbox_per_img, 0) + + # apply mask post-processing to each image individually + det_segm_results = [] + for i in range(num_imgs): + if det_bboxes[i].shape[0] == 0: + det_segm_results.append( + [[] for _ in range(self.mask_head.num_classes)]) + else: + segm_result = self.mask_head.get_seg_masks( + mask_preds[i], _bboxes[i], det_labels[i], + self.test_cfg, ori_shapes[i], scale_factors[i], + rescale) + det_segm_results.append(segm_result) + + # return results + if self.with_mask: + return list(zip(det_bbox_results, det_segm_results)) + else: + return det_bbox_results + + def aug_test(self, img_feats, proposal_list, img_metas, rescale=False): + if self.with_semantic: + semantic_feats = [ + self.semantic_head(feat)[1] for feat in img_feats + ] + else: + semantic_feats = [None] * len(img_metas) + + if self.with_glbctx: + glbctx_feats = [self.glbctx_head(feat)[1] for feat in img_feats] + else: + glbctx_feats = [None] * len(img_metas) + + rcnn_test_cfg = self.test_cfg + aug_bboxes = [] + aug_scores = [] + for x, img_meta, semantic_feat, glbctx_feat in zip( + img_feats, img_metas, semantic_feats, glbctx_feats): + # only one image in the batch + img_shape = img_meta[0]['img_shape'] + scale_factor = img_meta[0]['scale_factor'] + flip = img_meta[0]['flip'] + + proposals = bbox_mapping(proposal_list[0][:, :4], img_shape, + scale_factor, flip) + # "ms" in variable names means multi-stage + ms_scores = [] + + rois = bbox2roi([proposals]) + for i in range(self.num_stages): + bbox_head = self.bbox_head[i] + bbox_results = self._bbox_forward( + i, + x, + rois, + semantic_feat=semantic_feat, + glbctx_feat=glbctx_feat) + ms_scores.append(bbox_results['cls_score']) + if i < self.num_stages - 1: + bbox_label = bbox_results['cls_score'].argmax(dim=1) + rois = bbox_head.regress_by_class( + rois, bbox_label, bbox_results['bbox_pred'], + img_meta[0]) + + cls_score = sum(ms_scores) / float(len(ms_scores)) + bboxes, scores = self.bbox_head[-1].get_bboxes( + rois, + cls_score, + bbox_results['bbox_pred'], + img_shape, + scale_factor, + rescale=False, + cfg=None) + aug_bboxes.append(bboxes) + aug_scores.append(scores) + + # after merging, bboxes will be rescaled to the original image size + merged_bboxes, merged_scores = merge_aug_bboxes( + aug_bboxes, aug_scores, img_metas, rcnn_test_cfg) + det_bboxes, det_labels = multiclass_nms(merged_bboxes, merged_scores, + rcnn_test_cfg.score_thr, + rcnn_test_cfg.nms, + rcnn_test_cfg.max_per_img) + + det_bbox_results = bbox2result(det_bboxes, det_labels, + self.bbox_head[-1].num_classes) + + if self.with_mask: + if det_bboxes.shape[0] == 0: + det_segm_results = [[] + for _ in range(self.mask_head.num_classes)] + else: + aug_masks = [] + for x, img_meta, semantic_feat, glbctx_feat in zip( + img_feats, img_metas, semantic_feats, glbctx_feats): + img_shape = img_meta[0]['img_shape'] + scale_factor = img_meta[0]['scale_factor'] + flip = img_meta[0]['flip'] + _bboxes = bbox_mapping(det_bboxes[:, :4], img_shape, + scale_factor, flip) + mask_rois = bbox2roi([_bboxes]) + # get relay feature on mask_rois + bbox_results = self._bbox_forward( + -1, + x, + mask_rois, + semantic_feat=semantic_feat, + glbctx_feat=glbctx_feat) + relayed_feat = bbox_results['relayed_feat'] + relayed_feat = self.feat_relay_head(relayed_feat) + mask_results = self._mask_forward( + x, + mask_rois, + semantic_feat=semantic_feat, + glbctx_feat=glbctx_feat, + relayed_feat=relayed_feat) + mask_pred = mask_results['mask_pred'] + aug_masks.append(mask_pred.sigmoid().cpu().numpy()) + merged_masks = merge_aug_masks(aug_masks, img_metas, + self.test_cfg) + ori_shape = img_metas[0][0]['ori_shape'] + det_segm_results = self.mask_head.get_seg_masks( + merged_masks, + det_bboxes, + det_labels, + rcnn_test_cfg, + ori_shape, + scale_factor=1.0, + rescale=False) + return [(det_bbox_results, det_segm_results)] + else: + return [det_bbox_results] diff --git a/detection/mmdet/models/roi_heads/shared_heads/__init__.py b/detection/mmdet/models/roi_heads/shared_heads/__init__.py new file mode 100644 index 0000000..bbe7014 --- /dev/null +++ b/detection/mmdet/models/roi_heads/shared_heads/__init__.py @@ -0,0 +1,3 @@ +from .res_layer import ResLayer + +__all__ = ['ResLayer'] diff --git a/detection/mmdet/models/roi_heads/shared_heads/res_layer.py b/detection/mmdet/models/roi_heads/shared_heads/res_layer.py new file mode 100644 index 0000000..b5c3432 --- /dev/null +++ b/detection/mmdet/models/roi_heads/shared_heads/res_layer.py @@ -0,0 +1,77 @@ +import torch.nn as nn +from mmcv.cnn import constant_init, kaiming_init +from mmcv.runner import auto_fp16, load_checkpoint + +from mmdet.models.backbones import ResNet +from mmdet.models.builder import SHARED_HEADS +from mmdet.models.utils import ResLayer as _ResLayer +from mmdet.utils import get_root_logger + + +@SHARED_HEADS.register_module() +class ResLayer(nn.Module): + + def __init__(self, + depth, + stage=3, + stride=2, + dilation=1, + style='pytorch', + norm_cfg=dict(type='BN', requires_grad=True), + norm_eval=True, + with_cp=False, + dcn=None): + super(ResLayer, self).__init__() + self.norm_eval = norm_eval + self.norm_cfg = norm_cfg + self.stage = stage + self.fp16_enabled = False + block, stage_blocks = ResNet.arch_settings[depth] + stage_block = stage_blocks[stage] + planes = 64 * 2**stage + inplanes = 64 * 2**(stage - 1) * block.expansion + + res_layer = _ResLayer( + block, + inplanes, + planes, + stage_block, + stride=stride, + dilation=dilation, + style=style, + with_cp=with_cp, + norm_cfg=self.norm_cfg, + dcn=dcn) + self.add_module(f'layer{stage + 1}', res_layer) + + def init_weights(self, pretrained=None): + """Initialize the weights in the module. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + if isinstance(pretrained, str): + logger = get_root_logger() + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + for m in self.modules(): + if isinstance(m, nn.Conv2d): + kaiming_init(m) + elif isinstance(m, nn.BatchNorm2d): + constant_init(m, 1) + else: + raise TypeError('pretrained must be a str or None') + + @auto_fp16() + def forward(self, x): + res_layer = getattr(self, f'layer{self.stage + 1}') + out = res_layer(x) + return out + + def train(self, mode=True): + super(ResLayer, self).train(mode) + if self.norm_eval: + for m in self.modules(): + if isinstance(m, nn.BatchNorm2d): + m.eval() diff --git a/detection/mmdet/models/roi_heads/sparse_roi_head.py b/detection/mmdet/models/roi_heads/sparse_roi_head.py new file mode 100644 index 0000000..8d85ebc --- /dev/null +++ b/detection/mmdet/models/roi_heads/sparse_roi_head.py @@ -0,0 +1,311 @@ +import torch + +from mmdet.core import bbox2result, bbox2roi, bbox_xyxy_to_cxcywh +from mmdet.core.bbox.samplers import PseudoSampler +from ..builder import HEADS +from .cascade_roi_head import CascadeRoIHead + + +@HEADS.register_module() +class SparseRoIHead(CascadeRoIHead): + r"""The RoIHead for `Sparse R-CNN: End-to-End Object Detection with + Learnable Proposals `_ + + Args: + num_stages (int): Number of stage whole iterative process. + Defaults to 6. + stage_loss_weights (Tuple[float]): The loss + weight of each stage. By default all stages have + the same weight 1. + bbox_roi_extractor (dict): Config of box roi extractor. + bbox_head (dict): Config of box head. + train_cfg (dict, optional): Configuration information in train stage. + Defaults to None. + test_cfg (dict, optional): Configuration information in test stage. + Defaults to None. + + """ + + def __init__(self, + num_stages=6, + stage_loss_weights=(1, 1, 1, 1, 1, 1), + proposal_feature_channel=256, + bbox_roi_extractor=dict( + type='SingleRoIExtractor', + roi_layer=dict( + type='RoIAlign', output_size=7, sampling_ratio=2), + out_channels=256, + featmap_strides=[4, 8, 16, 32]), + bbox_head=dict( + type='DIIHead', + num_classes=80, + num_fcs=2, + num_heads=8, + num_cls_fcs=1, + num_reg_fcs=3, + feedforward_channels=2048, + hidden_channels=256, + dropout=0.0, + roi_feat_size=7, + ffn_act_cfg=dict(type='ReLU', inplace=True)), + train_cfg=None, + test_cfg=None): + assert bbox_roi_extractor is not None + assert bbox_head is not None + assert len(stage_loss_weights) == num_stages + self.num_stages = num_stages + self.stage_loss_weights = stage_loss_weights + self.proposal_feature_channel = proposal_feature_channel + super(SparseRoIHead, self).__init__( + num_stages, + stage_loss_weights, + bbox_roi_extractor=bbox_roi_extractor, + bbox_head=bbox_head, + train_cfg=train_cfg, + test_cfg=test_cfg) + # train_cfg would be None when run the test.py + if train_cfg is not None: + for stage in range(num_stages): + assert isinstance(self.bbox_sampler[stage], PseudoSampler), \ + 'Sparse R-CNN only support `PseudoSampler`' + + def _bbox_forward(self, stage, x, rois, object_feats, img_metas): + """Box head forward function used in both training and testing. Returns + all regression, classification results and a intermediate feature. + + Args: + stage (int): The index of current stage in + iterative process. + x (List[Tensor]): List of FPN features + rois (Tensor): Rois in total batch. With shape (num_proposal, 5). + the last dimension 5 represents (img_index, x1, y1, x2, y2). + object_feats (Tensor): The object feature extracted from + the previous stage. + img_metas (dict): meta information of images. + + Returns: + dict[str, Tensor]: a dictionary of bbox head outputs, + Containing the following results: + + - cls_score (Tensor): The score of each class, has + shape (batch_size, num_proposals, num_classes) + when use focal loss or + (batch_size, num_proposals, num_classes+1) + otherwise. + - decode_bbox_pred (Tensor): The regression results + with shape (batch_size, num_proposal, 4). + The last dimension 4 represents + [tl_x, tl_y, br_x, br_y]. + - object_feats (Tensor): The object feature extracted + from current stage + - detach_cls_score_list (list[Tensor]): The detached + classification results, length is batch_size, and + each tensor has shape (num_proposal, num_classes). + - detach_proposal_list (list[tensor]): The detached + regression results, length is batch_size, and each + tensor has shape (num_proposal, 4). The last + dimension 4 represents [tl_x, tl_y, br_x, br_y]. + """ + num_imgs = len(img_metas) + bbox_roi_extractor = self.bbox_roi_extractor[stage] + bbox_head = self.bbox_head[stage] + bbox_feats = bbox_roi_extractor(x[:bbox_roi_extractor.num_inputs], + rois) + cls_score, bbox_pred, object_feats = bbox_head(bbox_feats, + object_feats) + proposal_list = self.bbox_head[stage].refine_bboxes( + rois, + rois.new_zeros(len(rois)), # dummy arg + bbox_pred.view(-1, bbox_pred.size(-1)), + [rois.new_zeros(object_feats.size(1)) for _ in range(num_imgs)], + img_metas) + bbox_results = dict( + cls_score=cls_score, + decode_bbox_pred=torch.cat(proposal_list), + object_feats=object_feats, + # detach then use it in label assign + detach_cls_score_list=[ + cls_score[i].detach() for i in range(num_imgs) + ], + detach_proposal_list=[item.detach() for item in proposal_list]) + + return bbox_results + + def forward_train(self, + x, + proposal_boxes, + proposal_features, + img_metas, + gt_bboxes, + gt_labels, + gt_bboxes_ignore=None, + imgs_whwh=None, + gt_masks=None): + """Forward function in training stage. + + Args: + x (list[Tensor]): list of multi-level img features. + proposals (Tensor): Decoded proposal bboxes, has shape + (batch_size, num_proposals, 4) + proposal_features (Tensor): Expanded proposal + features, has shape + (batch_size, num_proposals, proposal_feature_channel) + img_metas (list[dict]): list of image info dict where + each dict has: 'img_shape', 'scale_factor', 'flip', + and may also contain 'filename', 'ori_shape', + 'pad_shape', and 'img_norm_cfg'. For details on the + values of these keys see + `mmdet/datasets/pipelines/formatting.py:Collect`. + gt_bboxes (list[Tensor]): Ground truth bboxes for each image with + shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + gt_labels (list[Tensor]): class indices corresponding to each box + gt_bboxes_ignore (None | list[Tensor]): specify which bounding + boxes can be ignored when computing the loss. + imgs_whwh (Tensor): Tensor with shape (batch_size, 4), + the dimension means + [img_width,img_height, img_width, img_height]. + gt_masks (None | Tensor) : true segmentation masks for each box + used if the architecture supports a segmentation task. + + Returns: + dict[str, Tensor]: a dictionary of loss components of all stage. + """ + + num_imgs = len(img_metas) + num_proposals = proposal_boxes.size(1) + imgs_whwh = imgs_whwh.repeat(1, num_proposals, 1) + all_stage_bbox_results = [] + proposal_list = [proposal_boxes[i] for i in range(len(proposal_boxes))] + object_feats = proposal_features + all_stage_loss = {} + for stage in range(self.num_stages): + rois = bbox2roi(proposal_list) + bbox_results = self._bbox_forward(stage, x, rois, object_feats, + img_metas) + all_stage_bbox_results.append(bbox_results) + if gt_bboxes_ignore is None: + # TODO support ignore + gt_bboxes_ignore = [None for _ in range(num_imgs)] + sampling_results = [] + cls_pred_list = bbox_results['detach_cls_score_list'] + proposal_list = bbox_results['detach_proposal_list'] + for i in range(num_imgs): + normalize_bbox_ccwh = bbox_xyxy_to_cxcywh(proposal_list[i] / + imgs_whwh[i]) + assign_result = self.bbox_assigner[stage].assign( + normalize_bbox_ccwh, cls_pred_list[i], gt_bboxes[i], + gt_labels[i], img_metas[i]) + sampling_result = self.bbox_sampler[stage].sample( + assign_result, proposal_list[i], gt_bboxes[i]) + sampling_results.append(sampling_result) + bbox_targets = self.bbox_head[stage].get_targets( + sampling_results, gt_bboxes, gt_labels, self.train_cfg[stage], + True) + cls_score = bbox_results['cls_score'] + decode_bbox_pred = bbox_results['decode_bbox_pred'] + + single_stage_loss = self.bbox_head[stage].loss( + cls_score.view(-1, cls_score.size(-1)), + decode_bbox_pred.view(-1, 4), + *bbox_targets, + imgs_whwh=imgs_whwh) + for key, value in single_stage_loss.items(): + all_stage_loss[f'stage{stage}_{key}'] = value * \ + self.stage_loss_weights[stage] + object_feats = bbox_results['object_feats'] + + return all_stage_loss + + def simple_test(self, + x, + proposal_boxes, + proposal_features, + img_metas, + imgs_whwh, + rescale=False): + """Test without augmentation. + + Args: + x (list[Tensor]): list of multi-level img features. + proposal_boxes (Tensor): Decoded proposal bboxes, has shape + (batch_size, num_proposals, 4) + proposal_features (Tensor): Expanded proposal + features, has shape + (batch_size, num_proposals, proposal_feature_channel) + img_metas (dict): meta information of images. + imgs_whwh (Tensor): Tensor with shape (batch_size, 4), + the dimension means + [img_width,img_height, img_width, img_height]. + rescale (bool): If True, return boxes in original image + space. Defaults to False. + + Returns: + bbox_results (list[tuple[np.ndarray]]): \ + [[cls1_det, cls2_det, ...], ...]. \ + The outer list indicates images, and the inner \ + list indicates per-class detected bboxes. The \ + np.ndarray has shape (num_det, 5) and the last \ + dimension 5 represents (x1, y1, x2, y2, score). + """ + assert self.with_bbox, 'Bbox head must be implemented.' + # Decode initial proposals + num_imgs = len(img_metas) + proposal_list = [proposal_boxes[i] for i in range(num_imgs)] + object_feats = proposal_features + for stage in range(self.num_stages): + rois = bbox2roi(proposal_list) + bbox_results = self._bbox_forward(stage, x, rois, object_feats, + img_metas) + object_feats = bbox_results['object_feats'] + cls_score = bbox_results['cls_score'] + proposal_list = bbox_results['detach_proposal_list'] + + num_classes = self.bbox_head[-1].num_classes + det_bboxes = [] + det_labels = [] + + if self.bbox_head[-1].loss_cls.use_sigmoid: + cls_score = cls_score.sigmoid() + else: + cls_score = cls_score.softmax(-1)[..., :-1] + + for img_id in range(num_imgs): + cls_score_per_img = cls_score[img_id] + scores_per_img, topk_indices = cls_score_per_img.flatten( + 0, 1).topk( + self.test_cfg.max_per_img, sorted=False) + labels_per_img = topk_indices % num_classes + bbox_pred_per_img = proposal_list[img_id][topk_indices // + num_classes] + if rescale: + scale_factor = img_metas[img_id]['scale_factor'] + bbox_pred_per_img /= bbox_pred_per_img.new_tensor(scale_factor) + det_bboxes.append( + torch.cat([bbox_pred_per_img, scores_per_img[:, None]], dim=1)) + det_labels.append(labels_per_img) + + bbox_results = [ + bbox2result(det_bboxes[i], det_labels[i], num_classes) + for i in range(num_imgs) + ] + + return bbox_results + + def aug_test(self, features, proposal_list, img_metas, rescale=False): + raise NotImplementedError('Sparse R-CNN does not support `aug_test`') + + def forward_dummy(self, x, proposal_boxes, proposal_features, img_metas): + """Dummy forward function when do the flops computing.""" + all_stage_bbox_results = [] + proposal_list = [proposal_boxes[i] for i in range(len(proposal_boxes))] + object_feats = proposal_features + if self.with_bbox: + for stage in range(self.num_stages): + rois = bbox2roi(proposal_list) + bbox_results = self._bbox_forward(stage, x, rois, object_feats, + img_metas) + + all_stage_bbox_results.append(bbox_results) + proposal_list = bbox_results['detach_proposal_list'] + object_feats = bbox_results['object_feats'] + return all_stage_bbox_results diff --git a/detection/mmdet/models/roi_heads/standard_roi_head.py b/detection/mmdet/models/roi_heads/standard_roi_head.py new file mode 100644 index 0000000..c530f2a --- /dev/null +++ b/detection/mmdet/models/roi_heads/standard_roi_head.py @@ -0,0 +1,295 @@ +import torch + +from mmdet.core import bbox2result, bbox2roi, build_assigner, build_sampler +from ..builder import HEADS, build_head, build_roi_extractor +from .base_roi_head import BaseRoIHead +from .test_mixins import BBoxTestMixin, MaskTestMixin + + +@HEADS.register_module() +class StandardRoIHead(BaseRoIHead, BBoxTestMixin, MaskTestMixin): + """Simplest base roi head including one bbox head and one mask head.""" + + def init_assigner_sampler(self): + """Initialize assigner and sampler.""" + self.bbox_assigner = None + self.bbox_sampler = None + if self.train_cfg: + self.bbox_assigner = build_assigner(self.train_cfg.assigner) + self.bbox_sampler = build_sampler( + self.train_cfg.sampler, context=self) + + def init_bbox_head(self, bbox_roi_extractor, bbox_head): + """Initialize ``bbox_head``""" + self.bbox_roi_extractor = build_roi_extractor(bbox_roi_extractor) + self.bbox_head = build_head(bbox_head) + + def init_mask_head(self, mask_roi_extractor, mask_head): + """Initialize ``mask_head``""" + if mask_roi_extractor is not None: + self.mask_roi_extractor = build_roi_extractor(mask_roi_extractor) + self.share_roi_extractor = False + else: + self.share_roi_extractor = True + self.mask_roi_extractor = self.bbox_roi_extractor + self.mask_head = build_head(mask_head) + + def init_weights(self, pretrained): + """Initialize the weights in head. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + if self.with_shared_head: + self.shared_head.init_weights(pretrained=pretrained) + if self.with_bbox: + self.bbox_roi_extractor.init_weights() + self.bbox_head.init_weights() + if self.with_mask: + self.mask_head.init_weights() + if not self.share_roi_extractor: + self.mask_roi_extractor.init_weights() + + def forward_dummy(self, x, proposals): + """Dummy forward function.""" + # bbox head + outs = () + rois = bbox2roi([proposals]) + if self.with_bbox: + bbox_results = self._bbox_forward(x, rois) + outs = outs + (bbox_results['cls_score'], + bbox_results['bbox_pred']) + # mask head + if self.with_mask: + mask_rois = rois[:100] + mask_results = self._mask_forward(x, mask_rois) + outs = outs + (mask_results['mask_pred'], ) + return outs + + def forward_train(self, + x, + img_metas, + proposal_list, + gt_bboxes, + gt_labels, + gt_bboxes_ignore=None, + gt_masks=None): + """ + Args: + x (list[Tensor]): list of multi-level img features. + img_metas (list[dict]): list of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmdet/datasets/pipelines/formatting.py:Collect`. + proposals (list[Tensors]): list of region proposals. + gt_bboxes (list[Tensor]): Ground truth bboxes for each image with + shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + gt_labels (list[Tensor]): class indices corresponding to each box + gt_bboxes_ignore (None | list[Tensor]): specify which bounding + boxes can be ignored when computing the loss. + gt_masks (None | Tensor) : true segmentation masks for each box + used if the architecture supports a segmentation task. + + Returns: + dict[str, Tensor]: a dictionary of loss components + """ + # assign gts and sample proposals + if self.with_bbox or self.with_mask: + num_imgs = len(img_metas) + if gt_bboxes_ignore is None: + gt_bboxes_ignore = [None for _ in range(num_imgs)] + sampling_results = [] + for i in range(num_imgs): + assign_result = self.bbox_assigner.assign( + proposal_list[i], gt_bboxes[i], gt_bboxes_ignore[i], + gt_labels[i]) + sampling_result = self.bbox_sampler.sample( + assign_result, + proposal_list[i], + gt_bboxes[i], + gt_labels[i], + feats=[lvl_feat[i][None] for lvl_feat in x]) + sampling_results.append(sampling_result) + + losses = dict() + # bbox head forward and loss + if self.with_bbox: + bbox_results = self._bbox_forward_train(x, sampling_results, + gt_bboxes, gt_labels, + img_metas) + losses.update(bbox_results['loss_bbox']) + + # mask head forward and loss + if self.with_mask: + mask_results = self._mask_forward_train(x, sampling_results, + bbox_results['bbox_feats'], + gt_masks, img_metas) + losses.update(mask_results['loss_mask']) + + return losses + + def _bbox_forward(self, x, rois): + """Box head forward function used in both training and testing.""" + # TODO: a more flexible way to decide which feature maps to use + bbox_feats = self.bbox_roi_extractor( + x[:self.bbox_roi_extractor.num_inputs], rois) + if self.with_shared_head: + bbox_feats = self.shared_head(bbox_feats) + cls_score, bbox_pred = self.bbox_head(bbox_feats) + + bbox_results = dict( + cls_score=cls_score, bbox_pred=bbox_pred, bbox_feats=bbox_feats) + return bbox_results + + def _bbox_forward_train(self, x, sampling_results, gt_bboxes, gt_labels, + img_metas): + """Run forward function and calculate loss for box head in training.""" + rois = bbox2roi([res.bboxes for res in sampling_results]) + bbox_results = self._bbox_forward(x, rois) + + bbox_targets = self.bbox_head.get_targets(sampling_results, gt_bboxes, + gt_labels, self.train_cfg) + loss_bbox = self.bbox_head.loss(bbox_results['cls_score'], + bbox_results['bbox_pred'], rois, + *bbox_targets) + + bbox_results.update(loss_bbox=loss_bbox) + return bbox_results + + def _mask_forward_train(self, x, sampling_results, bbox_feats, gt_masks, + img_metas): + """Run forward function and calculate loss for mask head in + training.""" + if not self.share_roi_extractor: + pos_rois = bbox2roi([res.pos_bboxes for res in sampling_results]) + mask_results = self._mask_forward(x, pos_rois) + else: + pos_inds = [] + device = bbox_feats.device + for res in sampling_results: + pos_inds.append( + torch.ones( + res.pos_bboxes.shape[0], + device=device, + dtype=torch.uint8)) + pos_inds.append( + torch.zeros( + res.neg_bboxes.shape[0], + device=device, + dtype=torch.uint8)) + pos_inds = torch.cat(pos_inds) + + mask_results = self._mask_forward( + x, pos_inds=pos_inds, bbox_feats=bbox_feats) + + mask_targets = self.mask_head.get_targets(sampling_results, gt_masks, + self.train_cfg) + pos_labels = torch.cat([res.pos_gt_labels for res in sampling_results]) + loss_mask = self.mask_head.loss(mask_results['mask_pred'], + mask_targets, pos_labels) + + mask_results.update(loss_mask=loss_mask, mask_targets=mask_targets) + return mask_results + + def _mask_forward(self, x, rois=None, pos_inds=None, bbox_feats=None): + """Mask head forward function used in both training and testing.""" + assert ((rois is not None) ^ + (pos_inds is not None and bbox_feats is not None)) + if rois is not None: + mask_feats = self.mask_roi_extractor( + x[:self.mask_roi_extractor.num_inputs], rois) + if self.with_shared_head: + mask_feats = self.shared_head(mask_feats) + else: + assert bbox_feats is not None + mask_feats = bbox_feats[pos_inds] + + mask_pred = self.mask_head(mask_feats) + mask_results = dict(mask_pred=mask_pred, mask_feats=mask_feats) + return mask_results + + async def async_simple_test(self, + x, + proposal_list, + img_metas, + proposals=None, + rescale=False): + """Async test without augmentation.""" + assert self.with_bbox, 'Bbox head must be implemented.' + + det_bboxes, det_labels = await self.async_test_bboxes( + x, img_metas, proposal_list, self.test_cfg, rescale=rescale) + bbox_results = bbox2result(det_bboxes, det_labels, + self.bbox_head.num_classes) + if not self.with_mask: + return bbox_results + else: + segm_results = await self.async_test_mask( + x, + img_metas, + det_bboxes, + det_labels, + rescale=rescale, + mask_test_cfg=self.test_cfg.get('mask')) + return bbox_results, segm_results + + def simple_test(self, + x, + proposal_list, + img_metas, + proposals=None, + rescale=False): + """Test without augmentation.""" + assert self.with_bbox, 'Bbox head must be implemented.' + + det_bboxes, det_labels = self.simple_test_bboxes( + x, img_metas, proposal_list, self.test_cfg, rescale=rescale) + if torch.onnx.is_in_onnx_export(): + if self.with_mask: + segm_results = self.simple_test_mask( + x, img_metas, det_bboxes, det_labels, rescale=rescale) + return det_bboxes, det_labels, segm_results + else: + return det_bboxes, det_labels + + bbox_results = [ + bbox2result(det_bboxes[i], det_labels[i], + self.bbox_head.num_classes) + for i in range(len(det_bboxes)) + ] + + if not self.with_mask: + return bbox_results + else: + segm_results = self.simple_test_mask( + x, img_metas, det_bboxes, det_labels, rescale=rescale) + return list(zip(bbox_results, segm_results)) + + def aug_test(self, x, proposal_list, img_metas, rescale=False): + """Test with augmentations. + + If rescale is False, then returned bboxes and masks will fit the scale + of imgs[0]. + """ + det_bboxes, det_labels = self.aug_test_bboxes(x, img_metas, + proposal_list, + self.test_cfg) + + if rescale: + _det_bboxes = det_bboxes + else: + _det_bboxes = det_bboxes.clone() + _det_bboxes[:, :4] *= det_bboxes.new_tensor( + img_metas[0][0]['scale_factor']) + bbox_results = bbox2result(_det_bboxes, det_labels, + self.bbox_head.num_classes) + + # det_bboxes always keep the original scale + if self.with_mask: + segm_results = self.aug_test_mask(x, img_metas, det_bboxes, + det_labels) + return [(bbox_results, segm_results)] + else: + return [bbox_results] diff --git a/detection/mmdet/models/roi_heads/test_mixins.py b/detection/mmdet/models/roi_heads/test_mixins.py new file mode 100644 index 0000000..78a092a --- /dev/null +++ b/detection/mmdet/models/roi_heads/test_mixins.py @@ -0,0 +1,348 @@ +import logging +import sys + +import torch + +from mmdet.core import (bbox2roi, bbox_mapping, merge_aug_bboxes, + merge_aug_masks, multiclass_nms) + +logger = logging.getLogger(__name__) + +if sys.version_info >= (3, 7): + from mmdet.utils.contextmanagers import completed + + +class BBoxTestMixin(object): + + if sys.version_info >= (3, 7): + + async def async_test_bboxes(self, + x, + img_metas, + proposals, + rcnn_test_cfg, + rescale=False, + bbox_semaphore=None, + global_lock=None): + """Asynchronized test for box head without augmentation.""" + rois = bbox2roi(proposals) + roi_feats = self.bbox_roi_extractor( + x[:len(self.bbox_roi_extractor.featmap_strides)], rois) + if self.with_shared_head: + roi_feats = self.shared_head(roi_feats) + sleep_interval = rcnn_test_cfg.get('async_sleep_interval', 0.017) + + async with completed( + __name__, 'bbox_head_forward', + sleep_interval=sleep_interval): + cls_score, bbox_pred = self.bbox_head(roi_feats) + + img_shape = img_metas[0]['img_shape'] + scale_factor = img_metas[0]['scale_factor'] + det_bboxes, det_labels = self.bbox_head.get_bboxes( + rois, + cls_score, + bbox_pred, + img_shape, + scale_factor, + rescale=rescale, + cfg=rcnn_test_cfg) + return det_bboxes, det_labels + + def simple_test_bboxes(self, + x, + img_metas, + proposals, + rcnn_test_cfg, + rescale=False): + """Test only det bboxes without augmentation. + + Args: + x (tuple[Tensor]): Feature maps of all scale level. + img_metas (list[dict]): Image meta info. + proposals (Tensor or List[Tensor]): Region proposals. + rcnn_test_cfg (obj:`ConfigDict`): `test_cfg` of R-CNN. + rescale (bool): If True, return boxes in original image space. + Default: False. + + Returns: + tuple[list[Tensor], list[Tensor]]: The first list contains + the boxes of the corresponding image in a batch, each + tensor has the shape (num_boxes, 5) and last dimension + 5 represent (tl_x, tl_y, br_x, br_y, score). Each Tensor + in the second list is the labels with shape (num_boxes, ). + The length of both lists should be equal to batch_size. + """ + # get origin input shape to support onnx dynamic input shape + if torch.onnx.is_in_onnx_export(): + assert len( + img_metas + ) == 1, 'Only support one input image while in exporting to ONNX' + img_shapes = img_metas[0]['img_shape_for_onnx'] + else: + img_shapes = tuple(meta['img_shape'] for meta in img_metas) + scale_factors = tuple(meta['scale_factor'] for meta in img_metas) + + # The length of proposals of different batches may be different. + # In order to form a batch, a padding operation is required. + if isinstance(proposals, list): + # padding to form a batch + max_size = max([proposal.size(0) for proposal in proposals]) + for i, proposal in enumerate(proposals): + supplement = proposal.new_full( + (max_size - proposal.size(0), proposal.size(1)), 0) + proposals[i] = torch.cat((supplement, proposal), dim=0) + rois = torch.stack(proposals, dim=0) + else: + rois = proposals + + batch_index = torch.arange( + rois.size(0), device=rois.device).float().view(-1, 1, 1).expand( + rois.size(0), rois.size(1), 1) + rois = torch.cat([batch_index, rois[..., :4]], dim=-1) + batch_size = rois.shape[0] + num_proposals_per_img = rois.shape[1] + + # Eliminate the batch dimension + rois = rois.view(-1, 5) + bbox_results = self._bbox_forward(x, rois) + cls_score = bbox_results['cls_score'] + bbox_pred = bbox_results['bbox_pred'] + + # Recover the batch dimension + rois = rois.reshape(batch_size, num_proposals_per_img, -1) + cls_score = cls_score.reshape(batch_size, num_proposals_per_img, -1) + + if not torch.onnx.is_in_onnx_export(): + # remove padding + supplement_mask = rois[..., -1] == 0 + cls_score[supplement_mask, :] = 0 + + # bbox_pred would be None in some detector when with_reg is False, + # e.g. Grid R-CNN. + if bbox_pred is not None: + # the bbox prediction of some detectors like SABL is not Tensor + if isinstance(bbox_pred, torch.Tensor): + bbox_pred = bbox_pred.reshape(batch_size, + num_proposals_per_img, -1) + if not torch.onnx.is_in_onnx_export(): + bbox_pred[supplement_mask, :] = 0 + else: + # TODO: Looking forward to a better way + # For SABL + bbox_preds = self.bbox_head.bbox_pred_split( + bbox_pred, num_proposals_per_img) + # apply bbox post-processing to each image individually + det_bboxes = [] + det_labels = [] + for i in range(len(proposals)): + # remove padding + supplement_mask = proposals[i][..., -1] == 0 + for bbox in bbox_preds[i]: + bbox[supplement_mask] = 0 + det_bbox, det_label = self.bbox_head.get_bboxes( + rois[i], + cls_score[i], + bbox_preds[i], + img_shapes[i], + scale_factors[i], + rescale=rescale, + cfg=rcnn_test_cfg) + det_bboxes.append(det_bbox) + det_labels.append(det_label) + return det_bboxes, det_labels + else: + bbox_pred = None + + return self.bbox_head.get_bboxes( + rois, + cls_score, + bbox_pred, + img_shapes, + scale_factors, + rescale=rescale, + cfg=rcnn_test_cfg) + + def aug_test_bboxes(self, feats, img_metas, proposal_list, rcnn_test_cfg): + """Test det bboxes with test time augmentation.""" + aug_bboxes = [] + aug_scores = [] + for x, img_meta in zip(feats, img_metas): + # only one image in the batch + img_shape = img_meta[0]['img_shape'] + scale_factor = img_meta[0]['scale_factor'] + flip = img_meta[0]['flip'] + flip_direction = img_meta[0]['flip_direction'] + # TODO more flexible + proposals = bbox_mapping(proposal_list[0][:, :4], img_shape, + scale_factor, flip, flip_direction) + rois = bbox2roi([proposals]) + bbox_results = self._bbox_forward(x, rois) + bboxes, scores = self.bbox_head.get_bboxes( + rois, + bbox_results['cls_score'], + bbox_results['bbox_pred'], + img_shape, + scale_factor, + rescale=False, + cfg=None) + aug_bboxes.append(bboxes) + aug_scores.append(scores) + # after merging, bboxes will be rescaled to the original image size + merged_bboxes, merged_scores = merge_aug_bboxes( + aug_bboxes, aug_scores, img_metas, rcnn_test_cfg) + det_bboxes, det_labels = multiclass_nms(merged_bboxes, merged_scores, + rcnn_test_cfg.score_thr, + rcnn_test_cfg.nms, + rcnn_test_cfg.max_per_img) + return det_bboxes, det_labels + + +class MaskTestMixin(object): + + if sys.version_info >= (3, 7): + + async def async_test_mask(self, + x, + img_metas, + det_bboxes, + det_labels, + rescale=False, + mask_test_cfg=None): + """Asynchronized test for mask head without augmentation.""" + # image shape of the first image in the batch (only one) + ori_shape = img_metas[0]['ori_shape'] + scale_factor = img_metas[0]['scale_factor'] + if det_bboxes.shape[0] == 0: + segm_result = [[] for _ in range(self.mask_head.num_classes)] + else: + if rescale and not isinstance(scale_factor, + (float, torch.Tensor)): + scale_factor = det_bboxes.new_tensor(scale_factor) + _bboxes = ( + det_bboxes[:, :4] * + scale_factor if rescale else det_bboxes) + mask_rois = bbox2roi([_bboxes]) + mask_feats = self.mask_roi_extractor( + x[:len(self.mask_roi_extractor.featmap_strides)], + mask_rois) + + if self.with_shared_head: + mask_feats = self.shared_head(mask_feats) + if mask_test_cfg and mask_test_cfg.get('async_sleep_interval'): + sleep_interval = mask_test_cfg['async_sleep_interval'] + else: + sleep_interval = 0.035 + async with completed( + __name__, + 'mask_head_forward', + sleep_interval=sleep_interval): + mask_pred = self.mask_head(mask_feats) + segm_result = self.mask_head.get_seg_masks( + mask_pred, _bboxes, det_labels, self.test_cfg, ori_shape, + scale_factor, rescale) + return segm_result + + def simple_test_mask(self, + x, + img_metas, + det_bboxes, + det_labels, + rescale=False): + """Simple test for mask head without augmentation.""" + # image shapes of images in the batch + ori_shapes = tuple(meta['ori_shape'] for meta in img_metas) + scale_factors = tuple(meta['scale_factor'] for meta in img_metas) + + # The length of proposals of different batches may be different. + # In order to form a batch, a padding operation is required. + if isinstance(det_bboxes, list): + # padding to form a batch + max_size = max([bboxes.size(0) for bboxes in det_bboxes]) + for i, (bbox, label) in enumerate(zip(det_bboxes, det_labels)): + supplement_bbox = bbox.new_full( + (max_size - bbox.size(0), bbox.size(1)), 0) + supplement_label = label.new_full((max_size - label.size(0), ), + 0) + det_bboxes[i] = torch.cat((supplement_bbox, bbox), dim=0) + det_labels[i] = torch.cat((supplement_label, label), dim=0) + det_bboxes = torch.stack(det_bboxes, dim=0) + det_labels = torch.stack(det_labels, dim=0) + + batch_size = det_bboxes.size(0) + num_proposals_per_img = det_bboxes.shape[1] + + # if det_bboxes is rescaled to the original image size, we need to + # rescale it back to the testing scale to obtain RoIs. + det_bboxes = det_bboxes[..., :4] + if rescale: + if not isinstance(scale_factors[0], float): + scale_factors = det_bboxes.new_tensor(scale_factors) + det_bboxes = det_bboxes * scale_factors.unsqueeze(1) + + batch_index = torch.arange( + det_bboxes.size(0), device=det_bboxes.device).float().view( + -1, 1, 1).expand(det_bboxes.size(0), det_bboxes.size(1), 1) + mask_rois = torch.cat([batch_index, det_bboxes], dim=-1) + mask_rois = mask_rois.view(-1, 5) + mask_results = self._mask_forward(x, mask_rois) + mask_pred = mask_results['mask_pred'] + + # Recover the batch dimension + mask_preds = mask_pred.reshape(batch_size, num_proposals_per_img, + *mask_pred.shape[1:]) + + # apply mask post-processing to each image individually + segm_results = [] + for i in range(batch_size): + mask_pred = mask_preds[i] + det_bbox = det_bboxes[i] + det_label = det_labels[i] + + # remove padding + supplement_mask = det_bbox[..., -1] != 0 + mask_pred = mask_pred[supplement_mask] + det_bbox = det_bbox[supplement_mask] + det_label = det_label[supplement_mask] + + if det_label.shape[0] == 0: + segm_results.append([[] + for _ in range(self.mask_head.num_classes) + ]) + else: + segm_result = self.mask_head.get_seg_masks( + mask_pred, det_bbox, det_label, self.test_cfg, + ori_shapes[i], scale_factors[i], rescale) + segm_results.append(segm_result) + return segm_results + + def aug_test_mask(self, feats, img_metas, det_bboxes, det_labels): + """Test for mask head with test time augmentation.""" + if det_bboxes.shape[0] == 0: + segm_result = [[] for _ in range(self.mask_head.num_classes)] + else: + aug_masks = [] + for x, img_meta in zip(feats, img_metas): + img_shape = img_meta[0]['img_shape'] + scale_factor = img_meta[0]['scale_factor'] + flip = img_meta[0]['flip'] + flip_direction = img_meta[0]['flip_direction'] + _bboxes = bbox_mapping(det_bboxes[:, :4], img_shape, + scale_factor, flip, flip_direction) + mask_rois = bbox2roi([_bboxes]) + mask_results = self._mask_forward(x, mask_rois) + # convert to numpy array to save memory + aug_masks.append( + mask_results['mask_pred'].sigmoid().cpu().numpy()) + merged_masks = merge_aug_masks(aug_masks, img_metas, self.test_cfg) + + ori_shape = img_metas[0][0]['ori_shape'] + segm_result = self.mask_head.get_seg_masks( + merged_masks, + det_bboxes, + det_labels, + self.test_cfg, + ori_shape, + scale_factor=1.0, + rescale=False) + return segm_result diff --git a/detection/mmdet/models/roi_heads/trident_roi_head.py b/detection/mmdet/models/roi_heads/trident_roi_head.py new file mode 100644 index 0000000..245569e --- /dev/null +++ b/detection/mmdet/models/roi_heads/trident_roi_head.py @@ -0,0 +1,119 @@ +import torch +from mmcv.ops import batched_nms + +from mmdet.core import (bbox2result, bbox2roi, bbox_mapping, merge_aug_bboxes, + multiclass_nms) +from mmdet.models.roi_heads.standard_roi_head import StandardRoIHead +from ..builder import HEADS + + +@HEADS.register_module() +class TridentRoIHead(StandardRoIHead): + """Trident roi head. + + Args: + num_branch (int): Number of branches in TridentNet. + test_branch_idx (int): In inference, all 3 branches will be used + if `test_branch_idx==-1`, otherwise only branch with index + `test_branch_idx` will be used. + """ + + def __init__(self, num_branch, test_branch_idx, **kwargs): + self.num_branch = num_branch + self.test_branch_idx = test_branch_idx + super(TridentRoIHead, self).__init__(**kwargs) + + def merge_trident_bboxes(self, trident_det_bboxes, trident_det_labels): + """Merge bbox predictions of each branch.""" + if trident_det_bboxes.numel() == 0: + det_bboxes = trident_det_bboxes.new_zeros((0, 5)) + det_labels = trident_det_bboxes.new_zeros((0, ), dtype=torch.long) + else: + nms_bboxes = trident_det_bboxes[:, :4] + nms_scores = trident_det_bboxes[:, 4].contiguous() + nms_inds = trident_det_labels + nms_cfg = self.test_cfg['nms'] + det_bboxes, keep = batched_nms(nms_bboxes, nms_scores, nms_inds, + nms_cfg) + det_labels = trident_det_labels[keep] + if self.test_cfg['max_per_img'] > 0: + det_labels = det_labels[:self.test_cfg['max_per_img']] + det_bboxes = det_bboxes[:self.test_cfg['max_per_img']] + + return det_bboxes, det_labels + + def simple_test(self, + x, + proposal_list, + img_metas, + proposals=None, + rescale=False): + """Test without augmentation as follows: + + 1. Compute prediction bbox and label per branch. + 2. Merge predictions of each branch according to scores of + bboxes, i.e., bboxes with higher score are kept to give + top-k prediction. + """ + assert self.with_bbox, 'Bbox head must be implemented.' + det_bboxes_list, det_labels_list = self.simple_test_bboxes( + x, img_metas, proposal_list, self.test_cfg, rescale=rescale) + num_branch = self.num_branch if self.test_branch_idx == -1 else 1 + for _ in range(len(det_bboxes_list)): + if det_bboxes_list[_].shape[0] == 0: + det_bboxes_list[_] = det_bboxes_list[_].new_empty((0, 5)) + det_bboxes, det_labels = [], [] + for i in range(len(img_metas) // num_branch): + det_result = self.merge_trident_bboxes( + torch.cat(det_bboxes_list[i * num_branch:(i + 1) * + num_branch]), + torch.cat(det_labels_list[i * num_branch:(i + 1) * + num_branch])) + det_bboxes.append(det_result[0]) + det_labels.append(det_result[1]) + + bbox_results = [ + bbox2result(det_bboxes[i], det_labels[i], + self.bbox_head.num_classes) + for i in range(len(det_bboxes)) + ] + return bbox_results + + def aug_test_bboxes(self, feats, img_metas, proposal_list, rcnn_test_cfg): + """Test det bboxes with test time augmentation.""" + aug_bboxes = [] + aug_scores = [] + for x, img_meta in zip(feats, img_metas): + # only one image in the batch + img_shape = img_meta[0]['img_shape'] + scale_factor = img_meta[0]['scale_factor'] + flip = img_meta[0]['flip'] + flip_direction = img_meta[0]['flip_direction'] + + trident_bboxes, trident_scores = [], [] + for branch_idx in range(len(proposal_list)): + proposals = bbox_mapping(proposal_list[0][:, :4], img_shape, + scale_factor, flip, flip_direction) + rois = bbox2roi([proposals]) + bbox_results = self._bbox_forward(x, rois) + bboxes, scores = self.bbox_head.get_bboxes( + rois, + bbox_results['cls_score'], + bbox_results['bbox_pred'], + img_shape, + scale_factor, + rescale=False, + cfg=None) + trident_bboxes.append(bboxes) + trident_scores.append(scores) + + aug_bboxes.append(torch.cat(trident_bboxes, 0)) + aug_scores.append(torch.cat(trident_scores, 0)) + # after merging, bboxes will be rescaled to the original image size + merged_bboxes, merged_scores = merge_aug_bboxes( + aug_bboxes, aug_scores, img_metas, rcnn_test_cfg) + det_bboxes, det_labels = multiclass_nms(merged_bboxes, merged_scores, + rcnn_test_cfg.score_thr, + rcnn_test_cfg.nms, + rcnn_test_cfg.max_per_img) + return det_bboxes, det_labels diff --git a/detection/mmdet/models/utils/__init__.py b/detection/mmdet/models/utils/__init__.py new file mode 100644 index 0000000..5165b22 --- /dev/null +++ b/detection/mmdet/models/utils/__init__.py @@ -0,0 +1,16 @@ +from .builder import build_positional_encoding, build_transformer +from .gaussian_target import gaussian_radius, gen_gaussian_target +from .positional_encoding import (LearnedPositionalEncoding, + SinePositionalEncoding) +from .res_layer import ResLayer, SimplifiedBasicBlock +from .transformer import (FFN, DynamicConv, MultiheadAttention, Transformer, + TransformerDecoder, TransformerDecoderLayer, + TransformerEncoder, TransformerEncoderLayer) + +__all__ = [ + 'ResLayer', 'gaussian_radius', 'gen_gaussian_target', 'MultiheadAttention', + 'FFN', 'TransformerEncoderLayer', 'TransformerEncoder', + 'TransformerDecoderLayer', 'TransformerDecoder', 'Transformer', + 'build_transformer', 'build_positional_encoding', 'SinePositionalEncoding', + 'LearnedPositionalEncoding', 'DynamicConv', 'SimplifiedBasicBlock' +] diff --git a/detection/mmdet/models/utils/builder.py b/detection/mmdet/models/utils/builder.py new file mode 100644 index 0000000..f362d1c --- /dev/null +++ b/detection/mmdet/models/utils/builder.py @@ -0,0 +1,14 @@ +from mmcv.utils import Registry, build_from_cfg + +TRANSFORMER = Registry('Transformer') +POSITIONAL_ENCODING = Registry('Position encoding') + + +def build_transformer(cfg, default_args=None): + """Builder for Transformer.""" + return build_from_cfg(cfg, TRANSFORMER, default_args) + + +def build_positional_encoding(cfg, default_args=None): + """Builder for Position Encoding.""" + return build_from_cfg(cfg, POSITIONAL_ENCODING, default_args) diff --git a/detection/mmdet/models/utils/gaussian_target.py b/detection/mmdet/models/utils/gaussian_target.py new file mode 100644 index 0000000..7bb7160 --- /dev/null +++ b/detection/mmdet/models/utils/gaussian_target.py @@ -0,0 +1,185 @@ +from math import sqrt + +import torch + + +def gaussian2D(radius, sigma=1, dtype=torch.float32, device='cpu'): + """Generate 2D gaussian kernel. + + Args: + radius (int): Radius of gaussian kernel. + sigma (int): Sigma of gaussian function. Default: 1. + dtype (torch.dtype): Dtype of gaussian tensor. Default: torch.float32. + device (str): Device of gaussian tensor. Default: 'cpu'. + + Returns: + h (Tensor): Gaussian kernel with a + ``(2 * radius + 1) * (2 * radius + 1)`` shape. + """ + x = torch.arange( + -radius, radius + 1, dtype=dtype, device=device).view(1, -1) + y = torch.arange( + -radius, radius + 1, dtype=dtype, device=device).view(-1, 1) + + h = (-(x * x + y * y) / (2 * sigma * sigma)).exp() + + h[h < torch.finfo(h.dtype).eps * h.max()] = 0 + return h + + +def gen_gaussian_target(heatmap, center, radius, k=1): + """Generate 2D gaussian heatmap. + + Args: + heatmap (Tensor): Input heatmap, the gaussian kernel will cover on + it and maintain the max value. + center (list[int]): Coord of gaussian kernel's center. + radius (int): Radius of gaussian kernel. + k (int): Coefficient of gaussian kernel. Default: 1. + + Returns: + out_heatmap (Tensor): Updated heatmap covered by gaussian kernel. + """ + diameter = 2 * radius + 1 + gaussian_kernel = gaussian2D( + radius, sigma=diameter / 6, dtype=heatmap.dtype, device=heatmap.device) + + x, y = center + + height, width = heatmap.shape[:2] + + left, right = min(x, radius), min(width - x, radius + 1) + top, bottom = min(y, radius), min(height - y, radius + 1) + + masked_heatmap = heatmap[y - top:y + bottom, x - left:x + right] + masked_gaussian = gaussian_kernel[radius - top:radius + bottom, + radius - left:radius + right] + out_heatmap = heatmap + torch.max( + masked_heatmap, + masked_gaussian * k, + out=out_heatmap[y - top:y + bottom, x - left:x + right]) + + return out_heatmap + + +def gaussian_radius(det_size, min_overlap): + r"""Generate 2D gaussian radius. + + This function is modified from the `official github repo + `_. + + Given ``min_overlap``, radius could computed by a quadratic equation + according to Vieta's formulas. + + There are 3 cases for computing gaussian radius, details are following: + + - Explanation of figure: ``lt`` and ``br`` indicates the left-top and + bottom-right corner of ground truth box. ``x`` indicates the + generated corner at the limited position when ``radius=r``. + + - Case1: one corner is inside the gt box and the other is outside. + + .. code:: text + + |< width >| + + lt-+----------+ - + | | | ^ + +--x----------+--+ + | | | | + | | | | height + | | overlap | | + | | | | + | | | | v + +--+---------br--+ - + | | | + +----------+--x + + To ensure IoU of generated box and gt box is larger than ``min_overlap``: + + .. math:: + \cfrac{(w-r)*(h-r)}{w*h+(w+h)r-r^2} \ge {iou} \quad\Rightarrow\quad + {r^2-(w+h)r+\cfrac{1-iou}{1+iou}*w*h} \ge 0 \\ + {a} = 1,\quad{b} = {-(w+h)},\quad{c} = {\cfrac{1-iou}{1+iou}*w*h} + {r} \le \cfrac{-b-\sqrt{b^2-4*a*c}}{2*a} + + - Case2: both two corners are inside the gt box. + + .. code:: text + + |< width >| + + lt-+----------+ - + | | | ^ + +--x-------+ | + | | | | + | |overlap| | height + | | | | + | +-------x--+ + | | | v + +----------+-br - + + To ensure IoU of generated box and gt box is larger than ``min_overlap``: + + .. math:: + \cfrac{(w-2*r)*(h-2*r)}{w*h} \ge {iou} \quad\Rightarrow\quad + {4r^2-2(w+h)r+(1-iou)*w*h} \ge 0 \\ + {a} = 4,\quad {b} = {-2(w+h)},\quad {c} = {(1-iou)*w*h} + {r} \le \cfrac{-b-\sqrt{b^2-4*a*c}}{2*a} + + - Case3: both two corners are outside the gt box. + + .. code:: text + + |< width >| + + x--+----------------+ + | | | + +-lt-------------+ | - + | | | | ^ + | | | | + | | overlap | | height + | | | | + | | | | v + | +------------br--+ - + | | | + +----------------+--x + + To ensure IoU of generated box and gt box is larger than ``min_overlap``: + + .. math:: + \cfrac{w*h}{(w+2*r)*(h+2*r)} \ge {iou} \quad\Rightarrow\quad + {4*iou*r^2+2*iou*(w+h)r+(iou-1)*w*h} \le 0 \\ + {a} = {4*iou},\quad {b} = {2*iou*(w+h)},\quad {c} = {(iou-1)*w*h} \\ + {r} \le \cfrac{-b+\sqrt{b^2-4*a*c}}{2*a} + + Args: + det_size (list[int]): Shape of object. + min_overlap (float): Min IoU with ground truth for boxes generated by + keypoints inside the gaussian kernel. + + Returns: + radius (int): Radius of gaussian kernel. + """ + height, width = det_size + + a1 = 1 + b1 = (height + width) + c1 = width * height * (1 - min_overlap) / (1 + min_overlap) + sq1 = sqrt(b1**2 - 4 * a1 * c1) + r1 = (b1 - sq1) / (2 * a1) + + a2 = 4 + b2 = 2 * (height + width) + c2 = (1 - min_overlap) * width * height + sq2 = sqrt(b2**2 - 4 * a2 * c2) + r2 = (b2 - sq2) / (2 * a2) + + a3 = 4 * min_overlap + b3 = -2 * min_overlap * (height + width) + c3 = (min_overlap - 1) * width * height + sq3 = sqrt(b3**2 - 4 * a3 * c3) + r3 = (b3 + sq3) / (2 * a3) + return min(r1, r2, r3) diff --git a/detection/mmdet/models/utils/positional_encoding.py b/detection/mmdet/models/utils/positional_encoding.py new file mode 100644 index 0000000..9bda2bb --- /dev/null +++ b/detection/mmdet/models/utils/positional_encoding.py @@ -0,0 +1,150 @@ +import math + +import torch +import torch.nn as nn +from mmcv.cnn import uniform_init + +from .builder import POSITIONAL_ENCODING + + +@POSITIONAL_ENCODING.register_module() +class SinePositionalEncoding(nn.Module): + """Position encoding with sine and cosine functions. + + See `End-to-End Object Detection with Transformers + `_ for details. + + Args: + num_feats (int): The feature dimension for each position + along x-axis or y-axis. Note the final returned dimension + for each position is 2 times of this value. + temperature (int, optional): The temperature used for scaling + the position embedding. Default 10000. + normalize (bool, optional): Whether to normalize the position + embedding. Default False. + scale (float, optional): A scale factor that scales the position + embedding. The scale will be used only when `normalize` is True. + Default 2*pi. + eps (float, optional): A value added to the denominator for + numerical stability. Default 1e-6. + """ + + def __init__(self, + num_feats, + temperature=10000, + normalize=False, + scale=2 * math.pi, + eps=1e-6): + super(SinePositionalEncoding, self).__init__() + if normalize: + assert isinstance(scale, (float, int)), 'when normalize is set,' \ + 'scale should be provided and in float or int type, ' \ + f'found {type(scale)}' + self.num_feats = num_feats + self.temperature = temperature + self.normalize = normalize + self.scale = scale + self.eps = eps + + def forward(self, mask): + """Forward function for `SinePositionalEncoding`. + + Args: + mask (Tensor): ByteTensor mask. Non-zero values representing + ignored positions, while zero values means valid positions + for this image. Shape [bs, h, w]. + + Returns: + pos (Tensor): Returned position embedding with shape + [bs, num_feats*2, h, w]. + """ + not_mask = ~mask + y_embed = not_mask.cumsum(1, dtype=torch.float32) + x_embed = not_mask.cumsum(2, dtype=torch.float32) + if self.normalize: + y_embed = y_embed / (y_embed[:, -1:, :] + self.eps) * self.scale + x_embed = x_embed / (x_embed[:, :, -1:] + self.eps) * self.scale + dim_t = torch.arange( + self.num_feats, dtype=torch.float32, device=mask.device) + dim_t = self.temperature**(2 * (dim_t // 2) / self.num_feats) + pos_x = x_embed[:, :, :, None] / dim_t + pos_y = y_embed[:, :, :, None] / dim_t + pos_x = torch.stack( + (pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), + dim=4).flatten(3) + pos_y = torch.stack( + (pos_y[:, :, :, 0::2].sin(), pos_y[:, :, :, 1::2].cos()), + dim=4).flatten(3) + pos = torch.cat((pos_y, pos_x), dim=3).permute(0, 3, 1, 2) + return pos + + def __repr__(self): + """str: a string that describes the module""" + repr_str = self.__class__.__name__ + repr_str += f'(num_feats={self.num_feats}, ' + repr_str += f'temperature={self.temperature}, ' + repr_str += f'normalize={self.normalize}, ' + repr_str += f'scale={self.scale}, ' + repr_str += f'eps={self.eps})' + return repr_str + + +@POSITIONAL_ENCODING.register_module() +class LearnedPositionalEncoding(nn.Module): + """Position embedding with learnable embedding weights. + + Args: + num_feats (int): The feature dimension for each position + along x-axis or y-axis. The final returned dimension for + each position is 2 times of this value. + row_num_embed (int, optional): The dictionary size of row embeddings. + Default 50. + col_num_embed (int, optional): The dictionary size of col embeddings. + Default 50. + """ + + def __init__(self, num_feats, row_num_embed=50, col_num_embed=50): + super(LearnedPositionalEncoding, self).__init__() + self.row_embed = nn.Embedding(row_num_embed, num_feats) + self.col_embed = nn.Embedding(col_num_embed, num_feats) + self.num_feats = num_feats + self.row_num_embed = row_num_embed + self.col_num_embed = col_num_embed + self.init_weights() + + def init_weights(self): + """Initialize the learnable weights.""" + uniform_init(self.row_embed) + uniform_init(self.col_embed) + + def forward(self, mask): + """Forward function for `LearnedPositionalEncoding`. + + Args: + mask (Tensor): ByteTensor mask. Non-zero values representing + ignored positions, while zero values means valid positions + for this image. Shape [bs, h, w]. + + Returns: + pos (Tensor): Returned position embedding with shape + [bs, num_feats*2, h, w]. + """ + h, w = mask.shape[-2:] + x = torch.arange(w, device=mask.device) + y = torch.arange(h, device=mask.device) + x_embed = self.col_embed(x) + y_embed = self.row_embed(y) + pos = torch.cat( + (x_embed.unsqueeze(0).repeat(h, 1, 1), y_embed.unsqueeze(1).repeat( + 1, w, 1)), + dim=-1).permute(2, 0, + 1).unsqueeze(0).repeat(mask.shape[0], 1, 1, 1) + return pos + + def __repr__(self): + """str: a string that describes the module""" + repr_str = self.__class__.__name__ + repr_str += f'(num_feats={self.num_feats}, ' + repr_str += f'row_num_embed={self.row_num_embed}, ' + repr_str += f'col_num_embed={self.col_num_embed})' + return repr_str diff --git a/detection/mmdet/models/utils/res_layer.py b/detection/mmdet/models/utils/res_layer.py new file mode 100644 index 0000000..4a4efd3 --- /dev/null +++ b/detection/mmdet/models/utils/res_layer.py @@ -0,0 +1,187 @@ +from mmcv.cnn import build_conv_layer, build_norm_layer +from torch import nn as nn + + +class ResLayer(nn.Sequential): + """ResLayer to build ResNet style backbone. + + Args: + block (nn.Module): block used to build ResLayer. + inplanes (int): inplanes of block. + planes (int): planes of block. + num_blocks (int): number of blocks. + stride (int): stride of the first block. Default: 1 + avg_down (bool): Use AvgPool instead of stride conv when + downsampling in the bottleneck. Default: False + conv_cfg (dict): dictionary to construct and config conv layer. + Default: None + norm_cfg (dict): dictionary to construct and config norm layer. + Default: dict(type='BN') + downsample_first (bool): Downsample at the first block or last block. + False for Hourglass, True for ResNet. Default: True + """ + + def __init__(self, + block, + inplanes, + planes, + num_blocks, + stride=1, + avg_down=False, + conv_cfg=None, + norm_cfg=dict(type='BN'), + downsample_first=True, + **kwargs): + self.block = block + + downsample = None + if stride != 1 or inplanes != planes * block.expansion: + downsample = [] + conv_stride = stride + if avg_down: + conv_stride = 1 + downsample.append( + nn.AvgPool2d( + kernel_size=stride, + stride=stride, + ceil_mode=True, + count_include_pad=False)) + downsample.extend([ + build_conv_layer( + conv_cfg, + inplanes, + planes * block.expansion, + kernel_size=1, + stride=conv_stride, + bias=False), + build_norm_layer(norm_cfg, planes * block.expansion)[1] + ]) + downsample = nn.Sequential(*downsample) + + layers = [] + if downsample_first: + layers.append( + block( + inplanes=inplanes, + planes=planes, + stride=stride, + downsample=downsample, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + **kwargs)) + inplanes = planes * block.expansion + for _ in range(1, num_blocks): + layers.append( + block( + inplanes=inplanes, + planes=planes, + stride=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + **kwargs)) + + else: # downsample_first=False is for HourglassModule + for _ in range(num_blocks - 1): + layers.append( + block( + inplanes=inplanes, + planes=inplanes, + stride=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + **kwargs)) + layers.append( + block( + inplanes=inplanes, + planes=planes, + stride=stride, + downsample=downsample, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + **kwargs)) + super(ResLayer, self).__init__(*layers) + + +class SimplifiedBasicBlock(nn.Module): + """Simplified version of original basic residual block. This is used in + `SCNet `_. + + - Norm layer is now optional + - Last ReLU in forward function is removed + """ + expansion = 1 + + def __init__(self, + inplanes, + planes, + stride=1, + dilation=1, + downsample=None, + style='pytorch', + with_cp=False, + conv_cfg=None, + norm_cfg=dict(type='BN'), + dcn=None, + plugins=None): + super(SimplifiedBasicBlock, self).__init__() + assert dcn is None, 'Not implemented yet.' + assert plugins is None, 'Not implemented yet.' + assert not with_cp, 'Not implemented yet.' + self.with_norm = norm_cfg is not None + with_bias = True if norm_cfg is None else False + self.conv1 = build_conv_layer( + conv_cfg, + inplanes, + planes, + 3, + stride=stride, + padding=dilation, + dilation=dilation, + bias=with_bias) + if self.with_norm: + self.norm1_name, norm1 = build_norm_layer( + norm_cfg, planes, postfix=1) + self.add_module(self.norm1_name, norm1) + self.conv2 = build_conv_layer( + conv_cfg, planes, planes, 3, padding=1, bias=with_bias) + if self.with_norm: + self.norm2_name, norm2 = build_norm_layer( + norm_cfg, planes, postfix=2) + self.add_module(self.norm2_name, norm2) + + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + self.dilation = dilation + self.with_cp = with_cp + + @property + def norm1(self): + """nn.Module: normalization layer after the first convolution layer""" + return getattr(self, self.norm1_name) if self.with_norm else None + + @property + def norm2(self): + """nn.Module: normalization layer after the second convolution layer""" + return getattr(self, self.norm2_name) if self.with_norm else None + + def forward(self, x): + """Forward function.""" + + identity = x + + out = self.conv1(x) + if self.with_norm: + out = self.norm1(out) + out = self.relu(out) + + out = self.conv2(out) + if self.with_norm: + out = self.norm2(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + + return out diff --git a/detection/mmdet/models/utils/transformer.py b/detection/mmdet/models/utils/transformer.py new file mode 100644 index 0000000..83870ee --- /dev/null +++ b/detection/mmdet/models/utils/transformer.py @@ -0,0 +1,860 @@ +import torch +import torch.nn as nn +from mmcv.cnn import (Linear, build_activation_layer, build_norm_layer, + xavier_init) + +from .builder import TRANSFORMER + + +class MultiheadAttention(nn.Module): + """A warpper for torch.nn.MultiheadAttention. + + This module implements MultiheadAttention with residual connection, + and positional encoding used in DETR is also passed as input. + + Args: + embed_dims (int): The embedding dimension. + num_heads (int): Parallel attention heads. Same as + `nn.MultiheadAttention`. + dropout (float): A Dropout layer on attn_output_weights. Default 0.0. + """ + + def __init__(self, embed_dims, num_heads, dropout=0.0): + super(MultiheadAttention, self).__init__() + assert embed_dims % num_heads == 0, 'embed_dims must be ' \ + f'divisible by num_heads. got {embed_dims} and {num_heads}.' + self.embed_dims = embed_dims + self.num_heads = num_heads + self.dropout = dropout + self.attn = nn.MultiheadAttention(embed_dims, num_heads, dropout) + self.dropout = nn.Dropout(dropout) + + def forward(self, + x, + key=None, + value=None, + residual=None, + query_pos=None, + key_pos=None, + attn_mask=None, + key_padding_mask=None): + """Forward function for `MultiheadAttention`. + + Args: + x (Tensor): The input query with shape [num_query, bs, + embed_dims]. Same in `nn.MultiheadAttention.forward`. + key (Tensor): The key tensor with shape [num_key, bs, + embed_dims]. Same in `nn.MultiheadAttention.forward`. + Default None. If None, the `query` will be used. + value (Tensor): The value tensor with same shape as `key`. + Same in `nn.MultiheadAttention.forward`. Default None. + If None, the `key` will be used. + residual (Tensor): The tensor used for addition, with the + same shape as `x`. Default None. If None, `x` will be used. + query_pos (Tensor): The positional encoding for query, with + the same shape as `x`. Default None. If not None, it will + be added to `x` before forward function. + key_pos (Tensor): The positional encoding for `key`, with the + same shape as `key`. Default None. If not None, it will + be added to `key` before forward function. If None, and + `query_pos` has the same shape as `key`, then `query_pos` + will be used for `key_pos`. + attn_mask (Tensor): ByteTensor mask with shape [num_query, + num_key]. Same in `nn.MultiheadAttention.forward`. + Default None. + key_padding_mask (Tensor): ByteTensor with shape [bs, num_key]. + Same in `nn.MultiheadAttention.forward`. Default None. + + Returns: + Tensor: forwarded results with shape [num_query, bs, embed_dims]. + """ + query = x + if key is None: + key = query + if value is None: + value = key + if residual is None: + residual = x + if key_pos is None: + if query_pos is not None and key is not None: + if query_pos.shape == key.shape: + key_pos = query_pos + if query_pos is not None: + query = query + query_pos + if key_pos is not None: + key = key + key_pos + out = self.attn( + query, + key, + value=value, + attn_mask=attn_mask, + key_padding_mask=key_padding_mask)[0] + + return residual + self.dropout(out) + + def __repr__(self): + """str: a string that describes the module""" + repr_str = self.__class__.__name__ + repr_str += f'(embed_dims={self.embed_dims}, ' + repr_str += f'num_heads={self.num_heads}, ' + repr_str += f'dropout={self.dropout})' + return repr_str + + +class FFN(nn.Module): + """Implements feed-forward networks (FFNs) with residual connection. + + Args: + embed_dims (int): The feature dimension. Same as + `MultiheadAttention`. + feedforward_channels (int): The hidden dimension of FFNs. + num_fcs (int, optional): The number of fully-connected layers in + FFNs. Defaults to 2. + act_cfg (dict, optional): The activation config for FFNs. + dropout (float, optional): Probability of an element to be + zeroed. Default 0.0. + add_residual (bool, optional): Add resudual connection. + Defaults to True. + """ + + def __init__(self, + embed_dims, + feedforward_channels, + num_fcs=2, + act_cfg=dict(type='ReLU', inplace=True), + dropout=0.0, + add_residual=True): + super(FFN, self).__init__() + assert num_fcs >= 2, 'num_fcs should be no less ' \ + f'than 2. got {num_fcs}.' + self.embed_dims = embed_dims + self.feedforward_channels = feedforward_channels + self.num_fcs = num_fcs + self.act_cfg = act_cfg + self.dropout = dropout + self.activate = build_activation_layer(act_cfg) + + layers = nn.ModuleList() + in_channels = embed_dims + for _ in range(num_fcs - 1): + layers.append( + nn.Sequential( + Linear(in_channels, feedforward_channels), self.activate, + nn.Dropout(dropout))) + in_channels = feedforward_channels + layers.append(Linear(feedforward_channels, embed_dims)) + self.layers = nn.Sequential(*layers) + self.dropout = nn.Dropout(dropout) + self.add_residual = add_residual + + def forward(self, x, residual=None): + """Forward function for `FFN`.""" + out = self.layers(x) + if not self.add_residual: + return out + if residual is None: + residual = x + return residual + self.dropout(out) + + def __repr__(self): + """str: a string that describes the module""" + repr_str = self.__class__.__name__ + repr_str += f'(embed_dims={self.embed_dims}, ' + repr_str += f'feedforward_channels={self.feedforward_channels}, ' + repr_str += f'num_fcs={self.num_fcs}, ' + repr_str += f'act_cfg={self.act_cfg}, ' + repr_str += f'dropout={self.dropout}, ' + repr_str += f'add_residual={self.add_residual})' + return repr_str + + +class TransformerEncoderLayer(nn.Module): + """Implements one encoder layer in DETR transformer. + + Args: + embed_dims (int): The feature dimension. Same as `FFN`. + num_heads (int): Parallel attention heads. + feedforward_channels (int): The hidden dimension for FFNs. + dropout (float): Probability of an element to be zeroed. Default 0.0. + order (tuple[str]): The order for encoder layer. Valid examples are + ('selfattn', 'norm', 'ffn', 'norm') and ('norm', 'selfattn', + 'norm', 'ffn'). Default ('selfattn', 'norm', 'ffn', 'norm'). + act_cfg (dict): The activation config for FFNs. Default ReLU. + norm_cfg (dict): Config dict for normalization layer. Default + layer normalization. + num_fcs (int): The number of fully-connected layers for FFNs. + Default 2. + """ + + def __init__(self, + embed_dims, + num_heads, + feedforward_channels, + dropout=0.0, + order=('selfattn', 'norm', 'ffn', 'norm'), + act_cfg=dict(type='ReLU', inplace=True), + norm_cfg=dict(type='LN'), + num_fcs=2): + super(TransformerEncoderLayer, self).__init__() + assert isinstance(order, tuple) and len(order) == 4 + assert set(order) == set(['selfattn', 'norm', 'ffn']) + self.embed_dims = embed_dims + self.num_heads = num_heads + self.feedforward_channels = feedforward_channels + self.dropout = dropout + self.order = order + self.act_cfg = act_cfg + self.norm_cfg = norm_cfg + self.num_fcs = num_fcs + self.pre_norm = order[0] == 'norm' + self.self_attn = MultiheadAttention(embed_dims, num_heads, dropout) + self.ffn = FFN(embed_dims, feedforward_channels, num_fcs, act_cfg, + dropout) + self.norms = nn.ModuleList() + self.norms.append(build_norm_layer(norm_cfg, embed_dims)[1]) + self.norms.append(build_norm_layer(norm_cfg, embed_dims)[1]) + + def forward(self, x, pos=None, attn_mask=None, key_padding_mask=None): + """Forward function for `TransformerEncoderLayer`. + + Args: + x (Tensor): The input query with shape [num_key, bs, + embed_dims]. Same in `MultiheadAttention.forward`. + pos (Tensor): The positional encoding for query. Default None. + Same as `query_pos` in `MultiheadAttention.forward`. + attn_mask (Tensor): ByteTensor mask with shape [num_key, + num_key]. Same in `MultiheadAttention.forward`. Default None. + key_padding_mask (Tensor): ByteTensor with shape [bs, num_key]. + Same in `MultiheadAttention.forward`. Default None. + + Returns: + Tensor: forwarded results with shape [num_key, bs, embed_dims]. + """ + norm_cnt = 0 + inp_residual = x + for layer in self.order: + if layer == 'selfattn': + # self attention + query = key = value = x + x = self.self_attn( + query, + key, + value, + inp_residual if self.pre_norm else None, + query_pos=pos, + key_pos=pos, + attn_mask=attn_mask, + key_padding_mask=key_padding_mask) + inp_residual = x + elif layer == 'norm': + x = self.norms[norm_cnt](x) + norm_cnt += 1 + elif layer == 'ffn': + x = self.ffn(x, inp_residual if self.pre_norm else None) + return x + + def __repr__(self): + """str: a string that describes the module""" + repr_str = self.__class__.__name__ + repr_str += f'(embed_dims={self.embed_dims}, ' + repr_str += f'num_heads={self.num_heads}, ' + repr_str += f'feedforward_channels={self.feedforward_channels}, ' + repr_str += f'dropout={self.dropout}, ' + repr_str += f'order={self.order}, ' + repr_str += f'act_cfg={self.act_cfg}, ' + repr_str += f'norm_cfg={self.norm_cfg}, ' + repr_str += f'num_fcs={self.num_fcs})' + return repr_str + + +class TransformerDecoderLayer(nn.Module): + """Implements one decoder layer in DETR transformer. + + Args: + embed_dims (int): The feature dimension. Same as + `TransformerEncoderLayer`. + num_heads (int): Parallel attention heads. + feedforward_channels (int): Same as `TransformerEncoderLayer`. + dropout (float): Same as `TransformerEncoderLayer`. Default 0.0. + order (tuple[str]): The order for decoder layer. Valid examples are + ('selfattn', 'norm', 'multiheadattn', 'norm', 'ffn', 'norm') and + ('norm', 'selfattn', 'norm', 'multiheadattn', 'norm', 'ffn'). + Default the former. + act_cfg (dict): Same as `TransformerEncoderLayer`. Default ReLU. + norm_cfg (dict): Config dict for normalization layer. Default + layer normalization. + num_fcs (int): The number of fully-connected layers in FFNs. + """ + + def __init__(self, + embed_dims, + num_heads, + feedforward_channels, + dropout=0.0, + order=('selfattn', 'norm', 'multiheadattn', 'norm', 'ffn', + 'norm'), + act_cfg=dict(type='ReLU', inplace=True), + norm_cfg=dict(type='LN'), + num_fcs=2): + super(TransformerDecoderLayer, self).__init__() + assert isinstance(order, tuple) and len(order) == 6 + assert set(order) == set(['selfattn', 'norm', 'multiheadattn', 'ffn']) + self.embed_dims = embed_dims + self.num_heads = num_heads + self.feedforward_channels = feedforward_channels + self.dropout = dropout + self.order = order + self.act_cfg = act_cfg + self.norm_cfg = norm_cfg + self.num_fcs = num_fcs + self.pre_norm = order[0] == 'norm' + self.self_attn = MultiheadAttention(embed_dims, num_heads, dropout) + self.multihead_attn = MultiheadAttention(embed_dims, num_heads, + dropout) + self.ffn = FFN(embed_dims, feedforward_channels, num_fcs, act_cfg, + dropout) + self.norms = nn.ModuleList() + # 3 norm layers in official DETR's TransformerDecoderLayer + for _ in range(3): + self.norms.append(build_norm_layer(norm_cfg, embed_dims)[1]) + + def forward(self, + x, + memory, + memory_pos=None, + query_pos=None, + memory_attn_mask=None, + target_attn_mask=None, + memory_key_padding_mask=None, + target_key_padding_mask=None): + """Forward function for `TransformerDecoderLayer`. + + Args: + x (Tensor): Input query with shape [num_query, bs, embed_dims]. + memory (Tensor): Tensor got from `TransformerEncoder`, with shape + [num_key, bs, embed_dims]. + memory_pos (Tensor): The positional encoding for `memory`. Default + None. Same as `key_pos` in `MultiheadAttention.forward`. + query_pos (Tensor): The positional encoding for `query`. Default + None. Same as `query_pos` in `MultiheadAttention.forward`. + memory_attn_mask (Tensor): ByteTensor mask for `memory`, with + shape [num_key, num_key]. Same as `attn_mask` in + `MultiheadAttention.forward`. Default None. + target_attn_mask (Tensor): ByteTensor mask for `x`, with shape + [num_query, num_query]. Same as `attn_mask` in + `MultiheadAttention.forward`. Default None. + memory_key_padding_mask (Tensor): ByteTensor for `memory`, with + shape [bs, num_key]. Same as `key_padding_mask` in + `MultiheadAttention.forward`. Default None. + target_key_padding_mask (Tensor): ByteTensor for `x`, with shape + [bs, num_query]. Same as `key_padding_mask` in + `MultiheadAttention.forward`. Default None. + + Returns: + Tensor: forwarded results with shape [num_query, bs, embed_dims]. + """ + norm_cnt = 0 + inp_residual = x + for layer in self.order: + if layer == 'selfattn': + query = key = value = x + x = self.self_attn( + query, + key, + value, + inp_residual if self.pre_norm else None, + query_pos, + key_pos=query_pos, + attn_mask=target_attn_mask, + key_padding_mask=target_key_padding_mask) + inp_residual = x + elif layer == 'norm': + x = self.norms[norm_cnt](x) + norm_cnt += 1 + elif layer == 'multiheadattn': + query = x + key = value = memory + x = self.multihead_attn( + query, + key, + value, + inp_residual if self.pre_norm else None, + query_pos, + key_pos=memory_pos, + attn_mask=memory_attn_mask, + key_padding_mask=memory_key_padding_mask) + inp_residual = x + elif layer == 'ffn': + x = self.ffn(x, inp_residual if self.pre_norm else None) + return x + + def __repr__(self): + """str: a string that describes the module""" + repr_str = self.__class__.__name__ + repr_str += f'(embed_dims={self.embed_dims}, ' + repr_str += f'num_heads={self.num_heads}, ' + repr_str += f'feedforward_channels={self.feedforward_channels}, ' + repr_str += f'dropout={self.dropout}, ' + repr_str += f'order={self.order}, ' + repr_str += f'act_cfg={self.act_cfg}, ' + repr_str += f'norm_cfg={self.norm_cfg}, ' + repr_str += f'num_fcs={self.num_fcs})' + return repr_str + + +class TransformerEncoder(nn.Module): + """Implements the encoder in DETR transformer. + + Args: + num_layers (int): The number of `TransformerEncoderLayer`. + embed_dims (int): Same as `TransformerEncoderLayer`. + num_heads (int): Same as `TransformerEncoderLayer`. + feedforward_channels (int): Same as `TransformerEncoderLayer`. + dropout (float): Same as `TransformerEncoderLayer`. Default 0.0. + order (tuple[str]): Same as `TransformerEncoderLayer`. + act_cfg (dict): Same as `TransformerEncoderLayer`. Default ReLU. + norm_cfg (dict): Same as `TransformerEncoderLayer`. Default + layer normalization. + num_fcs (int): Same as `TransformerEncoderLayer`. Default 2. + """ + + def __init__(self, + num_layers, + embed_dims, + num_heads, + feedforward_channels, + dropout=0.0, + order=('selfattn', 'norm', 'ffn', 'norm'), + act_cfg=dict(type='ReLU', inplace=True), + norm_cfg=dict(type='LN'), + num_fcs=2): + super(TransformerEncoder, self).__init__() + assert isinstance(order, tuple) and len(order) == 4 + assert set(order) == set(['selfattn', 'norm', 'ffn']) + self.num_layers = num_layers + self.embed_dims = embed_dims + self.num_heads = num_heads + self.feedforward_channels = feedforward_channels + self.dropout = dropout + self.order = order + self.act_cfg = act_cfg + self.norm_cfg = norm_cfg + self.num_fcs = num_fcs + self.pre_norm = order[0] == 'norm' + self.layers = nn.ModuleList() + for _ in range(num_layers): + self.layers.append( + TransformerEncoderLayer(embed_dims, num_heads, + feedforward_channels, dropout, order, + act_cfg, norm_cfg, num_fcs)) + self.norm = build_norm_layer(norm_cfg, + embed_dims)[1] if self.pre_norm else None + + def forward(self, x, pos=None, attn_mask=None, key_padding_mask=None): + """Forward function for `TransformerEncoder`. + + Args: + x (Tensor): Input query. Same in `TransformerEncoderLayer.forward`. + pos (Tensor): Positional encoding for query. Default None. + Same in `TransformerEncoderLayer.forward`. + attn_mask (Tensor): ByteTensor attention mask. Default None. + Same in `TransformerEncoderLayer.forward`. + key_padding_mask (Tensor): Same in + `TransformerEncoderLayer.forward`. Default None. + + Returns: + Tensor: Results with shape [num_key, bs, embed_dims]. + """ + for layer in self.layers: + x = layer(x, pos, attn_mask, key_padding_mask) + if self.norm is not None: + x = self.norm(x) + return x + + def __repr__(self): + """str: a string that describes the module""" + repr_str = self.__class__.__name__ + repr_str += f'(num_layers={self.num_layers}, ' + repr_str += f'embed_dims={self.embed_dims}, ' + repr_str += f'num_heads={self.num_heads}, ' + repr_str += f'feedforward_channels={self.feedforward_channels}, ' + repr_str += f'dropout={self.dropout}, ' + repr_str += f'order={self.order}, ' + repr_str += f'act_cfg={self.act_cfg}, ' + repr_str += f'norm_cfg={self.norm_cfg}, ' + repr_str += f'num_fcs={self.num_fcs})' + return repr_str + + +class TransformerDecoder(nn.Module): + """Implements the decoder in DETR transformer. + + Args: + num_layers (int): The number of `TransformerDecoderLayer`. + embed_dims (int): Same as `TransformerDecoderLayer`. + num_heads (int): Same as `TransformerDecoderLayer`. + feedforward_channels (int): Same as `TransformerDecoderLayer`. + dropout (float): Same as `TransformerDecoderLayer`. Default 0.0. + order (tuple[str]): Same as `TransformerDecoderLayer`. + act_cfg (dict): Same as `TransformerDecoderLayer`. Default ReLU. + norm_cfg (dict): Same as `TransformerDecoderLayer`. Default + layer normalization. + num_fcs (int): Same as `TransformerDecoderLayer`. Default 2. + """ + + def __init__(self, + num_layers, + embed_dims, + num_heads, + feedforward_channels, + dropout=0.0, + order=('selfattn', 'norm', 'multiheadattn', 'norm', 'ffn', + 'norm'), + act_cfg=dict(type='ReLU', inplace=True), + norm_cfg=dict(type='LN'), + num_fcs=2, + return_intermediate=False): + super(TransformerDecoder, self).__init__() + assert isinstance(order, tuple) and len(order) == 6 + assert set(order) == set(['selfattn', 'norm', 'multiheadattn', 'ffn']) + self.num_layers = num_layers + self.embed_dims = embed_dims + self.num_heads = num_heads + self.feedforward_channels = feedforward_channels + self.dropout = dropout + self.order = order + self.act_cfg = act_cfg + self.norm_cfg = norm_cfg + self.num_fcs = num_fcs + self.return_intermediate = return_intermediate + self.layers = nn.ModuleList() + for _ in range(num_layers): + self.layers.append( + TransformerDecoderLayer(embed_dims, num_heads, + feedforward_channels, dropout, order, + act_cfg, norm_cfg, num_fcs)) + self.norm = build_norm_layer(norm_cfg, embed_dims)[1] + + def forward(self, + x, + memory, + memory_pos=None, + query_pos=None, + memory_attn_mask=None, + target_attn_mask=None, + memory_key_padding_mask=None, + target_key_padding_mask=None): + """Forward function for `TransformerDecoder`. + + Args: + x (Tensor): Input query. Same in `TransformerDecoderLayer.forward`. + memory (Tensor): Same in `TransformerDecoderLayer.forward`. + memory_pos (Tensor): Same in `TransformerDecoderLayer.forward`. + Default None. + query_pos (Tensor): Same in `TransformerDecoderLayer.forward`. + Default None. + memory_attn_mask (Tensor): Same in + `TransformerDecoderLayer.forward`. Default None. + target_attn_mask (Tensor): Same in + `TransformerDecoderLayer.forward`. Default None. + memory_key_padding_mask (Tensor): Same in + `TransformerDecoderLayer.forward`. Default None. + target_key_padding_mask (Tensor): Same in + `TransformerDecoderLayer.forward`. Default None. + + Returns: + Tensor: Results with shape [num_query, bs, embed_dims]. + """ + intermediate = [] + for layer in self.layers: + x = layer(x, memory, memory_pos, query_pos, memory_attn_mask, + target_attn_mask, memory_key_padding_mask, + target_key_padding_mask) + if self.return_intermediate: + intermediate.append(self.norm(x)) + if self.norm is not None: + x = self.norm(x) + if self.return_intermediate: + intermediate.pop() + intermediate.append(x) + if self.return_intermediate: + return torch.stack(intermediate) + return x.unsqueeze(0) + + def __repr__(self): + """str: a string that describes the module""" + repr_str = self.__class__.__name__ + repr_str += f'(num_layers={self.num_layers}, ' + repr_str += f'embed_dims={self.embed_dims}, ' + repr_str += f'num_heads={self.num_heads}, ' + repr_str += f'feedforward_channels={self.feedforward_channels}, ' + repr_str += f'dropout={self.dropout}, ' + repr_str += f'order={self.order}, ' + repr_str += f'act_cfg={self.act_cfg}, ' + repr_str += f'norm_cfg={self.norm_cfg}, ' + repr_str += f'num_fcs={self.num_fcs}, ' + repr_str += f'return_intermediate={self.return_intermediate})' + return repr_str + + +@TRANSFORMER.register_module() +class Transformer(nn.Module): + """Implements the DETR transformer. + + Following the official DETR implementation, this module copy-paste + from torch.nn.Transformer with modifications: + + * positional encodings are passed in MultiheadAttention + * extra LN at the end of encoder is removed + * decoder returns a stack of activations from all decoding layers + + See `paper: End-to-End Object Detection with Transformers + `_ for details. + + Args: + embed_dims (int): The feature dimension. + num_heads (int): Parallel attention heads. Same as + `nn.MultiheadAttention`. + num_encoder_layers (int): Number of `TransformerEncoderLayer`. + num_decoder_layers (int): Number of `TransformerDecoderLayer`. + feedforward_channels (int): The hidden dimension for FFNs used in both + encoder and decoder. + dropout (float): Probability of an element to be zeroed. Default 0.0. + act_cfg (dict): Activation config for FFNs used in both encoder + and decoder. Default ReLU. + norm_cfg (dict): Config dict for normalization used in both encoder + and decoder. Default layer normalization. + num_fcs (int): The number of fully-connected layers in FFNs, which is + used for both encoder and decoder. + pre_norm (bool): Whether the normalization layer is ordered + first in the encoder and decoder. Default False. + return_intermediate_dec (bool): Whether to return the intermediate + output from each TransformerDecoderLayer or only the last + TransformerDecoderLayer. Default False. If False, the returned + `hs` has shape [num_decoder_layers, bs, num_query, embed_dims]. + If True, the returned `hs` will have shape [1, bs, num_query, + embed_dims]. + """ + + def __init__(self, + embed_dims=512, + num_heads=8, + num_encoder_layers=6, + num_decoder_layers=6, + feedforward_channels=2048, + dropout=0.0, + act_cfg=dict(type='ReLU', inplace=True), + norm_cfg=dict(type='LN'), + num_fcs=2, + pre_norm=False, + return_intermediate_dec=False): + super(Transformer, self).__init__() + self.embed_dims = embed_dims + self.num_heads = num_heads + self.num_encoder_layers = num_encoder_layers + self.num_decoder_layers = num_decoder_layers + self.feedforward_channels = feedforward_channels + self.dropout = dropout + self.act_cfg = act_cfg + self.norm_cfg = norm_cfg + self.num_fcs = num_fcs + self.pre_norm = pre_norm + self.return_intermediate_dec = return_intermediate_dec + if self.pre_norm: + encoder_order = ('norm', 'selfattn', 'norm', 'ffn') + decoder_order = ('norm', 'selfattn', 'norm', 'multiheadattn', + 'norm', 'ffn') + else: + encoder_order = ('selfattn', 'norm', 'ffn', 'norm') + decoder_order = ('selfattn', 'norm', 'multiheadattn', 'norm', + 'ffn', 'norm') + self.encoder = TransformerEncoder(num_encoder_layers, embed_dims, + num_heads, feedforward_channels, + dropout, encoder_order, act_cfg, + norm_cfg, num_fcs) + self.decoder = TransformerDecoder(num_decoder_layers, embed_dims, + num_heads, feedforward_channels, + dropout, decoder_order, act_cfg, + norm_cfg, num_fcs, + return_intermediate_dec) + + def init_weights(self, distribution='uniform'): + """Initialize the transformer weights.""" + # follow the official DETR to init parameters + for m in self.modules(): + if hasattr(m, 'weight') and m.weight.dim() > 1: + xavier_init(m, distribution=distribution) + + def forward(self, x, mask, query_embed, pos_embed): + """Forward function for `Transformer`. + + Args: + x (Tensor): Input query with shape [bs, c, h, w] where + c = embed_dims. + mask (Tensor): The key_padding_mask used for encoder and decoder, + with shape [bs, h, w]. + query_embed (Tensor): The query embedding for decoder, with shape + [num_query, c]. + pos_embed (Tensor): The positional encoding for encoder and + decoder, with the same shape as `x`. + + Returns: + tuple[Tensor]: results of decoder containing the following tensor. + + - out_dec: Output from decoder. If return_intermediate_dec \ + is True output has shape [num_dec_layers, bs, + num_query, embed_dims], else has shape [1, bs, \ + num_query, embed_dims]. + - memory: Output results from encoder, with shape \ + [bs, embed_dims, h, w]. + """ + bs, c, h, w = x.shape + x = x.flatten(2).permute(2, 0, 1) # [bs, c, h, w] -> [h*w, bs, c] + pos_embed = pos_embed.flatten(2).permute(2, 0, 1) + query_embed = query_embed.unsqueeze(1).repeat( + 1, bs, 1) # [num_query, dim] -> [num_query, bs, dim] + mask = mask.flatten(1) # [bs, h, w] -> [bs, h*w] + memory = self.encoder( + x, pos=pos_embed, attn_mask=None, key_padding_mask=mask) + target = torch.zeros_like(query_embed) + # out_dec: [num_layers, num_query, bs, dim] + out_dec = self.decoder( + target, + memory, + memory_pos=pos_embed, + query_pos=query_embed, + memory_attn_mask=None, + target_attn_mask=None, + memory_key_padding_mask=mask, + target_key_padding_mask=None) + out_dec = out_dec.transpose(1, 2) + memory = memory.permute(1, 2, 0).reshape(bs, c, h, w) + return out_dec, memory + + def __repr__(self): + """str: a string that describes the module""" + repr_str = self.__class__.__name__ + repr_str += f'(embed_dims={self.embed_dims}, ' + repr_str += f'num_heads={self.num_heads}, ' + repr_str += f'num_encoder_layers={self.num_encoder_layers}, ' + repr_str += f'num_decoder_layers={self.num_decoder_layers}, ' + repr_str += f'feedforward_channels={self.feedforward_channels}, ' + repr_str += f'dropout={self.dropout}, ' + repr_str += f'act_cfg={self.act_cfg}, ' + repr_str += f'norm_cfg={self.norm_cfg}, ' + repr_str += f'num_fcs={self.num_fcs}, ' + repr_str += f'pre_norm={self.pre_norm}, ' + repr_str += f'return_intermediate_dec={self.return_intermediate_dec})' + return repr_str + + +@TRANSFORMER.register_module() +class DynamicConv(nn.Module): + """Implements Dynamic Convolution. + + This module generate parameters for each sample and + use bmm to implement 1*1 convolution. Code is modified + from the `official github repo `_ . + + Args: + in_channels (int): The input feature channel. + Defaults to 256. + feat_channels (int): The inner feature channel. + Defaults to 64. + out_channels (int, optional): The output feature channel. + When not specified, it will be set to `in_channels` + by default + input_feat_shape (int): The shape of input feature. + Defaults to 7. + act_cfg (dict): The activation config for DynamicConv. + norm_cfg (dict): Config dict for normalization layer. Default + layer normalization. + """ + + def __init__(self, + in_channels=256, + feat_channels=64, + out_channels=None, + input_feat_shape=7, + act_cfg=dict(type='ReLU', inplace=True), + norm_cfg=dict(type='LN')): + super(DynamicConv, self).__init__() + self.in_channels = in_channels + self.feat_channels = feat_channels + self.out_channels_raw = out_channels + self.input_feat_shape = input_feat_shape + self.act_cfg = act_cfg + self.norm_cfg = norm_cfg + self.out_channels = out_channels if out_channels else in_channels + + self.num_params_in = self.in_channels * self.feat_channels + self.num_params_out = self.out_channels * self.feat_channels + self.dynamic_layer = nn.Linear( + self.in_channels, self.num_params_in + self.num_params_out) + + self.norm_in = build_norm_layer(norm_cfg, self.feat_channels)[1] + self.norm_out = build_norm_layer(norm_cfg, self.out_channels)[1] + + self.activation = build_activation_layer(act_cfg) + + num_output = self.out_channels * input_feat_shape**2 + self.fc_layer = nn.Linear(num_output, self.out_channels) + self.fc_norm = build_norm_layer(norm_cfg, self.out_channels)[1] + + def forward(self, param_feature, input_feature): + """Forward function for `DynamicConv`. + + Args: + param_feature (Tensor): The feature can be used + to generate the parameter, has shape + (num_all_proposals, in_channels). + input_feature (Tensor): Feature that + interact with parameters, has shape + (num_all_proposals, in_channels, H, W). + + Returns: + Tensor: The output feature has shape + (num_all_proposals, out_channels). + """ + num_proposals = param_feature.size(0) + input_feature = input_feature.view(num_proposals, self.in_channels, + -1).permute(2, 0, 1) + + input_feature = input_feature.permute(1, 0, 2) + parameters = self.dynamic_layer(param_feature) + + param_in = parameters[:, :self.num_params_in].view( + -1, self.in_channels, self.feat_channels) + param_out = parameters[:, -self.num_params_out:].view( + -1, self.feat_channels, self.out_channels) + + # input_feature has shape (num_all_proposals, H*W, in_channels) + # param_in has shape (num_all_proposals, in_channels, feat_channels) + # feature has shape (num_all_proposals, H*W, feat_channels) + features = torch.bmm(input_feature, param_in) + features = self.norm_in(features) + features = self.activation(features) + + # param_out has shape (batch_size, feat_channels, out_channels) + features = torch.bmm(features, param_out) + features = self.norm_out(features) + features = self.activation(features) + + features = features.flatten(1) + features = self.fc_layer(features) + features = self.fc_norm(features) + features = self.activation(features) + + return features + + def __repr__(self): + """str: a string that describes the module""" + repr_str = self.__class__.__name__ + repr_str += f'(in_channels={self.in_channels}, ' + repr_str += f'feat_channels={self.feat_channels}, ' + repr_str += f'out_channels={self.out_channels_raw}, ' + repr_str += f'input_feat_shape={self.input_feat_shape}, ' + repr_str += f'act_cfg={self.act_cfg}, ' + repr_str += f'norm_cfg={self.norm_cfg})' + return repr_str diff --git a/detection/mmdet/utils/__init__.py b/detection/mmdet/utils/__init__.py new file mode 100644 index 0000000..e79ad8c --- /dev/null +++ b/detection/mmdet/utils/__init__.py @@ -0,0 +1,5 @@ +from .collect_env import collect_env +from .logger import get_root_logger +from .optimizer import DistOptimizerHook + +__all__ = ['get_root_logger', 'collect_env', 'DistOptimizerHook'] diff --git a/detection/mmdet/utils/collect_env.py b/detection/mmdet/utils/collect_env.py new file mode 100644 index 0000000..89c064a --- /dev/null +++ b/detection/mmdet/utils/collect_env.py @@ -0,0 +1,16 @@ +from mmcv.utils import collect_env as collect_base_env +from mmcv.utils import get_git_hash + +import mmdet + + +def collect_env(): + """Collect the information of the running environments.""" + env_info = collect_base_env() + env_info['MMDetection'] = mmdet.__version__ + '+' + get_git_hash()[:7] + return env_info + + +if __name__ == '__main__': + for name, val in collect_env().items(): + print(f'{name}: {val}') diff --git a/detection/mmdet/utils/contextmanagers.py b/detection/mmdet/utils/contextmanagers.py new file mode 100644 index 0000000..38a6392 --- /dev/null +++ b/detection/mmdet/utils/contextmanagers.py @@ -0,0 +1,121 @@ +import asyncio +import contextlib +import logging +import os +import time +from typing import List + +import torch + +logger = logging.getLogger(__name__) + +DEBUG_COMPLETED_TIME = bool(os.environ.get('DEBUG_COMPLETED_TIME', False)) + + +@contextlib.asynccontextmanager +async def completed(trace_name='', + name='', + sleep_interval=0.05, + streams: List[torch.cuda.Stream] = None): + """Async context manager that waits for work to complete on given CUDA + streams.""" + if not torch.cuda.is_available(): + yield + return + + stream_before_context_switch = torch.cuda.current_stream() + if not streams: + streams = [stream_before_context_switch] + else: + streams = [s if s else stream_before_context_switch for s in streams] + + end_events = [ + torch.cuda.Event(enable_timing=DEBUG_COMPLETED_TIME) for _ in streams + ] + + if DEBUG_COMPLETED_TIME: + start = torch.cuda.Event(enable_timing=True) + stream_before_context_switch.record_event(start) + + cpu_start = time.monotonic() + logger.debug('%s %s starting, streams: %s', trace_name, name, streams) + grad_enabled_before = torch.is_grad_enabled() + try: + yield + finally: + current_stream = torch.cuda.current_stream() + assert current_stream == stream_before_context_switch + + if DEBUG_COMPLETED_TIME: + cpu_end = time.monotonic() + for i, stream in enumerate(streams): + event = end_events[i] + stream.record_event(event) + + grad_enabled_after = torch.is_grad_enabled() + + # observed change of torch.is_grad_enabled() during concurrent run of + # async_test_bboxes code + assert (grad_enabled_before == grad_enabled_after + ), 'Unexpected is_grad_enabled() value change' + + are_done = [e.query() for e in end_events] + logger.debug('%s %s completed: %s streams: %s', trace_name, name, + are_done, streams) + with torch.cuda.stream(stream_before_context_switch): + while not all(are_done): + await asyncio.sleep(sleep_interval) + are_done = [e.query() for e in end_events] + logger.debug( + '%s %s completed: %s streams: %s', + trace_name, + name, + are_done, + streams, + ) + + current_stream = torch.cuda.current_stream() + assert current_stream == stream_before_context_switch + + if DEBUG_COMPLETED_TIME: + cpu_time = (cpu_end - cpu_start) * 1000 + stream_times_ms = '' + for i, stream in enumerate(streams): + elapsed_time = start.elapsed_time(end_events[i]) + stream_times_ms += f' {stream} {elapsed_time:.2f} ms' + logger.info('%s %s %.2f ms %s', trace_name, name, cpu_time, + stream_times_ms) + + +@contextlib.asynccontextmanager +async def concurrent(streamqueue: asyncio.Queue, + trace_name='concurrent', + name='stream'): + """Run code concurrently in different streams. + + :param streamqueue: asyncio.Queue instance. + + Queue tasks define the pool of streams used for concurrent execution. + """ + if not torch.cuda.is_available(): + yield + return + + initial_stream = torch.cuda.current_stream() + + with torch.cuda.stream(initial_stream): + stream = await streamqueue.get() + assert isinstance(stream, torch.cuda.Stream) + + try: + with torch.cuda.stream(stream): + logger.debug('%s %s is starting, stream: %s', trace_name, name, + stream) + yield + current = torch.cuda.current_stream() + assert current == stream + logger.debug('%s %s has finished, stream: %s', trace_name, + name, stream) + finally: + streamqueue.task_done() + streamqueue.put_nowait(stream) diff --git a/detection/mmdet/utils/logger.py b/detection/mmdet/utils/logger.py new file mode 100644 index 0000000..6fc6e6b --- /dev/null +++ b/detection/mmdet/utils/logger.py @@ -0,0 +1,19 @@ +import logging + +from mmcv.utils import get_logger + + +def get_root_logger(log_file=None, log_level=logging.INFO): + """Get root logger. + + Args: + log_file (str, optional): File path of log. Defaults to None. + log_level (int, optional): The level of logger. + Defaults to logging.INFO. + + Returns: + :obj:`logging.Logger`: The obtained logger + """ + logger = get_logger(name='mmdet', log_file=log_file, log_level=log_level) + + return logger diff --git a/detection/mmdet/utils/optimizer.py b/detection/mmdet/utils/optimizer.py new file mode 100644 index 0000000..9c9d119 --- /dev/null +++ b/detection/mmdet/utils/optimizer.py @@ -0,0 +1,33 @@ +from mmcv.runner import OptimizerHook, HOOKS +try: + import apex +except: + print('apex is not installed') + + +@HOOKS.register_module() +class DistOptimizerHook(OptimizerHook): + """Optimizer hook for distributed training.""" + + def __init__(self, update_interval=1, grad_clip=None, coalesce=True, bucket_size_mb=-1, use_fp16=False): + self.grad_clip = grad_clip + self.coalesce = coalesce + self.bucket_size_mb = bucket_size_mb + self.update_interval = update_interval + self.use_fp16 = use_fp16 + + def before_run(self, runner): + runner.optimizer.zero_grad() + + def after_train_iter(self, runner): + runner.outputs['loss'] /= self.update_interval + if self.use_fp16: + with apex.amp.scale_loss(runner.outputs['loss'], runner.optimizer) as scaled_loss: + scaled_loss.backward() + else: + runner.outputs['loss'].backward() + if self.every_n_iters(runner, self.update_interval): + if self.grad_clip is not None: + self.clip_grads(runner.model.parameters()) + runner.optimizer.step() + runner.optimizer.zero_grad() diff --git a/detection/mmdet/utils/profiling.py b/detection/mmdet/utils/profiling.py new file mode 100644 index 0000000..4be9222 --- /dev/null +++ b/detection/mmdet/utils/profiling.py @@ -0,0 +1,39 @@ +import contextlib +import sys +import time + +import torch + +if sys.version_info >= (3, 7): + + @contextlib.contextmanager + def profile_time(trace_name, + name, + enabled=True, + stream=None, + end_stream=None): + """Print time spent by CPU and GPU. + + Useful as a temporary context manager to find sweet spots of code + suitable for async implementation. + """ + if (not enabled) or not torch.cuda.is_available(): + yield + return + stream = stream if stream else torch.cuda.current_stream() + end_stream = end_stream if end_stream else stream + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + stream.record_event(start) + try: + cpu_start = time.monotonic() + yield + finally: + cpu_end = time.monotonic() + end_stream.record_event(end) + end.synchronize() + cpu_time = (cpu_end - cpu_start) * 1000 + gpu_time = start.elapsed_time(end) + msg = f'{trace_name} {name} cpu_time {cpu_time:.2f} ms ' + msg += f'gpu_time {gpu_time:.2f} ms stream {stream}' + print(msg, end_stream) diff --git a/detection/mmdet/utils/util_mixins.py b/detection/mmdet/utils/util_mixins.py new file mode 100644 index 0000000..69669a3 --- /dev/null +++ b/detection/mmdet/utils/util_mixins.py @@ -0,0 +1,104 @@ +"""This module defines the :class:`NiceRepr` mixin class, which defines a +``__repr__`` and ``__str__`` method that only depend on a custom ``__nice__`` +method, which you must define. This means you only have to overload one +function instead of two. Furthermore, if the object defines a ``__len__`` +method, then the ``__nice__`` method defaults to something sensible, otherwise +it is treated as abstract and raises ``NotImplementedError``. + +To use simply have your object inherit from :class:`NiceRepr` +(multi-inheritance should be ok). + +This code was copied from the ubelt library: https://github.com/Erotemic/ubelt + +Example: + >>> # Objects that define __nice__ have a default __str__ and __repr__ + >>> class Student(NiceRepr): + ... def __init__(self, name): + ... self.name = name + ... def __nice__(self): + ... return self.name + >>> s1 = Student('Alice') + >>> s2 = Student('Bob') + >>> print(f's1 = {s1}') + >>> print(f's2 = {s2}') + s1 = + s2 = + +Example: + >>> # Objects that define __len__ have a default __nice__ + >>> class Group(NiceRepr): + ... def __init__(self, data): + ... self.data = data + ... def __len__(self): + ... return len(self.data) + >>> g = Group([1, 2, 3]) + >>> print(f'g = {g}') + g = +""" +import warnings + + +class NiceRepr(object): + """Inherit from this class and define ``__nice__`` to "nicely" print your + objects. + + Defines ``__str__`` and ``__repr__`` in terms of ``__nice__`` function + Classes that inherit from :class:`NiceRepr` should redefine ``__nice__``. + If the inheriting class has a ``__len__``, method then the default + ``__nice__`` method will return its length. + + Example: + >>> class Foo(NiceRepr): + ... def __nice__(self): + ... return 'info' + >>> foo = Foo() + >>> assert str(foo) == '' + >>> assert repr(foo).startswith('>> class Bar(NiceRepr): + ... pass + >>> bar = Bar() + >>> import pytest + >>> with pytest.warns(None) as record: + >>> assert 'object at' in str(bar) + >>> assert 'object at' in repr(bar) + + Example: + >>> class Baz(NiceRepr): + ... def __len__(self): + ... return 5 + >>> baz = Baz() + >>> assert str(baz) == '' + """ + + def __nice__(self): + """str: a "nice" summary string describing this module""" + if hasattr(self, '__len__'): + # It is a common pattern for objects to use __len__ in __nice__ + # As a convenience we define a default __nice__ for these objects + return str(len(self)) + else: + # In all other cases force the subclass to overload __nice__ + raise NotImplementedError( + f'Define the __nice__ method for {self.__class__!r}') + + def __repr__(self): + """str: the string of the module""" + try: + nice = self.__nice__() + classname = self.__class__.__name__ + return f'<{classname}({nice}) at {hex(id(self))}>' + except NotImplementedError as ex: + warnings.warn(str(ex), category=RuntimeWarning) + return object.__repr__(self) + + def __str__(self): + """str: the string of the module""" + try: + classname = self.__class__.__name__ + nice = self.__nice__() + return f'<{classname}({nice})>' + except NotImplementedError as ex: + warnings.warn(str(ex), category=RuntimeWarning) + return object.__repr__(self) diff --git a/detection/mmdet/utils/util_random.py b/detection/mmdet/utils/util_random.py new file mode 100644 index 0000000..e313e99 --- /dev/null +++ b/detection/mmdet/utils/util_random.py @@ -0,0 +1,33 @@ +"""Helpers for random number generators.""" +import numpy as np + + +def ensure_rng(rng=None): + """Coerces input into a random number generator. + + If the input is None, then a global random state is returned. + + If the input is a numeric value, then that is used as a seed to construct a + random state. Otherwise the input is returned as-is. + + Adapted from [1]_. + + Args: + rng (int | numpy.random.RandomState | None): + if None, then defaults to the global rng. Otherwise this can be an + integer or a RandomState class + Returns: + (numpy.random.RandomState) : rng - + a numpy random number generator + + References: + .. [1] https://gitlab.kitware.com/computer-vision/kwarray/blob/master/kwarray/util_random.py#L270 # noqa: E501 + """ + + if rng is None: + rng = np.random.mtrand._rand + elif isinstance(rng, int): + rng = np.random.RandomState(rng) + else: + rng = rng + return rng diff --git a/detection/mmdet/version.py b/detection/mmdet/version.py new file mode 100644 index 0000000..a3b741a --- /dev/null +++ b/detection/mmdet/version.py @@ -0,0 +1,19 @@ +# Copyright (c) Open-MMLab. All rights reserved. + +__version__ = '2.11.0' +short_version = __version__ + + +def parse_version_info(version_str): + version_info = [] + for x in version_str.split('.'): + if x.isdigit(): + version_info.append(int(x)) + elif x.find('rc') != -1: + patch_version = x.split('rc') + version_info.append(int(patch_version[0])) + version_info.append(f'rc{patch_version[1]}') + return tuple(version_info) + + +version_info = parse_version_info(__version__) diff --git a/detection/requirements.txt b/detection/requirements.txt new file mode 100644 index 0000000..6981bd7 --- /dev/null +++ b/detection/requirements.txt @@ -0,0 +1,4 @@ +-r requirements/build.txt +-r requirements/optional.txt +-r requirements/runtime.txt +-r requirements/tests.txt diff --git a/detection/setup.cfg b/detection/setup.cfg new file mode 100644 index 0000000..78eb65e --- /dev/null +++ b/detection/setup.cfg @@ -0,0 +1,13 @@ +[isort] +line_length = 79 +multi_line_output = 0 +known_standard_library = setuptools +known_first_party = mmdet +known_third_party = PIL,asynctest,cityscapesscripts,cv2,gather_models,matplotlib,mmcv,numpy,onnx,onnxruntime,pycocotools,pytest,seaborn,six,terminaltables,torch,ts +no_lines_before = STDLIB,LOCALFOLDER +default_section = THIRDPARTY + +[yapf] +BASED_ON_STYLE = pep8 +BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF = true +SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN = true diff --git a/detection/setup.py b/detection/setup.py new file mode 100644 index 0000000..55eea6b --- /dev/null +++ b/detection/setup.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python +import os +from setuptools import find_packages, setup + +import torch +from torch.utils.cpp_extension import (BuildExtension, CppExtension, + CUDAExtension) + + +def readme(): + with open('README.md', encoding='utf-8') as f: + content = f.read() + return content + + +version_file = 'mmdet/version.py' + + +def get_version(): + with open(version_file, 'r') as f: + exec(compile(f.read(), version_file, 'exec')) + return locals()['__version__'] + + +def make_cuda_ext(name, module, sources, sources_cuda=[]): + + define_macros = [] + extra_compile_args = {'cxx': []} + + if torch.cuda.is_available() or os.getenv('FORCE_CUDA', '0') == '1': + define_macros += [('WITH_CUDA', None)] + extension = CUDAExtension + extra_compile_args['nvcc'] = [ + '-D__CUDA_NO_HALF_OPERATORS__', + '-D__CUDA_NO_HALF_CONVERSIONS__', + '-D__CUDA_NO_HALF2_OPERATORS__', + ] + sources += sources_cuda + else: + print(f'Compiling {name} without CUDA') + extension = CppExtension + + return extension( + name=f'{module}.{name}', + sources=[os.path.join(*module.split('.'), p) for p in sources], + define_macros=define_macros, + extra_compile_args=extra_compile_args) + + +def parse_requirements(fname='requirements.txt', with_version=True): + """Parse the package dependencies listed in a requirements file but strips + specific versioning information. + + Args: + fname (str): path to requirements file + with_version (bool, default=False): if True include version specs + + Returns: + List[str]: list of requirements items + + CommandLine: + python -c "import setup; print(setup.parse_requirements())" + """ + import sys + from os.path import exists + import re + require_fpath = fname + + def parse_line(line): + """Parse information from a line in a requirements text file.""" + if line.startswith('-r '): + # Allow specifying requirements in other files + target = line.split(' ')[1] + for info in parse_require_file(target): + yield info + else: + info = {'line': line} + if line.startswith('-e '): + info['package'] = line.split('#egg=')[1] + elif '@git+' in line: + info['package'] = line + else: + # Remove versioning from the package + pat = '(' + '|'.join(['>=', '==', '>']) + ')' + parts = re.split(pat, line, maxsplit=1) + parts = [p.strip() for p in parts] + + info['package'] = parts[0] + if len(parts) > 1: + op, rest = parts[1:] + if ';' in rest: + # Handle platform specific dependencies + # http://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-platform-specific-dependencies + version, platform_deps = map(str.strip, + rest.split(';')) + info['platform_deps'] = platform_deps + else: + version = rest # NOQA + info['version'] = (op, version) + yield info + + def parse_require_file(fpath): + with open(fpath, 'r') as f: + for line in f.readlines(): + line = line.strip() + if line and not line.startswith('#'): + for info in parse_line(line): + yield info + + def gen_packages_items(): + if exists(require_fpath): + for info in parse_require_file(require_fpath): + parts = [info['package']] + if with_version and 'version' in info: + parts.extend(info['version']) + if not sys.version.startswith('3.4'): + # apparently package_deps are broken in 3.4 + platform_deps = info.get('platform_deps') + if platform_deps is not None: + parts.append(';' + platform_deps) + item = ''.join(parts) + yield item + + packages = list(gen_packages_items()) + return packages + + +if __name__ == '__main__': + setup( + name='mmdet', + version=get_version(), + description='OpenMMLab Detection Toolbox and Benchmark', + long_description=readme(), + long_description_content_type='text/markdown', + author='OpenMMLab', + author_email='openmmlab@gmail.com', + keywords='computer vision, object detection', + url='https://github.com/open-mmlab/mmdetection', + packages=find_packages(exclude=('configs', 'tools', 'demo')), + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + ], + license='Apache License 2.0', + setup_requires=parse_requirements('requirements/build.txt'), + tests_require=parse_requirements('requirements/tests.txt'), + install_requires=parse_requirements('requirements/runtime.txt'), + extras_require={ + 'all': parse_requirements('requirements.txt'), + 'tests': parse_requirements('requirements/tests.txt'), + 'build': parse_requirements('requirements/build.txt'), + 'optional': parse_requirements('requirements/optional.txt'), + }, + ext_modules=[], + cmdclass={'build_ext': BuildExtension}, + zip_safe=False) diff --git a/detection/tools/benchmark.py b/detection/tools/benchmark.py new file mode 100644 index 0000000..5f1c04a --- /dev/null +++ b/detection/tools/benchmark.py @@ -0,0 +1,189 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import argparse +import copy +import os +import time + +import torch +from mmcv import Config, DictAction +from mmcv.cnn import fuse_conv_bn +from mmcv.parallel import MMDistributedDataParallel +from mmcv.runner import init_dist, load_checkpoint, wrap_fp16_model + +from mmdet.datasets import (build_dataloader, build_dataset, + replace_ImageToTensor) +from mmdet.models import build_detector + + +def parse_args(): + parser = argparse.ArgumentParser(description='MMDet benchmark a model') + parser.add_argument('config', help='test config file path') + parser.add_argument('--checkpoint', help='checkpoint file') + parser.add_argument( + '--repeat-num', + type=int, + default=1, + help='number of repeat times of measurement for averaging the results') + parser.add_argument( + '--max-iter', type=int, default=2000, help='num of max iter') + parser.add_argument( + '--log-interval', type=int, default=50, help='interval of logging') + parser.add_argument( + '--fuse-conv-bn', + action='store_true', + help='Whether to fuse conv and bn, this will slightly increase' + 'the inference speed') + parser.add_argument( + '--cfg-options', + nargs='+', + action=DictAction, + help='override some settings in the used config, the key-value pair ' + 'in xxx=yyy format will be merged into config file. If the value to ' + 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' + 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' + 'Note that the quotation marks are necessary and that no white space ' + 'is allowed.') + parser.add_argument( + '--launcher', + choices=['none', 'pytorch', 'slurm', 'mpi'], + default='none', + help='job launcher') + parser.add_argument('--local_rank', type=int, default=0) + args = parser.parse_args() + if 'LOCAL_RANK' not in os.environ: + os.environ['LOCAL_RANK'] = str(args.local_rank) + return args + + +def measure_inference_speed(cfg, checkpoint, max_iter, log_interval, + is_fuse_conv_bn): + # set cudnn_benchmark + if cfg.get('cudnn_benchmark', False): + torch.backends.cudnn.benchmark = True + cfg.model.pretrained = None + cfg.data.test.test_mode = True + + # build the dataloader + samples_per_gpu = cfg.data.test.pop('samples_per_gpu', 1) + if samples_per_gpu > 1: + # Replace 'ImageToTensor' to 'DefaultFormatBundle' + cfg.data.test.pipeline = replace_ImageToTensor(cfg.data.test.pipeline) + dataset = build_dataset(cfg.data.test) + data_loader = build_dataloader( + dataset, + samples_per_gpu=1, + # Because multiple processes will occupy additional CPU resources, + # FPS statistics will be more unstable when workers_per_gpu is not 0. + # It is reasonable to set workers_per_gpu to 0. + workers_per_gpu=0, + dist=True, + shuffle=False) + + # build the model and load checkpoint + cfg.model.train_cfg = None + model = build_detector(cfg.model, test_cfg=cfg.get('test_cfg')) + fp16_cfg = cfg.get('fp16', None) + if fp16_cfg is not None: + wrap_fp16_model(model) + + load_checkpoint(model, checkpoint, map_location='cpu') + if is_fuse_conv_bn: + model = fuse_conv_bn(model) + + model = MMDistributedDataParallel( + model.cuda(), + device_ids=[torch.cuda.current_device()], + broadcast_buffers=False) + model.eval() + + # the first several iterations may be very slow so skip them + num_warmup = 5 + pure_inf_time = 0 + fps = 0 + + # benchmark with 2000 image and take the average + for i, data in enumerate(data_loader): + + torch.cuda.synchronize() + start_time = time.perf_counter() + + with torch.no_grad(): + model(return_loss=False, rescale=True, **data) + + torch.cuda.synchronize() + elapsed = time.perf_counter() - start_time + + if i >= num_warmup: + pure_inf_time += elapsed + if (i + 1) % log_interval == 0: + fps = (i + 1 - num_warmup) / pure_inf_time + print( + f'Done image [{i + 1:<3}/ {max_iter}], ' + f'fps: {fps:.1f} img / s, ' + f'times per image: {1000 / fps:.1f} ms / img', + flush=True) + + if (i + 1) == max_iter: + fps = (i + 1 - num_warmup) / pure_inf_time + print( + f'Overall fps: {fps:.1f} img / s, ' + f'times per image: {1000 / fps:.1f} ms / img', + flush=True) + break + return fps + + +def repeat_measure_inference_speed(cfg, + checkpoint, + max_iter, + log_interval, + is_fuse_conv_bn, + repeat_num=1): + assert repeat_num >= 1 + + fps_list = [] + + for _ in range(repeat_num): + # + cp_cfg = copy.deepcopy(cfg) + + fps_list.append( + measure_inference_speed(cp_cfg, checkpoint, max_iter, log_interval, + is_fuse_conv_bn)) + + if repeat_num > 1: + fps_list_ = [round(fps, 1) for fps in fps_list] + times_pre_image_list_ = [round(1000 / fps, 1) for fps in fps_list] + mean_fps_ = sum(fps_list_) / len(fps_list_) + mean_times_pre_image_ = sum(times_pre_image_list_) / len( + times_pre_image_list_) + print( + f'Overall fps: {fps_list_}[{mean_fps_:.1f}] img / s, ' + f'times per image: ' + f'{times_pre_image_list_}[{mean_times_pre_image_:.1f}] ms / img', + flush=True) + return fps_list + + return fps_list[0] + + +def main(): + args = parse_args() + + cfg = Config.fromfile(args.config) + + if args.cfg_options is not None: + cfg.merge_from_dict(args.cfg_options) + + if args.launcher == 'none': + raise NotImplementedError('Only supports distributed mode') + else: + init_dist(args.launcher, **cfg.dist_params) + + repeat_measure_inference_speed(cfg, args.checkpoint, args.max_iter, + args.log_interval, args.fuse_conv_bn, + args.repeat_num) + + +if __name__ == '__main__': + main() diff --git a/detection/tools/dist_test.sh b/detection/tools/dist_test.sh new file mode 100644 index 0000000..3c74ec6 --- /dev/null +++ b/detection/tools/dist_test.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +CONFIG=$1 +CHECKPOINT=$2 +GPUS=$3 +PORT=${PORT:-29500} + +PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \ +python -m torch.distributed.launch --nproc_per_node=$GPUS --master_port=$PORT \ + $(dirname "$0")/test.py $CONFIG $CHECKPOINT --launcher pytorch ${@:4} diff --git a/detection/tools/dist_train.sh b/detection/tools/dist_train.sh new file mode 100644 index 0000000..5b43fff --- /dev/null +++ b/detection/tools/dist_train.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +CONFIG=$1 +GPUS=$2 +PORT=${PORT:-29500} + +PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \ +python -m torch.distributed.launch --nproc_per_node=$GPUS --master_port=$PORT \ + $(dirname "$0")/train.py $CONFIG --launcher pytorch ${@:3} diff --git a/detection/tools/get_flops.py b/detection/tools/get_flops.py new file mode 100644 index 0000000..7e635bb --- /dev/null +++ b/detection/tools/get_flops.py @@ -0,0 +1,125 @@ +import argparse + +import torch +from mmcv import Config, DictAction + +from mmdet.models import build_detector + +try: + from mmcv.cnn import get_model_complexity_info +except ImportError: + raise ImportError('Please upgrade mmcv to >0.6.2') +from mmcv.cnn.utils.flops_counter import get_model_complexity_info, flops_to_string, params_to_string + +def parse_args(): + parser = argparse.ArgumentParser(description='Train a detector') + parser.add_argument('config', help='train config file path') + parser.add_argument( + '--shape', + type=int, + nargs='+', + default=[1280, 800], + help='input image size') + parser.add_argument( + '--cfg-options', + nargs='+', + action=DictAction, + help='override some settings in the used config, the key-value pair ' + 'in xxx=yyy format will be merged into config file. If the value to ' + 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' + 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' + 'Note that the quotation marks are necessary and that no white space ' + 'is allowed.') + args = parser.parse_args() + return args + + +def sra_flops(h, w, r, dim): + return 2 * h * w * (h // r) * (w // r) * dim + + +def li_sra_flops(h, w, dim): + return 2 * h * w * 7 * 7 * dim + +import math + +def msa_flops(H, W, dim): + N = H * W + return N * dim * N * 2 + +def hilo_flops(H, W, l_dim, h_dim, sr_ratio): + # H = int(N ** 0.5) + # ws = sr_ratio = 4 + ws = sr_ratio + Hp = ws * math.ceil(H / ws) + Wp = ws * math.ceil(W / ws) + Np = Hp * Wp + + nW = (Hp // ws) * (Wp // ws) + window_len = ws * ws + window_flops = window_len * window_len * h_dim * 2 + + high_flops = nW * window_flops + kv_len = (Hp // sr_ratio) * (Wp // sr_ratio) + low_flops = Np * l_dim * kv_len * 2 + + return high_flops + low_flops + +def get_flops(model, input_shape): + flops, params = get_model_complexity_info(model, input_shape, as_strings=False) + + backbone = model.backbone + _, H, W = input_shape + l_dim = int(backbone.alpha * backbone.num_heads[2]) * 32 + h_dim = (backbone.num_heads[2] - int(backbone.alpha * backbone.num_heads[2])) * 32 + stage3 = hilo_flops(H // 16, W // 16, l_dim, h_dim, backbone.local_ws[2]) * len(backbone.layers[2].blocks) + stage4 = msa_flops(H // 32, W // 32, backbone.num_heads[3] * 32) * len(backbone.layers[3].blocks) + flops += stage3 + stage4 + return flops_to_string(flops), params_to_string(params) + + +def main(): + args = parse_args() + + if len(args.shape) == 1: + input_shape = (3, args.shape[0], args.shape[0]) + elif len(args.shape) == 2: + input_shape = (3,) + tuple(args.shape) + else: + raise ValueError('invalid input shape') + + cfg = Config.fromfile(args.config) + if args.cfg_options is not None: + cfg.merge_from_dict(args.cfg_options) + # import modules from string list. + if cfg.get('custom_imports', None): + from mmcv.utils import import_modules_from_strings + import_modules_from_strings(**cfg['custom_imports']) + + model = build_detector( + cfg.model, + train_cfg=cfg.get('train_cfg'), + test_cfg=cfg.get('test_cfg')) + if torch.cuda.is_available(): + model.cuda() + model.eval() + + if hasattr(model, 'forward_dummy'): + model.forward = model.forward_dummy + else: + raise NotImplementedError( + 'FLOPs counter is currently not currently supported with {}'. + format(model.__class__.__name__)) + + flops, params = get_flops(model, input_shape) + + split_line = '=' * 30 + print(f'{split_line}\nInput shape: {input_shape}\n' + f'Flops: {flops}\nParams: {params}\n{split_line}') + print('!!!Please be cautious if you use the results in papers. ' + 'You may need to check if all ops are supported and verify that the ' + 'flops computation is correct.') + + +if __name__ == '__main__': + main() diff --git a/detection/tools/test.py b/detection/tools/test.py new file mode 100644 index 0000000..d7d9418 --- /dev/null +++ b/detection/tools/test.py @@ -0,0 +1,220 @@ +import argparse +import os +import warnings + +import mmcv +import torch +from mmcv import Config, DictAction +from mmcv.cnn import fuse_conv_bn +from mmcv.parallel import MMDataParallel, MMDistributedDataParallel +from mmcv.runner import (get_dist_info, init_dist, load_checkpoint, + wrap_fp16_model) + +from mmdet.apis import multi_gpu_test, single_gpu_test +from mmdet.datasets import (build_dataloader, build_dataset, + replace_ImageToTensor) +from mmdet.models import build_detector + + +def parse_args(): + parser = argparse.ArgumentParser( + description='MMDet test (and eval) a model') + parser.add_argument('config', help='test config file path') + parser.add_argument('checkpoint', help='checkpoint file') + parser.add_argument('--out', help='output result file in pickle format') + parser.add_argument( + '--fuse-conv-bn', + action='store_true', + help='Whether to fuse conv and bn, this will slightly increase' + 'the inference speed') + parser.add_argument( + '--format-only', + action='store_true', + help='Format the output results without perform evaluation. It is' + 'useful when you want to format the result to a specific format and ' + 'submit it to the test server') + parser.add_argument( + '--eval', + type=str, + nargs='+', + help='evaluation metrics, which depends on the dataset, e.g., "bbox",' + ' "segm", "proposal" for COCO, and "mAP", "recall" for PASCAL VOC') + parser.add_argument('--show', action='store_true', help='show results') + parser.add_argument( + '--show-dir', help='directory where painted images will be saved') + parser.add_argument( + '--show-score-thr', + type=float, + default=0.3, + help='score threshold (default: 0.3)') + parser.add_argument( + '--gpu-collect', + action='store_true', + help='whether to use gpu to collect results.') + parser.add_argument( + '--tmpdir', + help='tmp directory used for collecting results from multiple ' + 'workers, available when gpu-collect is not specified') + parser.add_argument( + '--cfg-options', + nargs='+', + action=DictAction, + help='override some settings in the used config, the key-value pair ' + 'in xxx=yyy format will be merged into config file. If the value to ' + 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' + 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' + 'Note that the quotation marks are necessary and that no white space ' + 'is allowed.') + parser.add_argument( + '--options', + nargs='+', + action=DictAction, + help='custom options for evaluation, the key-value pair in xxx=yyy ' + 'format will be kwargs for dataset.evaluate() function (deprecate), ' + 'change to --eval-options instead.') + parser.add_argument( + '--eval-options', + nargs='+', + action=DictAction, + help='custom options for evaluation, the key-value pair in xxx=yyy ' + 'format will be kwargs for dataset.evaluate() function') + parser.add_argument( + '--launcher', + choices=['none', 'pytorch', 'slurm', 'mpi'], + default='none', + help='job launcher') + parser.add_argument('--local_rank', type=int, default=0) + args = parser.parse_args() + if 'LOCAL_RANK' not in os.environ: + os.environ['LOCAL_RANK'] = str(args.local_rank) + + if args.options and args.eval_options: + raise ValueError( + '--options and --eval-options cannot be both ' + 'specified, --options is deprecated in favor of --eval-options') + if args.options: + warnings.warn('--options is deprecated in favor of --eval-options') + args.eval_options = args.options + return args + + +def main(): + args = parse_args() + + assert args.out or args.eval or args.format_only or args.show \ + or args.show_dir, \ + ('Please specify at least one operation (save/eval/format/show the ' + 'results / save the results) with the argument "--out", "--eval"' + ', "--format-only", "--show" or "--show-dir"') + + if args.eval and args.format_only: + raise ValueError('--eval and --format_only cannot be both specified') + + if args.out is not None and not args.out.endswith(('.pkl', '.pickle')): + raise ValueError('The output file must be a pkl file.') + + cfg = Config.fromfile(args.config) + if args.cfg_options is not None: + cfg.merge_from_dict(args.cfg_options) + # import modules from string list. + if cfg.get('custom_imports', None): + from mmcv.utils import import_modules_from_strings + import_modules_from_strings(**cfg['custom_imports']) + # set cudnn_benchmark + if cfg.get('cudnn_benchmark', False): + torch.backends.cudnn.benchmark = True + cfg.model.pretrained = None + if cfg.model.get('neck'): + if isinstance(cfg.model.neck, list): + for neck_cfg in cfg.model.neck: + if neck_cfg.get('rfp_backbone'): + if neck_cfg.rfp_backbone.get('pretrained'): + neck_cfg.rfp_backbone.pretrained = None + elif cfg.model.neck.get('rfp_backbone'): + if cfg.model.neck.rfp_backbone.get('pretrained'): + cfg.model.neck.rfp_backbone.pretrained = None + + # in case the test dataset is concatenated + samples_per_gpu = 1 + if isinstance(cfg.data.test, dict): + cfg.data.test.test_mode = True + samples_per_gpu = cfg.data.test.pop('samples_per_gpu', 1) + if samples_per_gpu > 1: + # Replace 'ImageToTensor' to 'DefaultFormatBundle' + cfg.data.test.pipeline = replace_ImageToTensor( + cfg.data.test.pipeline) + elif isinstance(cfg.data.test, list): + for ds_cfg in cfg.data.test: + ds_cfg.test_mode = True + samples_per_gpu = max( + [ds_cfg.pop('samples_per_gpu', 1) for ds_cfg in cfg.data.test]) + if samples_per_gpu > 1: + for ds_cfg in cfg.data.test: + ds_cfg.pipeline = replace_ImageToTensor(ds_cfg.pipeline) + + # init distributed env first, since logger depends on the dist info. + if args.launcher == 'none': + distributed = False + else: + distributed = True + init_dist(args.launcher, **cfg.dist_params) + + # build the dataloader + dataset = build_dataset(cfg.data.test) + data_loader = build_dataloader( + dataset, + samples_per_gpu=samples_per_gpu, + workers_per_gpu=cfg.data.workers_per_gpu, + dist=distributed, + shuffle=False) + + # build the model and load checkpoint + cfg.model.train_cfg = None + model = build_detector(cfg.model, test_cfg=cfg.get('test_cfg')) + fp16_cfg = cfg.get('fp16', None) + if fp16_cfg is not None: + wrap_fp16_model(model) + checkpoint = load_checkpoint(model, args.checkpoint, map_location='cpu') + if args.fuse_conv_bn: + model = fuse_conv_bn(model) + # old versions did not save class info in checkpoints, this walkaround is + # for backward compatibility + if 'CLASSES' in checkpoint.get('meta', {}): + model.CLASSES = checkpoint['meta']['CLASSES'] + else: + model.CLASSES = dataset.CLASSES + + if not distributed: + model = MMDataParallel(model, device_ids=[0]) + outputs = single_gpu_test(model, data_loader, args.show, args.show_dir, + args.show_score_thr) + else: + model = MMDistributedDataParallel( + model.cuda(), + device_ids=[torch.cuda.current_device()], + broadcast_buffers=False) + outputs = multi_gpu_test(model, data_loader, args.tmpdir, + args.gpu_collect) + + rank, _ = get_dist_info() + if rank == 0: + if args.out: + print(f'\nwriting results to {args.out}') + mmcv.dump(outputs, args.out) + kwargs = {} if args.eval_options is None else args.eval_options + if args.format_only: + dataset.format_results(outputs, **kwargs) + if args.eval: + eval_kwargs = cfg.get('evaluation', {}).copy() + # hard-code way to remove EvalHook args + for key in [ + 'interval', 'tmpdir', 'start', 'gpu_collect', 'save_best', + 'rule' + ]: + eval_kwargs.pop(key, None) + eval_kwargs.update(dict(metric=args.eval, **kwargs)) + print(dataset.evaluate(outputs, **eval_kwargs)) + + +if __name__ == '__main__': + main() diff --git a/detection/tools/train.py b/detection/tools/train.py new file mode 100644 index 0000000..e124d63 --- /dev/null +++ b/detection/tools/train.py @@ -0,0 +1,190 @@ +import argparse +import copy +import os +import os.path as osp +import time +import warnings + +import mmcv +import torch +from mmcv import Config, DictAction +from mmcv.runner import get_dist_info, init_dist +from mmcv.utils import get_git_hash + +from mmdet import __version__ +from mmdet.apis import set_random_seed, train_detector +from mmdet.datasets import build_dataset +from mmdet.models import build_detector +from mmdet.utils import collect_env, get_root_logger + + +def parse_args(): + parser = argparse.ArgumentParser(description='Train a detector') + parser.add_argument('config', help='train config file path') + parser.add_argument('--work-dir', help='the dir to save logs and models') + parser.add_argument( + '--resume-from', help='the checkpoint file to resume from') + parser.add_argument( + '--no-validate', + action='store_true', + help='whether not to evaluate the checkpoint during training') + group_gpus = parser.add_mutually_exclusive_group() + group_gpus.add_argument( + '--gpus', + type=int, + help='number of gpus to use ' + '(only applicable to non-distributed training)') + group_gpus.add_argument( + '--gpu-ids', + type=int, + nargs='+', + help='ids of gpus to use ' + '(only applicable to non-distributed training)') + parser.add_argument('--seed', type=int, default=None, help='random seed') + parser.add_argument( + '--deterministic', + action='store_true', + help='whether to set deterministic options for CUDNN backend.') + parser.add_argument( + '--options', + nargs='+', + action=DictAction, + help='override some settings in the used config, the key-value pair ' + 'in xxx=yyy format will be merged into config file (deprecate), ' + 'change to --cfg-options instead.') + parser.add_argument( + '--cfg-options', + nargs='+', + action=DictAction, + help='override some settings in the used config, the key-value pair ' + 'in xxx=yyy format will be merged into config file. If the value to ' + 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' + 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' + 'Note that the quotation marks are necessary and that no white space ' + 'is allowed.') + parser.add_argument( + '--launcher', + choices=['none', 'pytorch', 'slurm', 'mpi'], + default='none', + help='job launcher') + parser.add_argument('--local_rank', type=int, default=0) + args = parser.parse_args() + if 'LOCAL_RANK' not in os.environ: + os.environ['LOCAL_RANK'] = str(args.local_rank) + + if args.options and args.cfg_options: + raise ValueError( + '--options and --cfg-options cannot be both ' + 'specified, --options is deprecated in favor of --cfg-options') + if args.options: + warnings.warn('--options is deprecated in favor of --cfg-options') + args.cfg_options = args.options + + return args + + +def main(): + args = parse_args() + + cfg = Config.fromfile(args.config) + if args.cfg_options is not None: + cfg.merge_from_dict(args.cfg_options) + # import modules from string list. + if cfg.get('custom_imports', None): + from mmcv.utils import import_modules_from_strings + import_modules_from_strings(**cfg['custom_imports']) + # set cudnn_benchmark + if cfg.get('cudnn_benchmark', False): + torch.backends.cudnn.benchmark = True + + # work_dir is determined in this priority: CLI > segment in file > filename + if args.work_dir is not None: + # update configs according to CLI args if args.work_dir is not None + cfg.work_dir = args.work_dir + elif cfg.get('work_dir', None) is None: + # use config filename as default work_dir if cfg.work_dir is None + cfg.work_dir = osp.join('./work_dirs', + osp.splitext(osp.basename(args.config))[0]) + if args.resume_from is not None: + cfg.resume_from = args.resume_from + if args.gpu_ids is not None: + cfg.gpu_ids = args.gpu_ids + else: + cfg.gpu_ids = range(1) if args.gpus is None else range(args.gpus) + + # init distributed env first, since logger depends on the dist info. + if args.launcher == 'none': + distributed = False + else: + distributed = True + init_dist(args.launcher, **cfg.dist_params) + # re-set gpu_ids with distributed training mode + _, world_size = get_dist_info() + cfg.gpu_ids = range(world_size) + + # create work_dir + mmcv.mkdir_or_exist(osp.abspath(cfg.work_dir)) + # dump config + cfg.dump(osp.join(cfg.work_dir, osp.basename(args.config))) + # init the logger before other steps + timestamp = time.strftime('%Y%m%d_%H%M%S', time.localtime()) + log_file = osp.join(cfg.work_dir, f'{timestamp}.log') + logger = get_root_logger(log_file=log_file, log_level=cfg.log_level) + + # init the meta dict to record some important information such as + # environment info and seed, which will be logged + meta = dict() + # log env info + env_info_dict = collect_env() + env_info = '\n'.join([(f'{k}: {v}') for k, v in env_info_dict.items()]) + dash_line = '-' * 60 + '\n' + logger.info('Environment info:\n' + dash_line + env_info + '\n' + + dash_line) + meta['env_info'] = env_info + meta['config'] = cfg.pretty_text + # log some basic info + logger.info(f'Distributed training: {distributed}') + logger.info(f'Config:\n{cfg.pretty_text}') + + # set random seeds + if args.seed is not None: + logger.info(f'Set random seed to {args.seed}, ' + f'deterministic: {args.deterministic}') + set_random_seed(args.seed, deterministic=args.deterministic) + cfg.seed = args.seed + meta['seed'] = args.seed + meta['exp_name'] = osp.basename(args.config) + + model = build_detector( + cfg.model, + train_cfg=cfg.get('train_cfg'), + test_cfg=cfg.get('test_cfg')) + + # print(model) + n_parameters = sum(p.numel() for p in model.parameters() if p.requires_grad) + print('number of params:', n_parameters) + datasets = [build_dataset(cfg.data.train)] + if len(cfg.workflow) == 2: + val_dataset = copy.deepcopy(cfg.data.val) + val_dataset.pipeline = cfg.data.train.pipeline + datasets.append(build_dataset(val_dataset)) + if cfg.checkpoint_config is not None: + # save mmdet version, config file content and class names in + # checkpoints as meta data + cfg.checkpoint_config.meta = dict( + mmdet_version=__version__ + get_git_hash()[:7], + CLASSES=datasets[0].CLASSES) + # add an attribute for visualization convenience + model.CLASSES = datasets[0].CLASSES + train_detector( + model, + datasets, + cfg, + distributed=distributed, + validate=(not args.no_validate), + timestamp=timestamp, + meta=meta) + + +if __name__ == '__main__': + main() diff --git a/segmentation/.github/CODE_OF_CONDUCT.md b/segmentation/.github/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..efd4305 --- /dev/null +++ b/segmentation/.github/CODE_OF_CONDUCT.md @@ -0,0 +1,76 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at chenkaidev@gmail.com. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq diff --git a/segmentation/.github/CONTRIBUTING.md b/segmentation/.github/CONTRIBUTING.md new file mode 100644 index 0000000..112527e --- /dev/null +++ b/segmentation/.github/CONTRIBUTING.md @@ -0,0 +1,57 @@ +# Contributing to mmsegmentation + +All kinds of contributions are welcome, including but not limited to the following. + +- Fixes (typo, bugs) +- New features and components + +## Workflow + +1. fork and pull the latest mmsegmentation +2. checkout a new branch (do not use master branch for PRs) +3. commit your changes +4. create a PR + +Note + +- If you plan to add some new features that involve large changes, it is encouraged to open an issue for discussion first. +- If you are the author of some papers and would like to include your method to mmsegmentation, + please contact Kai Chen (chenkaidev[at]gmail[dot]com). We will much appreciate your contribution. + +## Code style + +### Python + +We adopt [PEP8](https://www.python.org/dev/peps/pep-0008/) as the preferred code style. + +We use the following tools for linting and formatting: + +- [flake8](http://flake8.pycqa.org/en/latest/): linter +- [yapf](https://github.com/google/yapf): formatter +- [isort](https://github.com/timothycrosley/isort): sort imports + +Style configurations of yapf and isort can be found in [setup.cfg](../setup.cfg) and [.isort.cfg](../.isort.cfg). + +We use [pre-commit hook](https://pre-commit.com/) that checks and formats for `flake8`, `yapf`, `isort`, `trailing whitespaces`, + fixes `end-of-files`, sorts `requirments.txt` automatically on every commit. +The config for a pre-commit hook is stored in [.pre-commit-config](../.pre-commit-config.yaml). + +After you clone the repository, you will need to install initialize pre-commit hook. + +```shell +pip install -U pre-commit +``` + +From the repository folder + +```shell +pre-commit install +``` + +After this on every commit check code linters and formatter will be enforced. + +>Before you create a PR, make sure that your code lints and is formatted by yapf. + +### C++ and CUDA + +We follow the [Google C++ Style Guide](https://google.github.io/styleguide/cppguide.html). diff --git a/segmentation/.github/ISSUE_TEMPLATE/config.yml b/segmentation/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..6eaae3e --- /dev/null +++ b/segmentation/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,6 @@ +blank_issues_enabled: false + +contact_links: + - name: MMSegmentation Documentation + url: https://mmsegmentation.readthedocs.io + about: Check the docs and FAQ to see if you question is already anwsered. diff --git a/segmentation/.github/ISSUE_TEMPLATE/error-report.md b/segmentation/.github/ISSUE_TEMPLATE/error-report.md new file mode 100644 index 0000000..73a63b7 --- /dev/null +++ b/segmentation/.github/ISSUE_TEMPLATE/error-report.md @@ -0,0 +1,48 @@ +--- +name: Error report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +Thanks for your error report and we appreciate it a lot. + +**Checklist** + +1. I have searched related issues but cannot get the expected help. +2. The bug has not been fixed in the latest version. + +**Describe the bug** +A clear and concise description of what the bug is. + +**Reproduction** + +1. What command or script did you run? + + ```none + A placeholder for the command. + ``` + +2. Did you make any modifications on the code or config? Did you understand what you have modified? +3. What dataset did you use? + +**Environment** + +1. Please run `python mmseg/utils/collect_env.py` to collect necessary environment infomation and paste it here. +2. You may add addition that may be helpful for locating the problem, such as + - How you installed PyTorch [e.g., pip, conda, source] + - Other environment variables that may be related (such as `$PATH`, `$LD_LIBRARY_PATH`, `$PYTHONPATH`, etc.) + +**Error traceback** + +If applicable, paste the error trackback here. + +```none +A placeholder for trackback. +``` + +**Bug fix** + +If you have already identified the reason, you can provide the information here. If you are willing to create a PR to fix it, please also leave a comment here and that would be much appreciated! diff --git a/segmentation/.github/ISSUE_TEMPLATE/feature_request.md b/segmentation/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..ec59b78 --- /dev/null +++ b/segmentation/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,22 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +# Describe the feature + +**Motivation** +A clear and concise description of the motivation of the feature. +Ex1. It is inconvenient when [....]. +Ex2. There is a recent paper [....], which is very helpful for [....]. + +**Related resources** +If there is an official code release or third-party implementations, please also provide the information here, which would be very helpful. + +**Additional context** +Add any other context or screenshots about the feature request here. +If you would like to implement the feature and create a PR, please leave a comment here and that would be much appreciated. diff --git a/segmentation/.github/ISSUE_TEMPLATE/general_questions.md b/segmentation/.github/ISSUE_TEMPLATE/general_questions.md new file mode 100644 index 0000000..b5a6451 --- /dev/null +++ b/segmentation/.github/ISSUE_TEMPLATE/general_questions.md @@ -0,0 +1,8 @@ +--- +name: General questions +about: Ask general questions to get help +title: '' +labels: '' +assignees: '' + +--- diff --git a/segmentation/.gitignore b/segmentation/.gitignore new file mode 100644 index 0000000..8ae9cb6 --- /dev/null +++ b/segmentation/.gitignore @@ -0,0 +1,117 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +data/ +data +.vscode +.idea + +# custom +*.pkl +*.pkl.json +*.log.json +work_dirs/ + +# Pytorch +*.pth \ No newline at end of file diff --git a/segmentation/LICENSE b/segmentation/LICENSE new file mode 100644 index 0000000..38e625b --- /dev/null +++ b/segmentation/LICENSE @@ -0,0 +1,203 @@ +Copyright 2020 The MMSegmentation Authors. All rights reserved. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2020 The MMSegmentation Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/segmentation/README.md b/segmentation/README.md new file mode 100644 index 0000000..d7612d5 --- /dev/null +++ b/segmentation/README.md @@ -0,0 +1,91 @@ +# Semantic Segmentation code for LITv2 + +## Installation + +1. Install [mmsegmentation](https://github.com/open-mmlab/mmsegmentation). + +2. Download ADE20K dataset from the [official website](https://groups.csail.mit.edu/vision/datasets/ADE20K/). The directory structure should look like + + ``` + ade + └── ADEChallengeData2016 + ├── annotations + │ ├── training + │ └── validation + └── images + ├── training + └── validation + ``` + + Next, create a symbolic link to the dataset. + + ```bash + cd segmentation/ + mkdir data + ln -s [path/to/ade20k] data/ + ``` + +3. Download LITv2 pretrained weights on ImageNet. + + + +## Training + +To train a model with pre-trained weights, run: + +```bash +# single-gpu training +python tools/train.py --options model.pretrained= [model.backbone.use_checkpoint=True] [other optional arguments] + +# multi-gpu training +tools/dist_train.sh --options model.pretrained= [model.backbone.use_checkpoint=True] [other optional arguments] +``` + +For example, to train a Semantic FPN model with a LITv2-S backbone on 8 GPUs, run: + +```bash +tools/dist_train.sh configs/litv2/litv2_s_fpn_r50_512x512_80k_ade20k.py 8 --options model.pretrained=litv2_s.pth +``` + +## Inference + +```bash +# single-gpu testing +python tools/test.py --eval mIoU + +# multi-gpu testing +tools/dist_test.sh --eval mIoU +``` + +For example, to evaluate a Semantic FPN model with a LITv2-S backbone, run: + +```bash +tools/dist_test.sh configs/litv2/litv2_s_fpn_r50_512x512_80k_ade20k.py litv2_s_fpn_r50_512x512_80k_ade20k.pth 8 --eval mIoU +``` + + + +## Benchmark + +To get the FLOPs, run + +```bash +python tools/get_flops.py configs/litv2/litv2_s_fpn_r50_512x512_80k_ade20k.py +``` + +This should give + +```bash +Input shape: (3, 512, 512) +Flops: 41.29 GFLOPs +Params: 31.45 M +``` + +To test the FPS, run + +```bash +python -m torch.distributed.launch --nproc_per_node=1 --master_port=29500 tools/benchmark.py \ + configs/lit/retinanet_litv2_s_fpn_1x_coco.py \ + --checkpoint retinanet_litv2_s_fpn_1x_coco.pth \ + --launcher pytorch +``` diff --git a/segmentation/configs/_base_/datasets/ade20k.py b/segmentation/configs/_base_/datasets/ade20k.py new file mode 100644 index 0000000..f2daee1 --- /dev/null +++ b/segmentation/configs/_base_/datasets/ade20k.py @@ -0,0 +1,54 @@ +# dataset settings +dataset_type = 'ADE20KDataset' +data_root = 'data/ade/ADEChallengeData2016' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +crop_size = (512, 512) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations', reduce_zero_label=True), + dict(type='Resize', img_scale=(2048, 512), ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg']), +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(2048, 512), + #img_ratios=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +data = dict( + samples_per_gpu=2, + workers_per_gpu=2, + train=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/training', + ann_dir='annotations/training', + pipeline=train_pipeline), + val=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/validation', + ann_dir='annotations/validation', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/validation', + ann_dir='annotations/validation', + pipeline=test_pipeline)) diff --git a/segmentation/configs/_base_/default_runtime.py b/segmentation/configs/_base_/default_runtime.py new file mode 100644 index 0000000..b564cc4 --- /dev/null +++ b/segmentation/configs/_base_/default_runtime.py @@ -0,0 +1,14 @@ +# yapf:disable +log_config = dict( + interval=50, + hooks=[ + dict(type='TextLoggerHook', by_epoch=False), + # dict(type='TensorboardLoggerHook') + ]) +# yapf:enable +dist_params = dict(backend='nccl') +log_level = 'INFO' +load_from = None +resume_from = None +workflow = [('train', 1)] +cudnn_benchmark = True diff --git a/segmentation/configs/_base_/models/fpn_r50_lit.py b/segmentation/configs/_base_/models/fpn_r50_lit.py new file mode 100644 index 0000000..e7e403d --- /dev/null +++ b/segmentation/configs/_base_/models/fpn_r50_lit.py @@ -0,0 +1,44 @@ +# model settings +norm_cfg = dict(type='SyncBN', requires_grad=True) +model = dict( + type='EncoderDecoder', + pretrained=None, + backbone=dict( + type='LITv2', + embed_dim=96, + depths=[2, 2, 6, 2], + num_heads=[3, 6, 12, 24], + window_size=7, + mlp_ratio=4., + qkv_bias=True, + qk_scale=None, + drop_rate=0., + attn_drop_rate=0., + drop_path_rate=0.2, + ape=False, + patch_norm=True, + out_indices=(0, 1, 2, 3), + use_checkpoint=False, + alpha=0.9, + local_ws=[0, 0, 2, 1] + ), + neck=dict( + type='FPN', + in_channels=[256, 512, 1024, 2048], + out_channels=256, + num_outs=4), + decode_head=dict( + type='FPNHead', + in_channels=[256, 256, 256, 256], + in_index=[0, 1, 2, 3], + feature_strides=[4, 8, 16, 32], + channels=128, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)), + # model training and testing settings + train_cfg=dict(), + test_cfg=dict(mode='whole')) diff --git a/segmentation/configs/_base_/schedules/schedule_160k.py b/segmentation/configs/_base_/schedules/schedule_160k.py new file mode 100644 index 0000000..5260389 --- /dev/null +++ b/segmentation/configs/_base_/schedules/schedule_160k.py @@ -0,0 +1,9 @@ +# optimizer +optimizer = dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0005) +optimizer_config = dict() +# learning policy +lr_config = dict(policy='poly', power=0.9, min_lr=1e-4, by_epoch=False) +# runtime settings +runner = dict(type='IterBasedRunner', max_iters=160000) +checkpoint_config = dict(by_epoch=False, interval=16000) +evaluation = dict(interval=16000, metric='mIoU') diff --git a/segmentation/configs/_base_/schedules/schedule_20k.py b/segmentation/configs/_base_/schedules/schedule_20k.py new file mode 100644 index 0000000..bf780a1 --- /dev/null +++ b/segmentation/configs/_base_/schedules/schedule_20k.py @@ -0,0 +1,9 @@ +# optimizer +optimizer = dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0005) +optimizer_config = dict() +# learning policy +lr_config = dict(policy='poly', power=0.9, min_lr=1e-4, by_epoch=False) +# runtime settings +runner = dict(type='IterBasedRunner', max_iters=20000) +checkpoint_config = dict(by_epoch=False, interval=2000) +evaluation = dict(interval=2000, metric='mIoU') diff --git a/segmentation/configs/_base_/schedules/schedule_40k.py b/segmentation/configs/_base_/schedules/schedule_40k.py new file mode 100644 index 0000000..cdbf841 --- /dev/null +++ b/segmentation/configs/_base_/schedules/schedule_40k.py @@ -0,0 +1,9 @@ +# optimizer +optimizer = dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0005) +optimizer_config = dict() +# learning policy +lr_config = dict(policy='poly', power=0.9, min_lr=1e-4, by_epoch=False) +# runtime settings +runner = dict(type='IterBasedRunner', max_iters=40000) +checkpoint_config = dict(by_epoch=False, interval=4000) +evaluation = dict(interval=4000, metric='mIoU') diff --git a/segmentation/configs/_base_/schedules/schedule_80k.py b/segmentation/configs/_base_/schedules/schedule_80k.py new file mode 100644 index 0000000..127d941 --- /dev/null +++ b/segmentation/configs/_base_/schedules/schedule_80k.py @@ -0,0 +1,9 @@ +# optimizer +optimizer = dict(type='SGD', lr=0.01, weight_decay=0.0005) +optimizer_config = dict() +# learning policy +lr_config = dict(policy='poly', power=0.9, min_lr=1e-6, by_epoch=False) +# runtime settings +runner = dict(type='IterBasedRunner', max_iters=80000) +checkpoint_config = dict(by_epoch=False, interval=8000) +evaluation = dict(interval=8000, metric='mIoU') \ No newline at end of file diff --git a/segmentation/configs/litv2/litv2_b_fpn_r50_512x512_80k_ade20k.py b/segmentation/configs/litv2/litv2_b_fpn_r50_512x512_80k_ade20k.py new file mode 100644 index 0000000..a3c4b78 --- /dev/null +++ b/segmentation/configs/litv2/litv2_b_fpn_r50_512x512_80k_ade20k.py @@ -0,0 +1,28 @@ +_base_ = [ + '../_base_/models/fpn_r50_lit.py', '../_base_/datasets/ade20k.py', + '../_base_/default_runtime.py', '../_base_/schedules/schedule_80k.py' +] +model = dict( + type='EncoderDecoder', + backbone=dict( + type='LITv2', + embed_dim=128, + depths=[2, 2, 18, 2], + num_heads=[ 4, 8, 16, 32 ], + window_size=7, + ape=False, + drop_path_rate=0.3, + patch_norm=True, + use_checkpoint=False, + alpha=0.9, + local_ws=[0, 0, 2, 1] + ), + neck=dict( + type='FPN', + in_channels=[128, 256, 512, 1024], + out_channels=256, + num_outs=4), + decode_head=dict(num_classes=150)) + +optimizer = dict(type='AdamW', lr=0.0001, weight_decay=0.0001) +optimizer_config = dict(grad_clip=None) \ No newline at end of file diff --git a/segmentation/configs/litv2/litv2_m_fpn_r50_512x512_80k_ade20k.py b/segmentation/configs/litv2/litv2_m_fpn_r50_512x512_80k_ade20k.py new file mode 100644 index 0000000..1c9763b --- /dev/null +++ b/segmentation/configs/litv2/litv2_m_fpn_r50_512x512_80k_ade20k.py @@ -0,0 +1,28 @@ +_base_ = [ + '../_base_/models/fpn_r50_lit.py', '../_base_/datasets/ade20k.py', + '../_base_/default_runtime.py', '../_base_/schedules/schedule_80k.py' +] +model = dict( + type='EncoderDecoder', + backbone=dict( + type='LITv2', + embed_dim=96, + depths=[2, 2, 18, 2], + num_heads=[3, 6, 12, 24], + window_size=7, + ape=False, + drop_path_rate=0.2, + patch_norm=True, + use_checkpoint=False, + alpha=0.9, + local_ws=[0, 0, 2, 1] + ), + neck=dict( + type='FPN', + in_channels=[96, 192, 384, 768], + out_channels=256, + num_outs=4), + decode_head=dict(num_classes=150)) + +optimizer = dict(type='AdamW', lr=0.0001, weight_decay=0.0001) +optimizer_config = dict(grad_clip=None) \ No newline at end of file diff --git a/segmentation/configs/litv2/litv2_s_fpn_r50_512x512_80k_ade20k.py b/segmentation/configs/litv2/litv2_s_fpn_r50_512x512_80k_ade20k.py new file mode 100644 index 0000000..091d429 --- /dev/null +++ b/segmentation/configs/litv2/litv2_s_fpn_r50_512x512_80k_ade20k.py @@ -0,0 +1,28 @@ +_base_ = [ + '../_base_/models/fpn_r50_lit.py', '../_base_/datasets/ade20k.py', + '../_base_/default_runtime.py', '../_base_/schedules/schedule_80k.py' +] +model = dict( + type='EncoderDecoder', + backbone=dict( + type='LITv2', + embed_dim=96, + depths=[2, 2, 6, 2], + num_heads=[3, 6, 12, 24], + window_size=7, + ape=False, + drop_path_rate=0.2, + patch_norm=True, + use_checkpoint=False, + alpha=0.9, + local_ws=[0, 0, 2, 1] + ), + neck=dict( + type='FPN', + in_channels=[96, 192, 384, 768], + out_channels=256, + num_outs=4), + decode_head=dict(num_classes=150)) + +optimizer = dict(type='AdamW', lr=0.0001, weight_decay=0.0001) +optimizer_config = dict(grad_clip=None) \ No newline at end of file diff --git a/segmentation/mm_modules/DCN/deform_conv2d_naive.py b/segmentation/mm_modules/DCN/deform_conv2d_naive.py new file mode 100644 index 0000000..100ce74 --- /dev/null +++ b/segmentation/mm_modules/DCN/deform_conv2d_naive.py @@ -0,0 +1,93 @@ +import torch +import torch.nn as nn +from torch.nn import init +import math +import numpy as np +from torch.nn.modules.module import Module +import torch.nn.functional as F +from torch.nn.modules.utils import _pair + +class deform_conv2d_naive(Module): + def __init__(self, in_channels, out_channels, + kernel_size, stride, padding, dilation=1, groups=1, deformable_groups=1, bias=True): + super(deform_conv2d_naive, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = _pair(kernel_size) + self.stride = _pair(stride) + self.padding = _pair(padding) + self.dilation = _pair(dilation) + self.groups = groups + self.deformable_groups = deformable_groups + self.use_bias = bias + + self.weight = nn.Parameter(torch.Tensor( + out_channels, in_channels//groups, *self.kernel_size)) + self.bias = nn.Parameter(torch.Tensor(out_channels)) + self.reset_parameters() + if not self.use_bias: + self.bias.requires_grad = False + self.bias.data.zero_() + + def reset_parameters(self): + n = self.in_channels + init.kaiming_uniform_(self.weight, a=math.sqrt(5)) + if self.bias is not None: + fan_in, _ = init._calculate_fan_in_and_fan_out(self.weight) + bound = 1 / math.sqrt(fan_in) + init.uniform_(self.bias, -bound, bound) + + def forward(self, input, offset): + N = input.size(0) + in_channels = self.in_channels + out_channels = self.out_channels + in_h = input.size(2) + in_w = input.size(3) + out_h = offset.size(2) + out_w = offset.size(3) + kernel_h = self.kernel_size[0] + kernel_w = self.kernel_size[1] + # [1, kernel_h * kernel_w, out_h, out_w, 2] + mesh = self.compute_mesh_grid(in_h, in_w).cuda(input.get_device()) + offset = offset.view(N, self.deformable_groups, kernel_h, kernel_w, 2, out_h, out_w) + # [N * dg * kernel_h * kernel_w, out_h, out_w, 2] + offset = offset.permute(0, 1, 2, 3, 5, 6, 4).contiguous().view(N * self.deformable_groups * kernel_h * kernel_w, out_h, out_w, 2) + offset_x_normalize = (offset[:, :, :, 1]) / ((in_w - 1) * 1.0 / 2) + offset_y_normalize = (offset[:, :, :, 0]) / ((in_h - 1) * 1.0 / 2) + # [N * dg * kernel_h * kernel_w, out_h, out_w, 2] + offset = torch.cat([offset_x_normalize[..., None], offset_y_normalize[..., None]], dim=3) + # [N * dg * kernel_h * kernel_w, out_h, out_w, 2] + grid = mesh.expand(N * self.deformable_groups, -1, -1, -1, -1).contiguous().view(-1, out_h, out_w, 2) + offset + # [N * kernel_h * kernel_w * dg, in_channels/dg, in_h, in_w] + input = input[:, None, ...].expand(-1, kernel_h * kernel_w, -1, -1, -1).contiguous().view( + N * kernel_h * kernel_w * self.deformable_groups, in_channels // self.deformable_groups, in_h, in_w) + sampled_feat = F.grid_sample(input, grid).view(N, kernel_h * kernel_w, in_channels, out_h, out_w).permute(2, 1, 0, 3, 4).contiguous().view(in_channels * kernel_h * kernel_w, -1) + output_feat = torch.matmul(self.weight.view(self.weight.size(0), -1), sampled_feat).view(out_channels, N, out_h, out_w).permute(1,0,2,3) + return output_feat + + def compute_mesh_grid(self, in_h, in_w): + kernel_h, kernel_w = self.kernel_size + stride_h, stride_w = self.stride + dilation_h, dilation_w = self.dilation + padding_h, padding_w = self.padding + out_h = (in_h + 2 * padding_h - (dilation_h * (kernel_h - 1) + 1)) // stride_h + 1 + out_w = (in_w + 2 * padding_w - (dilation_w * (kernel_w - 1) + 1)) // stride_w + 1 + # [out_h, out_w] + mesh_y, mesh_x = torch.meshgrid(torch.arange(out_h), torch.arange(out_w)) + mesh_y = mesh_y * stride_h - padding_h + mesh_x = mesh_x * stride_w - padding_w + # [1, out_h, out_w] + mesh_y = mesh_y.unsqueeze(0).float() + mesh_x = mesh_x.unsqueeze(0).float() + # [kernel_h, kernel_w] + kernel_offset_y, kernel_offset_x = torch.meshgrid(torch.arange(kernel_h), torch.arange(kernel_w)) + # [kernel_h * kernel_w, 1, 1] + kernel_offset_y = kernel_offset_y.float().view(kernel_h * kernel_w, 1, 1) * dilation_h + kernel_offset_x = kernel_offset_x.float().view(kernel_h * kernel_w, 1, 1) * dilation_w + # [kernel_h * kernel_w, out_h, out_w] + mesh_y = mesh_y + kernel_offset_y + mesh_x = mesh_x + kernel_offset_x + mesh_y = (mesh_y - (in_h - 1) / 2.) / ((in_h - 1) / 2.) + mesh_x = (mesh_x - (in_w - 1) / 2.) / ((in_w - 1) / 2.) + mesh = torch.cat([mesh_x[None, ..., None], mesh_y[None, ..., None]], dim=4) + return mesh diff --git a/segmentation/mm_modules/DCN/functions/__init__.py b/segmentation/mm_modules/DCN/functions/__init__.py new file mode 100644 index 0000000..a80bf4d --- /dev/null +++ b/segmentation/mm_modules/DCN/functions/__init__.py @@ -0,0 +1,2 @@ +from .deform_conv2d_func import DeformConv2dFunction +from .modulated_deform_conv2d_func import ModulatedDeformConv2dFunction diff --git a/segmentation/mm_modules/DCN/functions/deform_conv2d_func.py b/segmentation/mm_modules/DCN/functions/deform_conv2d_func.py new file mode 100644 index 0000000..d4dbf08 --- /dev/null +++ b/segmentation/mm_modules/DCN/functions/deform_conv2d_func.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import math +import torch +from torch import nn +from torch.autograd import Function +from torch.nn.modules.utils import _pair +from torch.autograd.function import once_differentiable +# try: +# from apex import amp +# except ImportError: +# raise ImportError("Please install apex from https://www.github.com/nvidia/apex to run this example.") +from torch.cuda import amp +import DCN + +class DeformConv2dFunction(Function): + @staticmethod + @amp.custom_fwd(cast_inputs=torch.float32) + # @amp.float_function + def forward(ctx, input, offset, weight, bias, + stride, padding, dilation, group, deformable_groups, im2col_step): + ctx.stride = _pair(stride) + ctx.padding = _pair(padding) + ctx.dilation = _pair(dilation) + ctx.kernel_size = _pair(weight.shape[2:4]) + ctx.group = group + ctx.deformable_groups = deformable_groups + ctx.im2col_step = im2col_step + output = DCN.deform_conv2d_forward(input, weight, bias, + offset, + ctx.kernel_size[0], ctx.kernel_size[1], + ctx.stride[0], ctx.stride[1], + ctx.padding[0], ctx.padding[1], + ctx.dilation[0], ctx.dilation[1], + ctx.group, + ctx.deformable_groups, + ctx.im2col_step) + ctx.save_for_backward(input, offset, weight, bias) + return output + + @staticmethod + @once_differentiable + # @amp.float_function + @amp.custom_bwd + def backward(ctx, grad_output): + input, offset, weight, bias = ctx.saved_tensors + grad_input, grad_offset, grad_weight, grad_bias = \ + DCN.deform_conv2d_backward(input, weight, + bias, + offset, + grad_output, + ctx.kernel_size[0], ctx.kernel_size[1], + ctx.stride[0], ctx.stride[1], + ctx.padding[0], ctx.padding[1], + ctx.dilation[0], ctx.dilation[1], + ctx.group, + ctx.deformable_groups, + ctx.im2col_step) + + return grad_input, grad_offset, grad_weight, grad_bias,\ + None, None, None, None, None, None diff --git a/segmentation/mm_modules/DCN/functions/modulated_deform_conv2d_func.py b/segmentation/mm_modules/DCN/functions/modulated_deform_conv2d_func.py new file mode 100644 index 0000000..be0dfbc --- /dev/null +++ b/segmentation/mm_modules/DCN/functions/modulated_deform_conv2d_func.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import math +import torch +from torch import nn +from torch.autograd import Function +from torch.nn.modules.utils import _pair +from torch.autograd.function import once_differentiable +# from torch.cuda import amp +import DCN +# try: +# from apex import amp +# except ImportError: +# raise ImportError("Please install apex from https://www.github.com/nvidia/apex to run this example.") +from torch.cuda import amp + +class ModulatedDeformConv2dFunction(Function): + @staticmethod + @amp.custom_fwd(cast_inputs=torch.float32) + # @amp.float_function + def forward(ctx, input, offset, mask, weight, bias, + stride, padding, dilation, groups, deformable_groups, im2col_step): + ctx.stride = _pair(stride) + ctx.padding = _pair(padding) + ctx.dilation = _pair(dilation) + ctx.kernel_size = _pair(weight.shape[2:4]) + ctx.groups = groups + ctx.deformable_groups = deformable_groups + ctx.im2col_step = im2col_step + output = DCN.modulated_deform_conv2d_forward(input, weight, bias, + offset, mask, + ctx.kernel_size[0], ctx.kernel_size[1], + ctx.stride[0], ctx.stride[1], + ctx.padding[0], ctx.padding[1], + ctx.dilation[0], ctx.dilation[1], + ctx.groups, + ctx.deformable_groups, + ctx.im2col_step) + ctx.save_for_backward(input, offset, mask, weight, bias) + return output + + @staticmethod + @once_differentiable + @amp.custom_bwd + # @amp.float_function + def backward(ctx, grad_output): + input, offset, mask, weight, bias = ctx.saved_tensors + grad_input, grad_offset, grad_mask, grad_weight, grad_bias = \ + DCN.modulated_deform_conv2d_backward(input, weight, + bias, + offset, mask, + grad_output, + ctx.kernel_size[0], ctx.kernel_size[1], + ctx.stride[0], ctx.stride[1], + ctx.padding[0], ctx.padding[1], + ctx.dilation[0], ctx.dilation[1], + ctx.groups, + ctx.deformable_groups, + ctx.im2col_step) + + return grad_input, grad_offset, grad_mask, grad_weight, grad_bias,\ + None, None, None, None, None, None diff --git a/segmentation/mm_modules/DCN/make.sh b/segmentation/mm_modules/DCN/make.sh new file mode 100644 index 0000000..b1cf55a --- /dev/null +++ b/segmentation/mm_modules/DCN/make.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# SBATCH --job-name=tiny-1-pool-in-pre + +#SBATCH --account=dl65 +#SBATCH --partition=m3g + +#SBATCH -n 1 +#SBATCH -c 8 +#SBATCH --gres=gpu:V100:1 +#SBATCH --mem=16GB +#SBATCH --time=1:00:00 + +#SBATCH --mail-user=zizhengpan98@gmail.com +#SBATCH --mail-type=END +#SBATCH --mail-type=FAIL + + +# Command to run a gpu job +# For example: +module load anaconda/2019.03-Python3.7-gcc5 +module load gcc/5.4.0 +module load cuda/10.1 +module load cudnn/7.6.5-cuda10.1 +export PROJECT=dl65 +export CONDA_ENVS_PATH=/projects/$PROJECT/$USER/conda_envs +export CONDA_PKGS_DIRS=/projects/$PROJECT/$USER/conda_pkgs +source activate /projects/$PROJECT/$USER/conda_envs/defconv +which python + + +nvidia-smi +nvcc -V +python setup.py build install diff --git a/segmentation/mm_modules/DCN/make_dsai.sh b/segmentation/mm_modules/DCN/make_dsai.sh new file mode 100644 index 0000000..db22bbd --- /dev/null +++ b/segmentation/mm_modules/DCN/make_dsai.sh @@ -0,0 +1,22 @@ +#!/bin/bash +#SBATCH --account=hhe +#SBATCH --time=48:00:00 +#SBATCH --ntasks=1 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=20 +#SBATCH --gres=gpu:1 +#SBATCH --mem=64GB +#SBATCH --exclude=node04 + +#SBATCH --mail-user=zizhengpan98@gmail.com +#SBATCH --mail-type=END +#SBATCH --mail-type=FAIL + +# Command to run a gpu job +# For example: +module load cuda-11.2.0-gcc-10.2.0-gsjevs3 +source activate torch171 + +nvidia-smi +nvcc -V +python setup.py build install diff --git a/segmentation/mm_modules/DCN/modules/__init__.py b/segmentation/mm_modules/DCN/modules/__init__.py new file mode 100644 index 0000000..552cca0 --- /dev/null +++ b/segmentation/mm_modules/DCN/modules/__init__.py @@ -0,0 +1,2 @@ +from .deform_conv2d import DeformConv2d, _DeformConv2d, DeformConv2dPack, DeformConv2dPackMore +from .modulated_deform_conv2d import ModulatedDeformConv2d, _ModulatedDeformConv2d, ModulatedDeformConv2dPack diff --git a/segmentation/mm_modules/DCN/modules/deform_conv2d.py b/segmentation/mm_modules/DCN/modules/deform_conv2d.py new file mode 100644 index 0000000..5d9698b --- /dev/null +++ b/segmentation/mm_modules/DCN/modules/deform_conv2d.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import torch +import math +from torch import nn +from torch.nn import init +from torch.nn.modules.utils import _pair + +from ..functions.deform_conv2d_func import DeformConv2dFunction + +class DeformConv2d(nn.Module): + + def __init__(self, in_channels, out_channels, + kernel_size, stride, padding, dilation=1, groups=1, deformable_groups=1, im2col_step=32, bias=True): + super(DeformConv2d, self).__init__() + + if in_channels % groups != 0: + raise ValueError('in_channels {} must be divisible by groups {}'.format(in_channels, groups)) + if out_channels % groups != 0: + raise ValueError('out_channels {} must be divisible by groups {}'.format(out_channels, groups)) + + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = _pair(kernel_size) + self.stride = _pair(stride) + self.padding = _pair(padding) + self.dilation = _pair(dilation) + self.groups = groups + self.deformable_groups = deformable_groups + self.im2col_step = im2col_step + self.use_bias = bias + + self.weight = nn.Parameter(torch.Tensor( + out_channels, in_channels//groups, *self.kernel_size)) + self.bias = nn.Parameter(torch.Tensor(out_channels)) + self.reset_parameters() + if not self.use_bias: + self.bias.requires_grad = False + self.bias.data.zero_() + + def reset_parameters(self): + n = self.in_channels + init.kaiming_uniform_(self.weight, a=math.sqrt(5)) + if self.bias is not None: + fan_in, _ = init._calculate_fan_in_and_fan_out(self.weight) + bound = 1 / math.sqrt(fan_in) + init.uniform_(self.bias, -bound, bound) + + def forward(self, input, offset): + assert 2 * self.deformable_groups * self.kernel_size[0] * self.kernel_size[1] == \ + offset.shape[1] + return DeformConv2dFunction.apply(input, offset, + self.weight, + self.bias, + self.stride, + self.padding, + self.dilation, + self.groups, + self.deformable_groups, + self.im2col_step) + +_DeformConv2d = DeformConv2dFunction.apply + +class DeformConv2dPack(DeformConv2d): + + def __init__(self, in_channels, out_channels, + kernel_size, stride, padding, + dilation=1, groups=1, deformable_groups=1, im2col_step=32, bias=True, lr_mult=0.1): + super(DeformConv2dPack, self).__init__(in_channels, out_channels, + kernel_size, stride, padding, dilation, groups, deformable_groups, im2col_step, bias) + + out_channels = self.deformable_groups * 2 * self.kernel_size[0] * self.kernel_size[1] + self.conv_offset = nn.Conv2d(self.in_channels, + out_channels, + kernel_size=self.kernel_size, + stride=self.stride, + padding=self.padding, + bias=True) + self.conv_offset.lr_mult = lr_mult + self.conv_offset.inited = True + self.init_offset() + + def init_offset(self): + self.conv_offset.weight.data.zero_() + self.conv_offset.bias.data.zero_() + + def forward(self, input, return_offset=False): + offset = self.conv_offset(input) + bs = input.size()[0] + im2col_step = bs // 2 if bs > 1 else 1 + # return DeformConv2dFunction.apply(input, offset, + # self.weight, + # self.bias, + # self.stride, + # self.padding, + # self.dilation, + # self.groups, + # self.deformable_groups, + # im2col_step) + out = DeformConv2dFunction.apply(input, offset, + self.weight, + self.bias, + self.stride, + self.padding, + self.dilation, + self.groups, + self.deformable_groups, + im2col_step) + if return_offset: + return out, offset + return out, None + + +class DeformConv2dPackMore(DeformConv2d): + + def __init__(self, in_channels, out_channels, + kernel_size, stride, padding, + dilation=1, groups=1, deformable_groups=1, im2col_step=64, bias=True, lr_mult=0.1): + super(DeformConv2dPackMore, self).__init__(in_channels, out_channels, + kernel_size, stride, padding, dilation, groups, deformable_groups, im2col_step, bias) + + out_channels = self.deformable_groups * 2 * self.kernel_size[0] * self.kernel_size[1] + self.conv_offset = nn.Sequential( + nn.Conv2d(self.in_channels, self.in_channels//4, kernel_size=1, bias=False), + nn.BatchNorm2d(self.in_channels//4), + nn.ReLU(inplace=True), + nn.Conv2d(self.in_channels//4, out_channels, kernel_size=self.kernel_size, stride=self.stride, padding=self.padding, bias=True) + ) + self.conv_offset[-1].lr_mult = lr_mult + self.conv_offset[-1].inited = True + self.init_offset() + + def init_offset(self): + self.conv_offset[-1].weight.data.zero_() + self.conv_offset[-1].bias.data.zero_() + + def forward(self, input): + offset = self.conv_offset(input) + bs = input.size()[0] + im2col_step = bs // 2 + return DeformConv2dFunction.apply(input, offset, + self.weight, + self.bias, + self.stride, + self.padding, + self.dilation, + self.groups, + self.deformable_groups, + im2col_step) diff --git a/segmentation/mm_modules/DCN/modules/modulated_deform_conv2d.py b/segmentation/mm_modules/DCN/modules/modulated_deform_conv2d.py new file mode 100644 index 0000000..2052051 --- /dev/null +++ b/segmentation/mm_modules/DCN/modules/modulated_deform_conv2d.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import torch +import math +from torch import nn +from torch.nn import init +from torch.nn.modules.utils import _pair + +from ..functions.modulated_deform_conv2d_func import ModulatedDeformConv2dFunction + +class ModulatedDeformConv2d(nn.Module): + + def __init__(self, in_channels, out_channels, + kernel_size, stride, padding, dilation=1, groups=1, deformable_groups=1, im2col_step=64, bias=True): + super(ModulatedDeformConv2d, self).__init__() + + if in_channels % groups != 0: + raise ValueError('in_channels {} must be divisible by groups {}'.format(in_channels, groups)) + if out_channels % groups != 0: + raise ValueError('out_channels {} must be divisible by groups {}'.format(out_channels, groups)) + + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = _pair(kernel_size) + self.stride = _pair(stride) + self.padding = _pair(padding) + self.dilation = _pair(dilation) + self.groups = groups + self.deformable_groups = deformable_groups + self.im2col_step = im2col_step + self.use_bias = bias + + self.weight = nn.Parameter(torch.Tensor( + out_channels, in_channels//groups, *self.kernel_size)) + self.bias = nn.Parameter(torch.Tensor(out_channels)) + self.reset_parameters() + if not self.use_bias: + self.bias.requires_grad = False + + def reset_parameters(self): + n = self.in_channels + init.kaiming_uniform_(self.weight, a=math.sqrt(5)) + if self.bias is not None: + fan_in, _ = init._calculate_fan_in_and_fan_out(self.weight) + bound = 1 / math.sqrt(fan_in) + init.uniform_(self.bias, -bound, bound) + + def forward(self, input, offset, mask): + assert 2 * self.deformable_groups * self.kernel_size[0] * self.kernel_size[1] == \ + offset.shape[1] + assert self.deformable_groups * self.kernel_size[0] * self.kernel_size[1] == \ + mask.shape[1] + return ModulatedDeformConv2dFunction.apply(input, offset, mask, + self.weight, + self.bias, + self.stride, + self.padding, + self.dilation, + self.groups, + self.deformable_groups, + self.im2col_step) + +_ModulatedDeformConv2d = ModulatedDeformConv2dFunction.apply + +class ModulatedDeformConv2dPack(ModulatedDeformConv2d): + + def __init__(self, in_channels, out_channels, + kernel_size, stride, padding, + dilation=1, groups=1, deformable_groups=1, im2col_step=64, bias=True, lr_mult=0.1): + super(ModulatedDeformConv2dPack, self).__init__(in_channels, out_channels, + kernel_size, stride, padding, dilation, groups, deformable_groups, im2col_step, bias) + + out_channels = self.deformable_groups * 3 * self.kernel_size[0] * self.kernel_size[1] + self.conv_offset_mask = nn.Conv2d(self.in_channels, + out_channels, + kernel_size=self.kernel_size, + stride=self.stride, + padding=self.padding, + bias=True) + self.conv_offset_mask.lr_mult = lr_mult + self.conv_offset_mask.inited = True + self.init_offset() + + def init_offset(self): + self.conv_offset_mask.weight.data.zero_() + self.conv_offset_mask.bias.data.zero_() + + def forward(self, input, return_offset=False): + out = self.conv_offset_mask(input) + o1, o2, mask = torch.chunk(out, 3, dim=1) + offset = torch.cat((o1, o2), dim=1) + mask = torch.sigmoid(mask) + + bs = input.size()[0] + im2col_step = bs // 2 + + out = ModulatedDeformConv2dFunction.apply(input, offset, mask, + self.weight, + self.bias, + self.stride, + self.padding, + self.dilation, + self.groups, + self.deformable_groups, + im2col_step) + if return_offset: + return out, offset + return out, None diff --git a/segmentation/mm_modules/DCN/setup.py b/segmentation/mm_modules/DCN/setup.py new file mode 100644 index 0000000..55cf53d --- /dev/null +++ b/segmentation/mm_modules/DCN/setup.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python + +import os +import glob + +import torch + +from torch.utils.cpp_extension import CUDA_HOME +from torch.utils.cpp_extension import CppExtension +from torch.utils.cpp_extension import CUDAExtension + +from setuptools import find_packages +from setuptools import setup + +requirements = ["torch", "torchvision"] + +def get_extensions(): + this_dir = os.path.dirname(os.path.abspath(__file__)) + extensions_dir = os.path.join(this_dir, "src") + + main_file = glob.glob(os.path.join(extensions_dir, "*.cpp")) + source_cpu = glob.glob(os.path.join(extensions_dir, "cpu", "*.cpp")) + source_cuda = glob.glob(os.path.join(extensions_dir, "cuda", "*.cu")) + + sources = main_file + source_cpu + extension = CppExtension + extra_compile_args = {"cxx": []} + define_macros = [] + + if torch.cuda.is_available() and CUDA_HOME is not None: + extension = CUDAExtension + sources += source_cuda + define_macros += [("WITH_CUDA", None)] + extra_compile_args["nvcc"] = [ + "-DCUDA_HAS_FP16=1", + "-D__CUDA_NO_HALF_OPERATORS__", + "-D__CUDA_NO_HALF_CONVERSIONS__", + "-D__CUDA_NO_HALF2_OPERATORS__", + ] + else: + raise NotImplementedError('Cuda is not availabel') + + sources = [os.path.join(extensions_dir, s) for s in sources] + include_dirs = [extensions_dir] + ext_modules = [ + extension( + "DCN", + sources, + include_dirs=include_dirs, + define_macros=define_macros, + extra_compile_args=extra_compile_args, + ) + ] + return ext_modules + +setup( + name="DCN", + version="1.0", + description="deformable convolutional networks", + packages=find_packages(exclude=("configs", "tests",)), + ext_modules=get_extensions(), + cmdclass={"build_ext": torch.utils.cpp_extension.BuildExtension}, +) diff --git a/segmentation/mm_modules/DCN/src/cpu/deform_conv2d_cpu.cpp b/segmentation/mm_modules/DCN/src/cpu/deform_conv2d_cpu.cpp new file mode 100644 index 0000000..64a67bb --- /dev/null +++ b/segmentation/mm_modules/DCN/src/cpu/deform_conv2d_cpu.cpp @@ -0,0 +1,47 @@ +#include + +#include +#include + + +at::Tensor +deform_conv2d_cpu_forward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + AT_ERROR("Not implement on cpu"); +} + +std::vector +deform_conv2d_cpu_backward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &grad_output, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + AT_ERROR("Not implement on cpu"); +} + diff --git a/segmentation/mm_modules/DCN/src/cpu/deform_conv2d_cpu.h b/segmentation/mm_modules/DCN/src/cpu/deform_conv2d_cpu.h new file mode 100644 index 0000000..585d3d8 --- /dev/null +++ b/segmentation/mm_modules/DCN/src/cpu/deform_conv2d_cpu.h @@ -0,0 +1,39 @@ +#pragma once +#include + +at::Tensor +deform_conv2d_cpu_forward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step); + +std::vector +deform_conv2d_cpu_backward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &grad_output, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step); + + diff --git a/segmentation/mm_modules/DCN/src/cpu/modulated_deform_conv2d_cpu.cpp b/segmentation/mm_modules/DCN/src/cpu/modulated_deform_conv2d_cpu.cpp new file mode 100644 index 0000000..b712a19 --- /dev/null +++ b/segmentation/mm_modules/DCN/src/cpu/modulated_deform_conv2d_cpu.cpp @@ -0,0 +1,49 @@ +#include + +#include +#include + + +at::Tensor +modulated_deform_conv2d_cpu_forward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &mask, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + AT_ERROR("Not implement on cpu"); +} + +std::vector +modulated_deform_conv2d_cpu_backward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &mask, + const at::Tensor &grad_output, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + AT_ERROR("Not implement on cpu"); +} + diff --git a/segmentation/mm_modules/DCN/src/cpu/modulated_deform_conv2d_cpu.h b/segmentation/mm_modules/DCN/src/cpu/modulated_deform_conv2d_cpu.h new file mode 100644 index 0000000..8f54d0e --- /dev/null +++ b/segmentation/mm_modules/DCN/src/cpu/modulated_deform_conv2d_cpu.h @@ -0,0 +1,41 @@ +#pragma once +#include + +at::Tensor +modulated_deform_conv2d_cpu_forward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &mask, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step); + +std::vector +modulated_deform_conv2d_cpu_backward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &mask, + const at::Tensor &grad_output, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step); + + diff --git a/segmentation/mm_modules/DCN/src/cuda/deform_2d_im2col_cuda.cuh b/segmentation/mm_modules/DCN/src/cuda/deform_2d_im2col_cuda.cuh new file mode 100644 index 0000000..266f3a2 --- /dev/null +++ b/segmentation/mm_modules/DCN/src/cuda/deform_2d_im2col_cuda.cuh @@ -0,0 +1,391 @@ +#include +#include +#include + +#include +#include + +// #include +#include +// #include + +#define CUDA_KERNEL_LOOP(i, n) \ + for (int i = blockIdx.x * blockDim.x + threadIdx.x; \ + i < (n); \ + i += blockDim.x * gridDim.x) + +const int CUDA_NUM_THREADS = 1024; +inline int GET_BLOCKS(const int N) +{ + return (N + CUDA_NUM_THREADS - 1) / CUDA_NUM_THREADS; +} + +template +__device__ scalar_t dmcn_2d_im2col_bilinear(const scalar_t *bottom_data, const int data_width, + const int height, const int width, scalar_t h, scalar_t w) +{ + int h_low = floor(h); + int w_low = floor(w); + int h_high = h_low + 1; + int w_high = w_low + 1; + + scalar_t lh = h - h_low; + scalar_t lw = w - w_low; + scalar_t hh = 1 - lh, hw = 1 - lw; + + scalar_t v1 = 0; + if (h_low >= 0 && w_low >= 0) + v1 = bottom_data[h_low * data_width + w_low]; + scalar_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) + v2 = bottom_data[h_low * data_width + w_high]; + scalar_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) + v3 = bottom_data[h_high * data_width + w_low]; + scalar_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) + v4 = bottom_data[h_high * data_width + w_high]; + + scalar_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + + scalar_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + return val; +} + +template +__device__ scalar_t dmcn_2d_get_gradient_weight(scalar_t argmax_h, scalar_t argmax_w, + const int h, const int w, const int height, const int width) +{ + if (argmax_h <= -1 || argmax_h >= height || argmax_w <= -1 || argmax_w >= width) + { + //empty + return 0; + } + + int argmax_h_low = floor(argmax_h); + int argmax_w_low = floor(argmax_w); + int argmax_h_high = argmax_h_low + 1; + int argmax_w_high = argmax_w_low + 1; + + scalar_t weight = 0; + if (h == argmax_h_low && w == argmax_w_low) + weight = (h + 1 - argmax_h) * (w + 1 - argmax_w); + if (h == argmax_h_low && w == argmax_w_high) + weight = (h + 1 - argmax_h) * (argmax_w + 1 - w); + if (h == argmax_h_high && w == argmax_w_low) + weight = (argmax_h + 1 - h) * (w + 1 - argmax_w); + if (h == argmax_h_high && w == argmax_w_high) + weight = (argmax_h + 1 - h) * (argmax_w + 1 - w); + return weight; +} + +template +__device__ scalar_t dmcn_2d_get_coordinate_weight(scalar_t argmax_h, scalar_t argmax_w, + const int height, const int width, const scalar_t *im_data, + const int data_width, const int bp_dir) +{ + if (argmax_h <= -1 || argmax_h >= height || argmax_w <= -1 || argmax_w >= width) + { + //empty + return 0; + } + + int argmax_h_low = floor(argmax_h); + int argmax_w_low = floor(argmax_w); + int argmax_h_high = argmax_h_low + 1; + int argmax_w_high = argmax_w_low + 1; + + scalar_t weight = 0; + + if (bp_dir == 0) + { + if (argmax_h_low >= 0 && argmax_w_low >= 0) + weight += -1 * (argmax_w_low + 1 - argmax_w) * im_data[argmax_h_low * data_width + argmax_w_low]; + if (argmax_h_low >= 0 && argmax_w_high <= width - 1) + weight += -1 * (argmax_w - argmax_w_low) * im_data[argmax_h_low * data_width + argmax_w_high]; + if (argmax_h_high <= height - 1 && argmax_w_low >= 0) + weight += (argmax_w_low + 1 - argmax_w) * im_data[argmax_h_high * data_width + argmax_w_low]; + if (argmax_h_high <= height - 1 && argmax_w_high <= width - 1) + weight += (argmax_w - argmax_w_low) * im_data[argmax_h_high * data_width + argmax_w_high]; + } + else if (bp_dir == 1) + { + if (argmax_h_low >= 0 && argmax_w_low >= 0) + weight += -1 * (argmax_h_low + 1 - argmax_h) * im_data[argmax_h_low * data_width + argmax_w_low]; + if (argmax_h_low >= 0 && argmax_w_high <= width - 1) + weight += (argmax_h_low + 1 - argmax_h) * im_data[argmax_h_low * data_width + argmax_w_high]; + if (argmax_h_high <= height - 1 && argmax_w_low >= 0) + weight += -1 * (argmax_h - argmax_h_low) * im_data[argmax_h_high * data_width + argmax_w_low]; + if (argmax_h_high <= height - 1 && argmax_w_high <= width - 1) + weight += (argmax_h - argmax_h_low) * im_data[argmax_h_high * data_width + argmax_w_high]; + } + + return weight; +} + +template +__global__ void deformable_2d_im2col_gpu_kernel(const int n, + const scalar_t *data_im, const scalar_t *data_offset, + const int height, const int width, const int kernel_h, + const int kernel_w, + const int pad_h, const int pad_w, + const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, const int num_channels, + const int deformable_group, + const int height_col, const int width_col, + scalar_t *data_col) +{ + // launch channels * batch_size * height_col * width_col cores + CUDA_KERNEL_LOOP(index, n) + { + // NOTE(CharlesShang): different from Dai Jifeng's MXNet implementation, col_buffer is of shape (c*kw*kh, N, oh, ow) + // here columns is of shape (N, c*kw*kh, oh * ow), need to adapt axis + // NOTE(Jiarui XU): different from CharlesShang's implementation, col_buffer is of shape (N, c*kw*kh, oh * ow) + // here columns is of shape (c*kw*kh, N, oh, ow), need to adapt axis + + // index index of output matrix + const int w_col = index % width_col; + const int h_col = (index / width_col) % height_col; + const int b_col = (index / width_col / height_col) % batch_size; + const int c_im = (index / width_col / height_col) / batch_size; + const int c_col = c_im * kernel_h * kernel_w; + + // compute deformable group index + const int deformable_group_index = c_im / channel_per_deformable_group; + + const int h_in = h_col * stride_h - pad_h; + const int w_in = w_col * stride_w - pad_w; + + scalar_t *data_col_ptr = data_col + ((c_col * batch_size + b_col) * height_col + h_col) * width_col + w_col; + // const scalar_t* data_im_ptr = data_im + ((b_col * num_channels + c_im) * height + h_in) * width + w_in; + const scalar_t *data_im_ptr = data_im + (b_col * num_channels + c_im) * height * width; + const scalar_t *data_offset_ptr = data_offset + (b_col * deformable_group + deformable_group_index) * 2 * kernel_h * kernel_w * height_col * width_col; + + for (int i = 0; i < kernel_h; ++i) + { + for (int j = 0; j < kernel_w; ++j) + { + const int data_offset_h_ptr = ((2 * (i * kernel_w + j)) * height_col + h_col) * width_col + w_col; + const int data_offset_w_ptr = ((2 * (i * kernel_w + j) + 1) * height_col + h_col) * width_col + w_col; + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + scalar_t val = static_cast(0); + const scalar_t h_im = h_in + i * dilation_h + offset_h; + const scalar_t w_im = w_in + j * dilation_w + offset_w; + if (h_im > -1 && w_im > -1 && h_im < height && w_im < width) + { + //const scalar_t map_h = i * dilation_h + offset_h; + //const scalar_t map_w = j * dilation_w + offset_w; + //const int cur_height = height - h_in; + //const int cur_width = width - w_in; + //val = dmcn_im2col_bilinear(data_im_ptr, width, cur_height, cur_width, map_h, map_w); + val = dmcn_2d_im2col_bilinear(data_im_ptr, width, height, width, h_im, w_im); + } + *data_col_ptr = val; + data_col_ptr += batch_size * height_col * width_col; + } + } + } +} + +template +__global__ void deformable_2d_col2im_gpu_kernel(const int n, + const scalar_t *data_col, const scalar_t *data_offset, + const int channels, const int height, const int width, + const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, + const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, const int deformable_group, + const int height_col, const int width_col, + scalar_t *grad_im) +{ + CUDA_KERNEL_LOOP(index, n) + { + const int j = (index / width_col / height_col / batch_size) % kernel_w; + const int i = (index / width_col / height_col / batch_size / kernel_w) % kernel_h; + const int c = index / width_col / height_col / batch_size / kernel_w / kernel_h; + // compute the start and end of the output + + const int deformable_group_index = c / channel_per_deformable_group; + + int w_out = index % width_col; + int h_out = (index / width_col) % height_col; + int b = (index / width_col / height_col) % batch_size; + int w_in = w_out * stride_w - pad_w; + int h_in = h_out * stride_h - pad_h; + + const scalar_t *data_offset_ptr = data_offset + (b * deformable_group + deformable_group_index) * 2 * kernel_h * kernel_w * height_col * width_col; + const int data_offset_h_ptr = ((2 * (i * kernel_w + j)) * height_col + h_out) * width_col + w_out; + const int data_offset_w_ptr = ((2 * (i * kernel_w + j) + 1) * height_col + h_out) * width_col + w_out; + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + const scalar_t cur_inv_h_data = h_in + i * dilation_h + offset_h; + const scalar_t cur_inv_w_data = w_in + j * dilation_w + offset_w; + + const scalar_t cur_top_grad = data_col[index]; + const int cur_h = (int)cur_inv_h_data; + const int cur_w = (int)cur_inv_w_data; + for (int dy = -2; dy <= 2; dy++) + { + for (int dx = -2; dx <= 2; dx++) + { + if (cur_h + dy >= 0 && cur_h + dy < height && + cur_w + dx >= 0 && cur_w + dx < width && + abs(cur_inv_h_data - (cur_h + dy)) < 1 && + abs(cur_inv_w_data - (cur_w + dx)) < 1) + { + int cur_bottom_grad_pos = ((b * channels + c) * height + cur_h + dy) * width + cur_w + dx; + scalar_t weight = dmcn_2d_get_gradient_weight(cur_inv_h_data, cur_inv_w_data, cur_h + dy, cur_w + dx, height, width); + atomicAdd(grad_im + cur_bottom_grad_pos, weight * cur_top_grad); + } + } + } + } +} + +template +__global__ void deformable_2d_col2im_coord_gpu_kernel(const int n, + const scalar_t *data_col, const scalar_t *data_im, + const scalar_t *data_offset, + const int channels, const int height, const int width, + const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, + const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, const int offset_channels, + const int deformable_group, + const int height_col, const int width_col, + scalar_t *grad_offset) +{ + CUDA_KERNEL_LOOP(index, n) + { + scalar_t val = 0; + int w = index % width_col; + int h = (index / width_col) % height_col; + int c = (index / width_col / height_col) % offset_channels; + int b = (index / width_col / height_col) / offset_channels; + // compute the start and end of the output + + const int deformable_group_index = c / (2 * kernel_h * kernel_w); + const int col_step = kernel_h * kernel_w; + int cnt = 0; + const scalar_t *data_col_ptr = data_col + deformable_group_index * channel_per_deformable_group * batch_size * width_col * height_col; + const scalar_t *data_im_ptr = data_im + (b * deformable_group + deformable_group_index) * channel_per_deformable_group / kernel_h / kernel_w * height * width; + const scalar_t *data_offset_ptr = data_offset + (b * deformable_group + deformable_group_index) * 2 * kernel_h * kernel_w * height_col * width_col; + + const int offset_c = c - deformable_group_index * 2 * kernel_h * kernel_w; + + for (int col_c = (offset_c / 2); col_c < channel_per_deformable_group; col_c += col_step) + { + const int col_pos = (((col_c * batch_size + b) * height_col) + h) * width_col + w; + const int bp_dir = offset_c % 2; + + int j = (col_pos / width_col / height_col / batch_size) % kernel_w; + int i = (col_pos / width_col / height_col / batch_size / kernel_w) % kernel_h; + int w_out = col_pos % width_col; + int h_out = (col_pos / width_col) % height_col; + int w_in = w_out * stride_w - pad_w; + int h_in = h_out * stride_h - pad_h; + const int data_offset_h_ptr = (((2 * (i * kernel_w + j)) * height_col + h_out) * width_col + w_out); + const int data_offset_w_ptr = (((2 * (i * kernel_w + j) + 1) * height_col + h_out) * width_col + w_out); + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + scalar_t inv_h = h_in + i * dilation_h + offset_h; + scalar_t inv_w = w_in + j * dilation_w + offset_w; + if (inv_h <= -1 || inv_w <= -1 || inv_h >= height || inv_w >= width) + { + inv_h = inv_w = -2; + } + const scalar_t weight = dmcn_2d_get_coordinate_weight( + inv_h, inv_w, + height, width, data_im_ptr + cnt * height * width, width, bp_dir); + val += weight * data_col_ptr[col_pos]; + cnt += 1; + } + // KERNEL_ASSIGN(grad_offset[index], offset_req, val); + grad_offset[index] = val; + } +} + +template +void deformable_2d_im2col_cuda(cudaStream_t stream, + const scalar_t *data_im, const scalar_t *data_offset, + const int batch_size, const int channels, const int height_im, const int width_im, + const int height_col, const int width_col, const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int deformable_group, scalar_t *data_col) { + // num_axes should be smaller than block size + const int channel_per_deformable_group = channels / deformable_group; + const int num_kernels = channels * batch_size * height_col * width_col; + deformable_2d_im2col_gpu_kernel + <<>>( + num_kernels, data_im, data_offset, height_im, width_im, kernel_h, kernel_w, + pad_h, pad_w, stride_h, stride_w, dilation_h, dilation_w, channel_per_deformable_group, + batch_size, channels, deformable_group, height_col, width_col, data_col); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in deformable_im2col_cuda: %s\n", cudaGetErrorString(err)); + } + +} + +template +void deformable_2d_col2im_cuda(cudaStream_t stream, + const scalar_t *data_col, const scalar_t *data_offset, + const int batch_size, const int channels, const int height_im, const int width_im, + const int height_col, const int width_col, const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int deformable_group, scalar_t *grad_im){ + + const int channel_per_deformable_group = channels / deformable_group; + const int num_kernels = channels * kernel_h * kernel_w * batch_size * height_col * width_col; + deformable_2d_col2im_gpu_kernel + <<>>( + num_kernels, data_col, data_offset, channels, height_im, width_im, + kernel_h, kernel_w, pad_h, pad_h, stride_h, stride_w, + dilation_h, dilation_w, channel_per_deformable_group, + batch_size, deformable_group, height_col, width_col, grad_im); + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in deformable_col2im_cuda: %s\n", cudaGetErrorString(err)); + } + +} + +template +void deformable_2d_col2im_coord_cuda(cudaStream_t stream, + const scalar_t *data_col, const scalar_t *data_im, const scalar_t *data_offset, + const int batch_size, const int channels, const int height_im, const int width_im, + const int height_col, const int width_col, const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int deformable_group, + scalar_t *grad_offset) { + const int num_kernels = batch_size * height_col * width_col * 2 * kernel_h * kernel_w * deformable_group; + const int channel_per_deformable_group = channels * kernel_h * kernel_w / deformable_group; + deformable_2d_col2im_coord_gpu_kernel + <<>>( + num_kernels, data_col, data_im, data_offset, channels, height_im, width_im, + kernel_h, kernel_w, pad_h, pad_w, stride_h, stride_w, + dilation_h, dilation_w, channel_per_deformable_group, + batch_size, 2 * kernel_h * kernel_w * deformable_group, deformable_group, height_col, width_col, + grad_offset); + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in deformable_col2im_coord_cuda: %s\n", cudaGetErrorString(err)); + } +} \ No newline at end of file diff --git a/segmentation/mm_modules/DCN/src/cuda/deform_conv2d_cuda.cu b/segmentation/mm_modules/DCN/src/cuda/deform_conv2d_cuda.cu new file mode 100644 index 0000000..0493211 --- /dev/null +++ b/segmentation/mm_modules/DCN/src/cuda/deform_conv2d_cuda.cu @@ -0,0 +1,273 @@ +#include +#include "cuda/deform_2d_im2col_cuda.cuh" + +#include +#include +#include +#include + +// #include +// #include +// #include + +// extern THCState *state; + +// author: Charles Shang +// https://github.com/torch/cunn/blob/master/lib/THCUNN/generic/SpatialConvolutionMM.cu + + +at::Tensor +deform_conv2d_cuda_forward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + // THCAssertSameGPU(THCudaTensor_checkGPU(state, 5, input, weight, bias, offset, mask)); + + AT_ASSERTM(input.is_contiguous(), "input tensor has to be contiguous"); + AT_ASSERTM(weight.is_contiguous(), "weight tensor has to be contiguous"); + + AT_ASSERTM(input.type().is_cuda(), "input must be a CUDA tensor"); + AT_ASSERTM(weight.type().is_cuda(), "weight must be a CUDA tensor"); + AT_ASSERTM(bias.type().is_cuda(), "bias must be a CUDA tensor"); + AT_ASSERTM(offset.type().is_cuda(), "offset must be a CUDA tensor"); + + const int batch = input.size(0); + const int channels = input.size(1); + const int height = input.size(2); + const int width = input.size(3); + + const int channels_out = weight.size(0); + const int channels_kernel = weight.size(1); + const int kernel_h_ = weight.size(2); + const int kernel_w_ = weight.size(3); + + const int im2col_step_ = std::min(batch, im2col_step); + // if (batch % im2col_step_ != 0) { + // printf("batch: %d im2col_step_: %d\n", batch, im2col_step_); + // } + + AT_ASSERTM(batch % im2col_step_ == 0, "batch(%d) must divide im2col_step(%d)", batch, im2col_step_); + AT_ASSERTM((channels % group == 0) && (channels_out % group == 0), + "channels(%d) and channels_out(%d) must divide group(%d)", channels, channels_out, group); + + // printf("Kernels: %d %d %d %d\n", kernel_h_, kernel_w_, kernel_w, kernel_h); + // printf("Channels: %d %d\n", channels, channels_kernel); + // printf("Channels: %d %d\n", channels_out, channels_kernel); + + AT_ASSERTM(kernel_h_ == kernel_h && kernel_w_ == kernel_w, + "Input shape and kernel shape wont match: (%d x %d vs %d x %d).", kernel_h_, kernel_w, kernel_h_, kernel_w_); + + AT_ASSERTM(channels == (channels_kernel * group), + "Input shape and kernel channels wont match: (%d vs %d).", channels, channels_kernel * group); + + const int height_out = (height + 2 * pad_h - (dilation_h * (kernel_h - 1) + 1)) / stride_h + 1; + const int width_out = (width + 2 * pad_w - (dilation_w * (kernel_w - 1) + 1)) / stride_w + 1; + + auto output = at::empty({batch * height_out * width_out, channels_out}, input.options()); + + // prepare group weight and bias + auto weight_g = weight.view({group, channels_out/group, channels_kernel, kernel_h, kernel_w}); + auto bias_g = bias.view({group, channels_out/group}); + + // define alias for easy use + const int batch_n = im2col_step_; + const int per_input_size = channels * height * width; + const int per_offset_size = offset.size(1) * offset.size(2) * offset.size(3); + auto output_n = output.view({batch/im2col_step_, batch_n * height_out * width_out, channels_out}); + for (int n = 0; n < batch/im2col_step_; ++n) + { + auto columns = at::empty({channels * kernel_h * kernel_w, batch_n * height_out * width_out}, input.options()); + AT_DISPATCH_FLOATING_TYPES(input.type(), "deform_conv_forward_cuda", ([&] { + deformable_2d_im2col_cuda(at::cuda::getCurrentCUDAStream(), + input.data() + n * im2col_step_ * per_input_size, + offset.data() + n * im2col_step_ * per_offset_size, + batch_n, channels, height, width, + height_out, width_out, kernel_h, kernel_w, + pad_h, pad_w, stride_h, stride_w, dilation_h, dilation_w, + deformable_group, + columns.data()); + + })); + + // auto columns_m = columns.t(); + // auto weight_m = weight.view({channels_out, channels_kernel * kernel_h * kernel_w}).t(); + // output = at::addmm(bias, columns_m, weight_m); + auto columns_g = columns.view({group, channels/group * kernel_h * kernel_w, batch_n * height_out * width_out}); + auto output_g = output_n.select(0, n).view({batch_n * height_out * width_out, group, channels_out/group}); + for (int g = 0; g < group; ++g) + { + auto columns_gm = columns_g.select(0, g).t(); + auto weight_gm = weight_g.select(0, g).view({channels_out/group, channels_kernel * kernel_h * kernel_w}).t(); + auto output_m = at::addmm(bias_g.select(0, g), columns_gm, weight_gm); + output_g.select(1, g) = output_m.view({batch_n * height_out * width_out, channels_out/group}); + } + + } + + output = output.view({batch, height_out, width_out, channels_out}).permute({0, 3, 1, 2}).contiguous(); + + return output; +} + +std::vector deform_conv2d_cuda_backward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &grad_output, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + + AT_ASSERTM(input.is_contiguous(), "input tensor has to be contiguous"); + AT_ASSERTM(weight.is_contiguous(), "weight tensor has to be contiguous"); + + AT_ASSERTM(input.type().is_cuda(), "input must be a CUDA tensor"); + AT_ASSERTM(weight.type().is_cuda(), "weight must be a CUDA tensor"); + AT_ASSERTM(bias.type().is_cuda(), "bias must be a CUDA tensor"); + AT_ASSERTM(offset.type().is_cuda(), "offset must be a CUDA tensor"); + + const int batch = input.size(0); + const int channels = input.size(1); + const int height = input.size(2); + const int width = input.size(3); + + const int channels_out = weight.size(0); + const int channels_kernel = weight.size(1); + const int kernel_h_ = weight.size(2); + const int kernel_w_ = weight.size(3); + + const int batch_ = grad_output.size(0); + const int channels_out_ = grad_output.size(1); + const int height_out_ = grad_output.size(2); + const int width_out_ = grad_output.size(3); + + const int im2col_step_ = std::min(im2col_step, batch); + + AT_ASSERTM(batch % im2col_step_ == 0, "batch(%d) must divide im2col_step(%d)", batch, im2col_step_); + + AT_ASSERTM((channels % group == 0) && (channels_out % group == 0), + "channels(%d) and channels_out(%d) must divide group(%d)", channels, channels_out, group); + + AT_ASSERTM(kernel_h_ == kernel_h && kernel_w_ == kernel_w, + "Input shape and kernel shape wont match: (%d x %d vs %d x %d).", kernel_h_, kernel_w, kernel_h_, kernel_w_); + + AT_ASSERTM(channels == (channels_kernel * group), + "Input shape and kernel channels wont match: (%d vs %d).", channels, channels_kernel * group); + + const int height_out = (height + 2 * pad_h - (dilation_h * (kernel_h - 1) + 1)) / stride_h + 1; + const int width_out = (width + 2 * pad_w - (dilation_w * (kernel_w - 1) + 1)) / stride_w + 1; + + AT_ASSERTM(batch == batch_, + "Input shape and grad_out batch wont match: (%d vs %d).", batch, batch_); + + AT_ASSERTM(channels_out == channels_out_, + "Input shape and grad_out channels_out wont match: (%d vs %d).", channels_out, channels_out_); + + AT_ASSERTM(height_out == height_out_ && width_out == width_out_, + "Input shape and grad_out shape wont match: (%d x %d vs %d x %d).", height_out, height_out_, width_out, width_out_); + + auto grad_input = at::zeros_like(input); + auto grad_offset = at::zeros_like(offset); + auto grad_weight = at::zeros_like(weight); + auto grad_bias = at::zeros_like(bias); + + // auto grad_output_m = grad_output.permute({1, 0, 2, 3}).contiguous().view({channels_out, batch * height_out * width_out}); + // auto weight_m = weight.view({channels_out, channels_kernel * kernel_h * kernel_w}).t(); + // columns = at::mm(weight_m, grad_output_m); + + // prepare group weight and bias + auto weight_g = weight.view({group, channels_out/group, channels_kernel, kernel_h, kernel_w}); + auto grad_weight_g = grad_weight.view({group, channels_out/group, channels_kernel, kernel_h, kernel_w}); + auto grad_bias_g = grad_bias.view({group, channels_out/group}); + + const int batch_n = im2col_step_; + const int per_input_size = channels * height * width; + const int per_offset_size = offset.size(1) * offset.size(2) * offset.size(3); + auto grad_output_n = grad_output.view({batch/im2col_step_, batch_n, channels_out, height_out, width_out}); + for (int n = 0; n < batch/im2col_step_; ++n) + { + auto grad_output_g = grad_output_n.select(0, n).view({batch_n, group, channels_out/group, height_out, width_out}); + auto ones = at::ones({batch_n * height_out * width_out}, input.options()); + auto columns = at::empty({channels * kernel_h * kernel_w, batch_n * 1 * height_out * width_out}, input.options()); + auto columns_g = columns.view({group, channels/group * kernel_h * kernel_w, batch_n * height_out * width_out}); + for (int g = 0; g < group; ++g) + { + auto grad_output_gm = grad_output_g.select(1, g).permute({1, 0, 2, 3}).contiguous().view({channels_out/group, batch_n * height_out * width_out}); + auto weight_gm = weight_g.select(0, g).view({channels_out/group, channels_kernel * kernel_h * kernel_w}).t(); + columns_g.select(0, g) = at::mm(weight_gm, grad_output_gm); + } + + AT_DISPATCH_FLOATING_TYPES(input.type(), "deform_conv_backward_cuda", ([&] { + deformable_2d_col2im_coord_cuda(at::cuda::getCurrentCUDAStream(), + columns.data(), + input.data() + n * im2col_step_ * per_input_size, + offset.data() + n * im2col_step_ * per_offset_size, + batch_n, channels, height, width, + height_out, width_out, kernel_h, kernel_w, + pad_h, pad_w, stride_h, stride_w, + dilation_h, dilation_w, deformable_group, + grad_offset.data() + n * im2col_step_ * per_offset_size); + // gradient w.r.t. input data + deformable_2d_col2im_cuda(at::cuda::getCurrentCUDAStream(), + columns.data(), + offset.data() + n * im2col_step_ * per_offset_size, + batch_n, channels, height, width, + height_out, width_out, kernel_h, kernel_w, + pad_h, pad_w, stride_h, stride_w, + dilation_h, dilation_w, deformable_group, + grad_input.data() + n * im2col_step_ * per_input_size); + + // gradient w.r.t. weight, dWeight should accumulate across the batch and group + deformable_2d_im2col_cuda(at::cuda::getCurrentCUDAStream(), + input.data() + n * im2col_step_ * per_input_size, + offset.data() + n * im2col_step_ * per_offset_size, + batch_n, channels, height, width, + height_out, width_out, kernel_h, kernel_w, + pad_h, pad_w, stride_h, stride_w, + dilation_h, dilation_w, deformable_group, + columns.data()); + + })); + + // auto grad_output_m = grad_output.permute({1, 0, 2, 3}).contiguous().view({channels_out, batch * height_out * width_out}); + // grad_weight = at::mm(grad_output_m, columns.t()).view_as(weight); + // grad_bias = at::mv(grad_output_m, ones); + // auto grad_output_g = grad_output.view({batch, group, channels_out/group, height_out, width_out}); + // auto columns_g = columns.view({group, channels/group * kernel_h * kernel_w, batch * height_out * width_out}); + for (int g = 0; g < group; ++g) + { + auto grad_output_gm = grad_output_g.select(1, g).permute({1, 0, 2, 3}).contiguous().view({channels_out/group, batch_n * height_out * width_out}); + auto columns_gm = columns_g.select(0, g).t(); + auto grad_weight_gm = grad_weight_g.select(0, g).view({channels_out/group, channels_kernel * kernel_h * kernel_w}); + auto grad_bias_gm = grad_bias_g.select(0, g); + grad_weight_g.select(0, g) = at::addmm(grad_weight_gm, grad_output_gm, columns_gm).view_as(grad_weight_g.select(0, g)); + grad_bias_g.select(0, g) = at::addmv(grad_bias_gm, grad_output_gm, ones); + } + + } + + return { + grad_input, grad_offset, grad_weight, grad_bias + }; +} diff --git a/segmentation/mm_modules/DCN/src/cuda/deform_conv2d_cuda.h b/segmentation/mm_modules/DCN/src/cuda/deform_conv2d_cuda.h new file mode 100644 index 0000000..0958453 --- /dev/null +++ b/segmentation/mm_modules/DCN/src/cuda/deform_conv2d_cuda.h @@ -0,0 +1,38 @@ +#pragma once +#include + +at::Tensor +deform_conv2d_cuda_forward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step); + +std::vector +deform_conv2d_cuda_backward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &grad_output, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step); + diff --git a/segmentation/mm_modules/DCN/src/cuda/modulated_deform_2d_im2col_cuda.cuh b/segmentation/mm_modules/DCN/src/cuda/modulated_deform_2d_im2col_cuda.cuh new file mode 100644 index 0000000..0ff18cb --- /dev/null +++ b/segmentation/mm_modules/DCN/src/cuda/modulated_deform_2d_im2col_cuda.cuh @@ -0,0 +1,420 @@ +#include +#include +#include + +#include +#include + +// #include +#include +// #include + +#define CUDA_KERNEL_LOOP(i, n) \ + for (int i = blockIdx.x * blockDim.x + threadIdx.x; \ + i < (n); \ + i += blockDim.x * gridDim.x) + +const int CUDA_NUM_THREADS = 1024; +inline int GET_BLOCKS(const int N) +{ + return (N + CUDA_NUM_THREADS - 1) / CUDA_NUM_THREADS; +} + + +template +__device__ scalar_t mdmcn_2d_im2col_bilinear(const scalar_t *bottom_data, const int data_width, + const int height, const int width, scalar_t h, scalar_t w) +{ + int h_low = floor(h); + int w_low = floor(w); + int h_high = h_low + 1; + int w_high = w_low + 1; + + scalar_t lh = h - h_low; + scalar_t lw = w - w_low; + scalar_t hh = 1 - lh, hw = 1 - lw; + + scalar_t v1 = 0; + if (h_low >= 0 && w_low >= 0) + v1 = bottom_data[h_low * data_width + w_low]; + scalar_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) + v2 = bottom_data[h_low * data_width + w_high]; + scalar_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) + v3 = bottom_data[h_high * data_width + w_low]; + scalar_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) + v4 = bottom_data[h_high * data_width + w_high]; + + scalar_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + + scalar_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + return val; +} + +template +__device__ scalar_t mdmcn_2d_get_gradient_weight(scalar_t argmax_h, scalar_t argmax_w, + const int h, const int w, const int height, const int width) +{ + if (argmax_h <= -1 || argmax_h >= height || argmax_w <= -1 || argmax_w >= width) + { + //empty + return 0; + } + + int argmax_h_low = floor(argmax_h); + int argmax_w_low = floor(argmax_w); + int argmax_h_high = argmax_h_low + 1; + int argmax_w_high = argmax_w_low + 1; + + scalar_t weight = 0; + if (h == argmax_h_low && w == argmax_w_low) + weight = (h + 1 - argmax_h) * (w + 1 - argmax_w); + if (h == argmax_h_low && w == argmax_w_high) + weight = (h + 1 - argmax_h) * (argmax_w + 1 - w); + if (h == argmax_h_high && w == argmax_w_low) + weight = (argmax_h + 1 - h) * (w + 1 - argmax_w); + if (h == argmax_h_high && w == argmax_w_high) + weight = (argmax_h + 1 - h) * (argmax_w + 1 - w); + return weight; +} + +template +__device__ scalar_t mdmcn_2d_get_coordinate_weight(scalar_t argmax_h, scalar_t argmax_w, + const int height, const int width, const scalar_t *im_data, + const int data_width, const int bp_dir) +{ + if (argmax_h <= -1 || argmax_h >= height || argmax_w <= -1 || argmax_w >= width) + { + //empty + return 0; + } + + int argmax_h_low = floor(argmax_h); + int argmax_w_low = floor(argmax_w); + int argmax_h_high = argmax_h_low + 1; + int argmax_w_high = argmax_w_low + 1; + + scalar_t weight = 0; + + if (bp_dir == 0) + { + if (argmax_h_low >= 0 && argmax_w_low >= 0) + weight += -1 * (argmax_w_low + 1 - argmax_w) * im_data[argmax_h_low * data_width + argmax_w_low]; + if (argmax_h_low >= 0 && argmax_w_high <= width - 1) + weight += -1 * (argmax_w - argmax_w_low) * im_data[argmax_h_low * data_width + argmax_w_high]; + if (argmax_h_high <= height - 1 && argmax_w_low >= 0) + weight += (argmax_w_low + 1 - argmax_w) * im_data[argmax_h_high * data_width + argmax_w_low]; + if (argmax_h_high <= height - 1 && argmax_w_high <= width - 1) + weight += (argmax_w - argmax_w_low) * im_data[argmax_h_high * data_width + argmax_w_high]; + } + else if (bp_dir == 1) + { + if (argmax_h_low >= 0 && argmax_w_low >= 0) + weight += -1 * (argmax_h_low + 1 - argmax_h) * im_data[argmax_h_low * data_width + argmax_w_low]; + if (argmax_h_low >= 0 && argmax_w_high <= width - 1) + weight += (argmax_h_low + 1 - argmax_h) * im_data[argmax_h_low * data_width + argmax_w_high]; + if (argmax_h_high <= height - 1 && argmax_w_low >= 0) + weight += -1 * (argmax_h - argmax_h_low) * im_data[argmax_h_high * data_width + argmax_w_low]; + if (argmax_h_high <= height - 1 && argmax_w_high <= width - 1) + weight += (argmax_h - argmax_h_low) * im_data[argmax_h_high * data_width + argmax_w_high]; + } + + return weight; +} + +template +__global__ void modulated_deformable_2d_im2col_gpu_kernel(const int n, + const scalar_t *data_im, const scalar_t *data_offset, + const scalar_t *data_mask, + const int height, const int width, const int kernel_h, + const int kernel_w, + const int pad_h, const int pad_w, + const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, const int num_channels, + const int deformable_group, + const int height_col, const int width_col, + scalar_t *data_col) +{ + // launch channels * batch_size * height_col * width_col cores + CUDA_KERNEL_LOOP(index, n) + { + // NOTE(CharlesShang): different from Dai Jifeng's MXNet implementation, col_buffer is of shape (c*kw*kh, N, oh, ow) + // here columns is of shape (N, c*kw*kh, oh * ow), need to adapt axis + // NOTE(Jiarui XU): different from CharlesShang's implementation, col_buffer is of shape (N, c*kw*kh, oh * ow) + // here columns is of shape (c*kw*kh, N, oh, ow), need to adapt axis + + // index index of output matrix + const int w_col = index % width_col; + const int h_col = (index / width_col) % height_col; + const int b_col = (index / width_col / height_col) % batch_size; + const int c_im = (index / width_col / height_col) / batch_size; + const int c_col = c_im * kernel_h * kernel_w; + + // compute deformable group index + const int deformable_group_index = c_im / channel_per_deformable_group; + + const int h_in = h_col * stride_h - pad_h; + const int w_in = w_col * stride_w - pad_w; + + scalar_t *data_col_ptr = data_col + ((c_col * batch_size + b_col) * height_col + h_col) * width_col + w_col; + //const scalar_t* data_im_ptr = data_im + ((b_col * num_channels + c_im) * height + h_in) * width + w_in; + const scalar_t *data_im_ptr = data_im + (b_col * num_channels + c_im) * height * width; + const scalar_t *data_offset_ptr = data_offset + (b_col * deformable_group + deformable_group_index) * 2 * kernel_h * kernel_w * height_col * width_col; + + const scalar_t *data_mask_ptr = data_mask + (b_col * deformable_group + deformable_group_index) * kernel_h * kernel_w * height_col * width_col; + + for (int i = 0; i < kernel_h; ++i) + { + for (int j = 0; j < kernel_w; ++j) + { + const int data_offset_h_ptr = ((2 * (i * kernel_w + j)) * height_col + h_col) * width_col + w_col; + const int data_offset_w_ptr = ((2 * (i * kernel_w + j) + 1) * height_col + h_col) * width_col + w_col; + const int data_mask_hw_ptr = ((i * kernel_w + j) * height_col + h_col) * width_col + w_col; + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + const scalar_t mask = data_mask_ptr[data_mask_hw_ptr]; + scalar_t val = static_cast(0); + const scalar_t h_im = h_in + i * dilation_h + offset_h; + const scalar_t w_im = w_in + j * dilation_w + offset_w; + if (h_im > -1 && w_im > -1 && h_im < height && w_im < width) + { + //const scalar_t map_h = i * dilation_h + offset_h; + //const scalar_t map_w = j * dilation_w + offset_w; + //const int cur_height = height - h_in; + //const int cur_width = width - w_in; + //val = dmcn_im2col_bilinear(data_im_ptr, width, cur_height, cur_width, map_h, map_w); + val = mdmcn_2d_im2col_bilinear(data_im_ptr, width, height, width, h_im, w_im); + } + *data_col_ptr = val * mask; + data_col_ptr += batch_size * height_col * width_col; + } + } + } +} + +template +__global__ void modulated_deformable_2d_col2im_gpu_kernel(const int n, + const scalar_t *data_col, const scalar_t *data_offset, + const scalar_t *data_mask, + const int channels, const int height, const int width, + const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, + const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, const int deformable_group, + const int height_col, const int width_col, + scalar_t *grad_im) +{ + CUDA_KERNEL_LOOP(index, n) + { + const int j = (index / width_col / height_col / batch_size) % kernel_w; + const int i = (index / width_col / height_col / batch_size / kernel_w) % kernel_h; + const int c = index / width_col / height_col / batch_size / kernel_w / kernel_h; + // compute the start and end of the output + + const int deformable_group_index = c / channel_per_deformable_group; + + int w_out = index % width_col; + int h_out = (index / width_col) % height_col; + int b = (index / width_col / height_col) % batch_size; + int w_in = w_out * stride_w - pad_w; + int h_in = h_out * stride_h - pad_h; + + const scalar_t *data_offset_ptr = data_offset + (b * deformable_group + deformable_group_index) * 2 * kernel_h * kernel_w * height_col * width_col; + const scalar_t *data_mask_ptr = data_mask + (b * deformable_group + deformable_group_index) * kernel_h * kernel_w * height_col * width_col; + const int data_offset_h_ptr = ((2 * (i * kernel_w + j)) * height_col + h_out) * width_col + w_out; + const int data_offset_w_ptr = ((2 * (i * kernel_w + j) + 1) * height_col + h_out) * width_col + w_out; + const int data_mask_hw_ptr = ((i * kernel_w + j) * height_col + h_out) * width_col + w_out; + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + const scalar_t mask = data_mask_ptr[data_mask_hw_ptr]; + const scalar_t cur_inv_h_data = h_in + i * dilation_h + offset_h; + const scalar_t cur_inv_w_data = w_in + j * dilation_w + offset_w; + + const scalar_t cur_top_grad = data_col[index] * mask; + const int cur_h = (int)cur_inv_h_data; + const int cur_w = (int)cur_inv_w_data; + for (int dy = -2; dy <= 2; dy++) + { + for (int dx = -2; dx <= 2; dx++) + { + if (cur_h + dy >= 0 && cur_h + dy < height && + cur_w + dx >= 0 && cur_w + dx < width && + abs(cur_inv_h_data - (cur_h + dy)) < 1 && + abs(cur_inv_w_data - (cur_w + dx)) < 1) + { + int cur_bottom_grad_pos = ((b * channels + c) * height + cur_h + dy) * width + cur_w + dx; + scalar_t weight = mdmcn_2d_get_gradient_weight(cur_inv_h_data, cur_inv_w_data, cur_h + dy, cur_w + dx, height, width); + atomicAdd(grad_im + cur_bottom_grad_pos, weight * cur_top_grad); + } + } + } + } +} + +template +__global__ void modulated_deformable_2d_col2im_coord_gpu_kernel(const int n, + const scalar_t *data_col, const scalar_t *data_im, + const scalar_t *data_offset, const scalar_t *data_mask, + const int channels, const int height, const int width, + const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, + const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, const int offset_channels, + const int deformable_group, + const int height_col, const int width_col, + scalar_t *grad_offset, scalar_t *grad_mask) +{ + CUDA_KERNEL_LOOP(index, n) + { + scalar_t val = 0, mval = 0; + int w = index % width_col; + int h = (index / width_col) % height_col; + int c = (index / width_col / height_col) % offset_channels; + int b = (index / width_col / height_col) / offset_channels; + // compute the start and end of the output + + const int deformable_group_index = c / (2 * kernel_h * kernel_w); + const int col_step = kernel_h * kernel_w; + int cnt = 0; + const scalar_t *data_col_ptr = data_col + deformable_group_index * channel_per_deformable_group * batch_size * width_col * height_col; + const scalar_t *data_im_ptr = data_im + (b * deformable_group + deformable_group_index) * channel_per_deformable_group / kernel_h / kernel_w * height * width; + const scalar_t *data_offset_ptr = data_offset + (b * deformable_group + deformable_group_index) * 2 * kernel_h * kernel_w * height_col * width_col; + const scalar_t *data_mask_ptr = data_mask + (b * deformable_group + deformable_group_index) * kernel_h * kernel_w * height_col * width_col; + + const int offset_c = c - deformable_group_index * 2 * kernel_h * kernel_w; + + for (int col_c = (offset_c / 2); col_c < channel_per_deformable_group; col_c += col_step) + { + const int col_pos = (((col_c * batch_size + b) * height_col) + h) * width_col + w; + const int bp_dir = offset_c % 2; + + int j = (col_pos / width_col / height_col / batch_size) % kernel_w; + int i = (col_pos / width_col / height_col / batch_size / kernel_w) % kernel_h; + int w_out = col_pos % width_col; + int h_out = (col_pos / width_col) % height_col; + int w_in = w_out * stride_w - pad_w; + int h_in = h_out * stride_h - pad_h; + const int data_offset_h_ptr = (((2 * (i * kernel_w + j)) * height_col + h_out) * width_col + w_out); + const int data_offset_w_ptr = (((2 * (i * kernel_w + j) + 1) * height_col + h_out) * width_col + w_out); + const int data_mask_hw_ptr = (((i * kernel_w + j) * height_col + h_out) * width_col + w_out); + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + const scalar_t mask = data_mask_ptr[data_mask_hw_ptr]; + scalar_t inv_h = h_in + i * dilation_h + offset_h; + scalar_t inv_w = w_in + j * dilation_w + offset_w; + if (inv_h <= -1 || inv_w <= -1 || inv_h >= height || inv_w >= width) + { + inv_h = inv_w = -2; + } + else + { + mval += data_col_ptr[col_pos] * mdmcn_2d_im2col_bilinear(data_im_ptr + cnt * height * width, width, height, width, inv_h, inv_w); + } + const scalar_t weight = mdmcn_2d_get_coordinate_weight( + inv_h, inv_w, + height, width, data_im_ptr + cnt * height * width, width, bp_dir); + val += weight * data_col_ptr[col_pos] * mask; + cnt += 1; + } + // KERNEL_ASSIGN(grad_offset[index], offset_req, val); + grad_offset[index] = val; + if (offset_c % 2 == 0) + // KERNEL_ASSIGN(grad_mask[(((b * deformable_group + deformable_group_index) * kernel_h * kernel_w + offset_c / 2) * height_col + h) * width_col + w], mask_req, mval); + grad_mask[(((b * deformable_group + deformable_group_index) * kernel_h * kernel_w + offset_c / 2) * height_col + h) * width_col + w] = mval; + } +} + +template +void modulated_deformable_2d_im2col_cuda(cudaStream_t stream, + const scalar_t *data_im, const scalar_t *data_offset, + const scalar_t *data_mask, + const int batch_size, const int channels, const int height_im, + const int width_im, + const int height_col, const int width_col, const int kernel_h, + const int kernel_w, + const int pad_h, const int pad_w, const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int deformable_group, scalar_t *data_col) { + // num_axes should be smaller than block size + const int channel_per_deformable_group = channels / deformable_group; + const int num_kernels = channels * batch_size * height_col * width_col; + modulated_deformable_2d_im2col_gpu_kernel + <<>>( + num_kernels, data_im, data_offset, data_mask, height_im, width_im, kernel_h, kernel_w, + pad_h, pad_w, stride_h, stride_w, dilation_h, dilation_w, channel_per_deformable_group, + batch_size, channels, deformable_group, height_col, width_col, data_col); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in modulated_deformable_im2col_cuda: %s\n", cudaGetErrorString(err)); + } + +} + +template +void modulated_deformable_2d_col2im_cuda(cudaStream_t stream, + const scalar_t *data_col, const scalar_t *data_offset, + const scalar_t *data_mask, + const int batch_size, const int channels, const int height_im, + const int width_im, + const int height_col, const int width_col, const int kernel_h, + const int kernel_w, + const int pad_h, const int pad_w, const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int deformable_group, scalar_t *grad_im){ + + const int channel_per_deformable_group = channels / deformable_group; + const int num_kernels = channels * kernel_h * kernel_w * batch_size * height_col * width_col; + modulated_deformable_2d_col2im_gpu_kernel + <<>>( + num_kernels, data_col, data_offset, data_mask, channels, height_im, width_im, + kernel_h, kernel_w, pad_h, pad_h, stride_h, stride_w, + dilation_h, dilation_w, channel_per_deformable_group, + batch_size, deformable_group, height_col, width_col, grad_im); + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in modulated_deformable_col2im_cuda: %s\n", cudaGetErrorString(err)); + } + +} + +template +void modulated_deformable_2d_col2im_coord_cuda(cudaStream_t stream, + const scalar_t *data_col, const scalar_t *data_im, + const scalar_t *data_offset, const scalar_t *data_mask, + const int batch_size, const int channels, const int height_im, + const int width_im, + const int height_col, const int width_col, const int kernel_h, + const int kernel_w, + const int pad_h, const int pad_w, const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int deformable_group, + scalar_t *grad_offset, scalar_t *grad_mask) { + const int num_kernels = batch_size * height_col * width_col * 2 * kernel_h * kernel_w * deformable_group; + const int channel_per_deformable_group = channels * kernel_h * kernel_w / deformable_group; + modulated_deformable_2d_col2im_coord_gpu_kernel + <<>>( + num_kernels, data_col, data_im, data_offset, data_mask, channels, height_im, width_im, + kernel_h, kernel_w, pad_h, pad_w, stride_h, stride_w, + dilation_h, dilation_w, channel_per_deformable_group, + batch_size, 2 * kernel_h * kernel_w * deformable_group, deformable_group, height_col, width_col, + grad_offset, grad_mask); + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in modulated_deformable_col2im_coord_cuda: %s\n", cudaGetErrorString(err)); + } +} \ No newline at end of file diff --git a/segmentation/mm_modules/DCN/src/cuda/modulated_deform_conv2d_cuda.cu b/segmentation/mm_modules/DCN/src/cuda/modulated_deform_conv2d_cuda.cu new file mode 100644 index 0000000..18ec02b --- /dev/null +++ b/segmentation/mm_modules/DCN/src/cuda/modulated_deform_conv2d_cuda.cu @@ -0,0 +1,280 @@ +#include +#include "cuda/modulated_deform_2d_im2col_cuda.cuh" + +#include +#include +#include +#include + +// #include +// #include +// #include + +// extern THCState *state; + +// author: Charles Shang +// https://github.com/torch/cunn/blob/master/lib/THCUNN/generic/SpatialConvolutionMM.cu + + +at::Tensor +modulated_deform_conv2d_cuda_forward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &mask, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + // THCAssertSameGPU(THCudaTensor_checkGPU(state, 5, input, weight, bias, offset, mask)); + + AT_ASSERTM(input.is_contiguous(), "input tensor has to be contiguous"); + AT_ASSERTM(weight.is_contiguous(), "weight tensor has to be contiguous"); + + AT_ASSERTM(input.type().is_cuda(), "input must be a CUDA tensor"); + AT_ASSERTM(weight.type().is_cuda(), "weight must be a CUDA tensor"); + AT_ASSERTM(bias.type().is_cuda(), "bias must be a CUDA tensor"); + AT_ASSERTM(offset.type().is_cuda(), "offset must be a CUDA tensor"); + AT_ASSERTM(mask.type().is_cuda(), "mask must be a CUDA tensor"); + + const int batch = input.size(0); + const int channels = input.size(1); + const int height = input.size(2); + const int width = input.size(3); + + const int channels_out = weight.size(0); + const int channels_kernel = weight.size(1); + const int kernel_h_ = weight.size(2); + const int kernel_w_ = weight.size(3); + + const int im2col_step_ = std::min(batch, im2col_step); + + AT_ASSERTM(batch % im2col_step_ == 0, "batch(%d) must divide im2col_step(%d)", batch, im2col_step_); + + AT_ASSERTM((channels % group == 0) && (channels_out % group == 0), + "channels(%d) and channels_out(%d) must divide group(%d)", channels, channels_out, group); + + // printf("Kernels: %d %d %d %d\n", kernel_h_, kernel_w_, kernel_w, kernel_h); + // printf("Channels: %d %d\n", channels, channels_kernel); + // printf("Channels: %d %d\n", channels_out, channels_kernel); + + AT_ASSERTM(kernel_h_ == kernel_h && kernel_w_ == kernel_w, + "Input shape and kernel shape wont match: (%d x %d vs %d x %d).", kernel_h_, kernel_w, kernel_h_, kernel_w_); + + AT_ASSERTM(channels == (channels_kernel * group), + "Input shape and kernel channels wont match: (%d vs %d).", channels, channels_kernel * group); + + const int height_out = (height + 2 * pad_h - (dilation_h * (kernel_h - 1) + 1)) / stride_h + 1; + const int width_out = (width + 2 * pad_w - (dilation_w * (kernel_w - 1) + 1)) / stride_w + 1; + + auto output = at::empty({batch * height_out * width_out, channels_out}, input.options()); + + // prepare group weight and bias + auto weight_g = weight.view({group, channels_out/group, channels_kernel, kernel_h, kernel_w}); + auto bias_g = bias.view({group, channels_out/group}); + + // define alias for easy use + const int batch_n = im2col_step_; + const int per_input_size = channels * height * width; + const int per_offset_size = offset.size(1) * offset.size(2) * offset.size(3); + const int per_mask_size = mask.size(1) * mask.size(2) * mask.size(3); + auto output_n = output.view({batch/im2col_step_, batch_n * height_out * width_out, channels_out}); + for (int n = 0; n < batch/im2col_step_; ++n) + { + auto columns = at::empty({channels * kernel_h * kernel_w, batch_n * height_out * width_out}, input.options()); + AT_DISPATCH_FLOATING_TYPES(input.type(), "deform_conv_forward_cuda", ([&] { + modulated_deformable_2d_im2col_cuda(at::cuda::getCurrentCUDAStream(), + input.data() + n * im2col_step_ * per_input_size, + offset.data() + n * im2col_step_ * per_offset_size, + mask.data() + n * im2col_step_ * per_mask_size, + batch_n, channels, height, width, + height_out, width_out, kernel_h, kernel_w, + pad_h, pad_w, stride_h, stride_w, dilation_h, dilation_w, + deformable_group, + columns.data()); + + })); + + auto columns_g = columns.view({group, channels/group * kernel_h * kernel_w, batch_n * height_out * width_out}); + auto output_g = output_n.select(0, n).view({batch_n * height_out * width_out, group, channels_out/group}); + for (int g = 0; g < group; ++g) + { + auto columns_gm = columns_g.select(0, g).t(); + auto weight_gm = weight_g.select(0, g).view({channels_out/group, channels_kernel * kernel_h * kernel_w}).t(); + auto output_m = at::addmm(bias_g.select(0, g), columns_gm, weight_gm); + output_g.select(1, g) = output_m.view({batch_n * height_out * width_out, channels_out/group}); + } + + } + + output = output.view({batch, height_out, width_out, channels_out}).permute({0, 3, 1, 2}).contiguous(); + + return output; +} + + +std::vector modulated_deform_conv2d_cuda_backward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &mask, + const at::Tensor &grad_output, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + + AT_ASSERTM(input.is_contiguous(), "input tensor has to be contiguous"); + AT_ASSERTM(weight.is_contiguous(), "weight tensor has to be contiguous"); + + AT_ASSERTM(input.type().is_cuda(), "input must be a CUDA tensor"); + AT_ASSERTM(weight.type().is_cuda(), "weight must be a CUDA tensor"); + AT_ASSERTM(bias.type().is_cuda(), "bias must be a CUDA tensor"); + AT_ASSERTM(offset.type().is_cuda(), "offset must be a CUDA tensor"); + AT_ASSERTM(mask.type().is_cuda(), "mask must be a CUDA tensor"); + + const int batch = input.size(0); + const int channels = input.size(1); + const int height = input.size(2); + const int width = input.size(3); + + const int channels_out = weight.size(0); + const int channels_kernel = weight.size(1); + const int kernel_h_ = weight.size(2); + const int kernel_w_ = weight.size(3); + + const int batch_ = grad_output.size(0); + const int channels_out_ = grad_output.size(1); + const int height_out_ = grad_output.size(2); + const int width_out_ = grad_output.size(3); + + const int im2col_step_ = std::min(im2col_step, batch); + + AT_ASSERTM(batch % im2col_step_ == 0, "batch(%d) must divide im2col_step(%d)", batch, im2col_step_); + + AT_ASSERTM((channels % group == 0) && (channels_out % group == 0), + "channels(%d) and channels_out(%d) must divide group(%d)", channels, channels_out, group); + + AT_ASSERTM(kernel_h_ == kernel_h && kernel_w_ == kernel_w, + "Input shape and kernel shape wont match: (%d x %d vs %d x %d).", kernel_h_, kernel_w, kernel_h_, kernel_w_); + + AT_ASSERTM(channels == (channels_kernel * group), + "Input shape and kernel channels wont match: (%d vs %d).", channels, channels_kernel * group); + + const int height_out = (height + 2 * pad_h - (dilation_h * (kernel_h - 1) + 1)) / stride_h + 1; + const int width_out = (width + 2 * pad_w - (dilation_w * (kernel_w - 1) + 1)) / stride_w + 1; + + AT_ASSERTM(batch == batch_, + "Input shape and grad_out batch wont match: (%d vs %d).", batch, batch_); + + AT_ASSERTM(channels_out == channels_out_, + "Input shape and grad_out channels_out wont match: (%d vs %d).", channels_out, channels_out_); + + AT_ASSERTM(height_out == height_out_ && width_out == width_out_, + "Input shape and grad_out shape wont match: (%d x %d vs %d x %d).", height_out, height_out_, width_out, width_out_); + + auto ones = at::ones({batch * height_out * width_out}, input.options()); + auto columns = at::empty({channels * kernel_h * kernel_w, batch * 1 * height_out * width_out}, input.options()); + + auto grad_input = at::zeros_like(input); + auto grad_weight = at::zeros_like(weight); + auto grad_bias = at::zeros_like(bias); + auto grad_offset = at::zeros_like(offset); + auto grad_mask = at::zeros_like(mask); + + // prepare group weight and bias + auto weight_g = weight.view({group, channels_out/group, channels_kernel, kernel_h, kernel_w}); + auto grad_weight_g = grad_weight.view({group, channels_out/group, channels_kernel, kernel_h, kernel_w}); + auto grad_bias_g = grad_bias.view({group, channels_out/group}); + + const int batch_n = im2col_step_; + const int per_input_size = channels * height * width; + const int per_offset_size = offset.size(1) * offset.size(2) * offset.size(3); + const int per_mask_size = mask.size(1) * mask.size(2) * mask.size(3); + auto grad_output_n = grad_output.view({batch/im2col_step_, batch_n, channels_out, height_out, width_out}); + for (int n = 0; n < batch/im2col_step_; ++n) + { + auto grad_output_g = grad_output_n.select(0, n).view({batch_n, group, channels_out/group, height_out, width_out}); + auto ones = at::ones({batch_n * height_out * width_out}, input.options()); + auto columns = at::empty({channels * kernel_h * kernel_w, batch_n * 1 * height_out * width_out}, input.options()); + auto columns_g = columns.view({group, channels/group * kernel_h * kernel_w, batch_n * height_out * width_out}); + for (int g = 0; g < group; ++g) + { + auto grad_output_gm = grad_output_g.select(1, g).permute({1, 0, 2, 3}).contiguous().view({channels_out/group, batch_n * height_out * width_out}); + auto weight_gm = weight_g.select(0, g).view({channels_out/group, channels_kernel * kernel_h * kernel_w}).t(); + columns_g.select(0, g) = at::mm(weight_gm, grad_output_gm); + } + + AT_DISPATCH_FLOATING_TYPES(input.type(), "deform_conv_backward_cuda", ([&] { + modulated_deformable_2d_col2im_coord_cuda(at::cuda::getCurrentCUDAStream(), + columns.data(), + input.data() + n * im2col_step_ * per_input_size, + offset.data() + n * im2col_step_ * per_offset_size, + mask.data() + n * im2col_step_ * per_mask_size, + batch_n, channels, height, width, + height_out, width_out, kernel_h, kernel_w, + pad_h, pad_w, stride_h, stride_w, + dilation_h, dilation_w, deformable_group, + grad_offset.data() + n * im2col_step_ * per_offset_size, + grad_mask.data() + n * im2col_step_ * per_mask_size); + // gradient w.r.t. input data + modulated_deformable_2d_col2im_cuda(at::cuda::getCurrentCUDAStream(), + columns.data(), + offset.data() + n * im2col_step_ * per_offset_size, + mask.data() + n * im2col_step_ * per_mask_size, + batch_n, channels, height, width, + height_out, width_out, kernel_h, kernel_w, + pad_h, pad_w, stride_h, stride_w, + dilation_h, dilation_w, deformable_group, + grad_input.data() + n * im2col_step_ * per_input_size); + + // gradient w.r.t. weight, dWeight should accumulate across the batch and group + modulated_deformable_2d_im2col_cuda(at::cuda::getCurrentCUDAStream(), + input.data() + n * im2col_step_ * per_input_size, + offset.data() + n * im2col_step_ * per_offset_size, + mask.data() + n * im2col_step_ * per_mask_size, + batch_n, channels, height, width, + height_out, width_out, kernel_h, kernel_w, + pad_h, pad_w, stride_h, stride_w, + dilation_h, dilation_w, deformable_group, + columns.data()); + + })); + + // auto grad_output_m = grad_output.permute({1, 0, 2, 3}).contiguous().view({channels_out, batch * height_out * width_out}); + // grad_weight = at::mm(grad_output_m, columns.t()).view_as(weight); + // grad_bias = at::mv(grad_output_m, ones); + // auto grad_output_g = grad_output.view({batch, group, channels_out/group, height_out, width_out}); + // auto columns_g = columns.view({group, channels/group * kernel_h * kernel_w, batch * height_out * width_out}); + for (int g = 0; g < group; ++g) + { + auto grad_output_gm = grad_output_g.select(1, g).permute({1, 0, 2, 3}).contiguous().view({channels_out/group, batch_n * height_out * width_out}); + auto columns_gm = columns_g.select(0, g).t(); + auto grad_weight_gm = grad_weight_g.select(0, g).view({channels_out/group, channels_kernel * kernel_h * kernel_w}); + auto grad_bias_gm = grad_bias_g.select(0, g); + grad_weight_g.select(0, g) = at::addmm(grad_weight_gm, grad_output_gm, columns_gm).view_as(grad_weight_g.select(0, g)); + grad_bias_g.select(0, g) = at::addmv(grad_bias_gm, grad_output_gm, ones); + } + + } + + return { + grad_input, grad_offset, grad_mask, grad_weight, grad_bias + }; +} diff --git a/segmentation/mm_modules/DCN/src/cuda/modulated_deform_conv2d_cuda.h b/segmentation/mm_modules/DCN/src/cuda/modulated_deform_conv2d_cuda.h new file mode 100644 index 0000000..4ee2dce --- /dev/null +++ b/segmentation/mm_modules/DCN/src/cuda/modulated_deform_conv2d_cuda.h @@ -0,0 +1,40 @@ +#pragma once +#include + +at::Tensor +modulated_deform_conv2d_cuda_forward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &mask, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step); + +std::vector +modulated_deform_conv2d_cuda_backward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &mask, + const at::Tensor &grad_output, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step); + diff --git a/segmentation/mm_modules/DCN/src/deform_conv2d.h b/segmentation/mm_modules/DCN/src/deform_conv2d.h new file mode 100644 index 0000000..bf0af29 --- /dev/null +++ b/segmentation/mm_modules/DCN/src/deform_conv2d.h @@ -0,0 +1,84 @@ +#pragma once + +#include "cpu/deform_conv2d_cpu.h" + +#ifdef WITH_CUDA +#include "cuda/deform_conv2d_cuda.h" +#endif + + +at::Tensor +deform_conv2d_forward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + if (input.type().is_cuda()) + { +#ifdef WITH_CUDA + return deform_conv2d_cuda_forward(input, weight, bias, offset, + kernel_h, kernel_w, + stride_h, stride_w, + pad_h, pad_w, + dilation_h, dilation_w, + group, + deformable_group, + im2col_step); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} + +std::vector +deform_conv2d_backward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &grad_output, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + if (input.type().is_cuda()) + { +#ifdef WITH_CUDA + return deform_conv2d_cuda_backward(input, + weight, + bias, + offset, + grad_output, + kernel_h, kernel_w, + stride_h, stride_w, + pad_h, pad_w, + dilation_h, dilation_w, + group, + deformable_group, + im2col_step); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} + diff --git a/segmentation/mm_modules/DCN/src/modulated_deform_conv2d.h b/segmentation/mm_modules/DCN/src/modulated_deform_conv2d.h new file mode 100644 index 0000000..9c8043e --- /dev/null +++ b/segmentation/mm_modules/DCN/src/modulated_deform_conv2d.h @@ -0,0 +1,87 @@ +#pragma once + +#include "cpu/modulated_deform_conv2d_cpu.h" + +#ifdef WITH_CUDA +#include "cuda/modulated_deform_conv2d_cuda.h" +#endif + + +at::Tensor +modulated_deform_conv2d_forward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &mask, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + if (input.type().is_cuda()) + { +#ifdef WITH_CUDA + return modulated_deform_conv2d_cuda_forward(input, weight, bias, offset, mask, + kernel_h, kernel_w, + stride_h, stride_w, + pad_h, pad_w, + dilation_h, dilation_w, + group, + deformable_group, + im2col_step); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} + +std::vector +modulated_deform_conv2d_backward(const at::Tensor &input, + const at::Tensor &weight, + const at::Tensor &bias, + const at::Tensor &offset, + const at::Tensor &mask, + const at::Tensor &grad_output, + const int kernel_h, + const int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const int im2col_step) +{ + if (input.type().is_cuda()) + { +#ifdef WITH_CUDA + return modulated_deform_conv2d_cuda_backward(input, + weight, + bias, + offset, + mask, + grad_output, + kernel_h, kernel_w, + stride_h, stride_w, + pad_h, pad_w, + dilation_h, dilation_w, + group, + deformable_group, + im2col_step); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} + diff --git a/segmentation/mm_modules/DCN/src/vision.cpp b/segmentation/mm_modules/DCN/src/vision.cpp new file mode 100644 index 0000000..5043fea --- /dev/null +++ b/segmentation/mm_modules/DCN/src/vision.cpp @@ -0,0 +1,10 @@ + +#include "deform_conv2d.h" +#include "modulated_deform_conv2d.h" + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("deform_conv2d_forward", &deform_conv2d_forward, "deform_conv2d_forward"); + m.def("deform_conv2d_backward", &deform_conv2d_backward, "deform_conv2d_backward"); + m.def("modulated_deform_conv2d_forward", &modulated_deform_conv2d_forward, "modulated_deform_conv2d_forward"); + m.def("modulated_deform_conv2d_backward", &modulated_deform_conv2d_backward, "modulated_deform_conv2d_backward"); +} diff --git a/segmentation/mm_modules/__init__.py b/segmentation/mm_modules/__init__.py new file mode 100644 index 0000000..faaaf79 --- /dev/null +++ b/segmentation/mm_modules/__init__.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + + diff --git a/segmentation/mm_modules/cocoapi_evaluator.py b/segmentation/mm_modules/cocoapi_evaluator.py new file mode 100644 index 0000000..0d47637 --- /dev/null +++ b/segmentation/mm_modules/cocoapi_evaluator.py @@ -0,0 +1,209 @@ +import json +import tempfile +import sys +from tqdm import tqdm + +from pycocotools.cocoeval import COCOeval +from torch.autograd import Variable + +from dataset.cocodataset import * +from dataset.data_augment import ValTransform +from utils.utils import * +from utils import distributed_util +from utils.vis_utils import make_vis, make_pred_vis +import time +import apex + +DEBUG =False + +def _accumulate_predictions_from_multiple_gpus(predictions_per_gpu): + all_predictions = distributed_util.scatter_gather(predictions_per_gpu) + if not distributed_util.is_main_process(): + return + # merge the list of dicts + predictions = [] + for p in all_predictions: + for a in p: + predictions.append(a) + + return predictions + +class COCOAPIEvaluator(): + """ + COCO AP Evaluation class. + All the data in the val2017 dataset are processed \ + and evaluated by COCO API. + """ + def __init__(self, data_dir, img_size, confthre, nmsthre, testset=False, voc=False, vis=False): + """ + Args: + data_dir (str): dataset root directory + img_size (int): image size after preprocess. images are resized \ + to squares whose shape is (img_size, img_size). + confthre (float): + confidence threshold ranging from 0 to 1, \ + which is defined in the config file. + nmsthre (float): + IoU threshold of non-max supression ranging from 0 to 1. + """ + json_f = 'instances_val2017.json' + name='val2017' + if testset: + json_f = 'image_info_test-dev2017.json' + name='test2017' + if voc: + json_f = 'pascal_test2007.json' + + self.testset= testset + self.dataset = COCODataset(data_dir=data_dir, + img_size=img_size, + json_file=json_f, + preproc = ValTransform(rgb_means=(0.485, 0.456, 0.406),std=(0.229, 0.224, 0.225)), + name=name, + voc = voc) + + self.num_images = len(self.dataset) + self.dataloader = torch.utils.data.DataLoader( + self.dataset, batch_size=1, shuffle=False, num_workers=0) + self.img_size = img_size + self.confthre = confthre + self.nmsthre = nmsthre + self.voc = voc + self.vis = vis + + def evaluate(self, model, half=False, distributed=False): + """ + COCO average precision (AP) Evaluation. Iterate inference on the test dataset + and the results are evaluated by COCO API. + Args: + model : model object + Returns: + ap50_95 (float) : calculated COCO AP for IoU=50:95 + ap50 (float) : calculated COCO AP for IoU=50 + """ + if isinstance(model, apex.parallel.DistributedDataParallel): + model = model.module + distributed=True + + model=model.eval() + cuda = torch.cuda.is_available() + if half: + Tensor = torch.cuda.HalfTensor if cuda else torch.HalfTensor + else: + Tensor = torch.cuda.FloatTensor if cuda else torch.FloatTensor + ids = [] + data_dict = [] + img_num = 0 + + indices = list(range(self.num_images)) + if distributed: + dis_indices = indices[distributed_util.get_rank()::distributed_util.get_world_size()] + else: + dis_indices = indices + progress_bar = tqdm if distributed_util.is_main_process() else iter + num_classes = 80 if not self.voc else 20 + + inference_time=0 + nms_time=0 + n_samples=len(dis_indices)-10 + + for k, i in enumerate(progress_bar(dis_indices)): + img, _, info_img, id_ = self.dataset[i] # load a batch + info_img = [float(info) for info in info_img] + id_ = int(id_) + ids.append(id_) + with torch.no_grad(): + img = Variable(img.type(Tensor).unsqueeze(0)) + if k > 9: + start=time.time() + + if self.vis: + outputs,fuse_weights,fused_f = model(img) + else: + outputs = model(img) + + if k > 9: + infer_end=time.time() + inference_time += (infer_end-start) + + outputs = postprocess( + outputs, num_classes, self.confthre, self.nmsthre) + + if k > 9: + nms_end=time.time() + nms_time +=(nms_end-infer_end) + + if outputs[0] is None: + continue + outputs = outputs[0].cpu().data + + bboxes = outputs[:, 0:4] + bboxes[:, 0::2] *= info_img[0] / self.img_size[0] + bboxes[:, 1::2] *= info_img[1] / self.img_size[1] + bboxes[:, 2] = bboxes[:,2] - bboxes[:,0] + bboxes[:, 3] = bboxes[:,3] - bboxes[:,1] + cls = outputs[:, 6] + scores = outputs[:, 4]* outputs[:,5] + for ind in range(bboxes.shape[0]): + label = self.dataset.class_ids[int(cls[ind])] + A = {"image_id": id_, "category_id": label, "bbox": bboxes[ind].numpy().tolist(), + "score": scores[ind].numpy().item(), "segmentation": []} # COCO json format + data_dict.append(A) + + if self.vis: + o_img,_,_,_ = self.dataset.pull_item(i) + make_vis('COCO', i, o_img, fuse_weights, fused_f) + class_names = self.dataset._classes + make_pred_vis('COCO', i, o_img, class_names, bboxes, cls, scores) + + if DEBUG and distributed_util.is_main_process(): + o_img,_ = self.dataset.pull_item(i) + class_names = self.dataset._classes + make_pred_vis('COCO', i, o_img, class_names, bboxes, cls, scores) + + if distributed: + distributed_util.synchronize() + data_dict = _accumulate_predictions_from_multiple_gpus(data_dict) + inference_time = torch.FloatTensor(1).type(Tensor).fill_(inference_time) + nms_time = torch.FloatTensor(1).type(Tensor).fill_(nms_time) + n_samples = torch.LongTensor(1).type(Tensor).fill_(n_samples) + distributed_util.synchronize() + torch.distributed.reduce(inference_time, dst=0) + torch.distributed.reduce(nms_time, dst=0) + torch.distributed.reduce(n_samples, dst=0) + inference_time = inference_time.item() + nms_time = nms_time.item() + n_samples = n_samples.item() + + if not distributed_util.is_main_process(): + return 0, 0 + + + print('Main process Evaluating...') + + annType = ['segm', 'bbox', 'keypoints'] + a_infer_time = 1000*inference_time / (n_samples) + a_nms_time= 1000*nms_time / (n_samples) + + print('Average forward time: %.2f ms, Average NMS time: %.2f ms, Average inference time: %.2f ms' %(a_infer_time, \ + a_nms_time, (a_infer_time+a_nms_time))) + + # Evaluate the Dt (detection) json comparing with the ground truth + if len(data_dict) > 0: + cocoGt = self.dataset.coco + # workaround: temporarily write data to json file because pycocotools can't process dict in py36. + if self.testset: + json.dump(data_dict, open('yolov3_2017.json', 'w')) + cocoDt = cocoGt.loadRes('yolov3_2017.json') + else: + _, tmp = tempfile.mkstemp() + json.dump(data_dict, open(tmp, 'w')) + cocoDt = cocoGt.loadRes(tmp) + cocoEval = COCOeval(self.dataset.coco, cocoDt, annType[1]) + cocoEval.evaluate() + cocoEval.accumulate() + cocoEval.summarize() + return cocoEval.stats[0], cocoEval.stats[1] + else: + return 0, 0 + diff --git a/segmentation/mm_modules/distributed_util.py b/segmentation/mm_modules/distributed_util.py new file mode 100644 index 0000000..dcd2479 --- /dev/null +++ b/segmentation/mm_modules/distributed_util.py @@ -0,0 +1,162 @@ +import os +import pickle +import tempfile +import time + +import torch + + +def get_world_size(): + if not torch.distributed.is_initialized(): + return 1 + return torch.distributed.get_world_size() + + +def get_rank(): + if not torch.distributed.is_initialized(): + return 0 + return torch.distributed.get_rank() + + +def is_main_process(): + if not torch.distributed.is_initialized(): + return True + return torch.distributed.get_rank() == 0 + + +def synchronize(): + """ + Helper function to synchronize between multiple processes when + using distributed training + """ + if not torch.distributed.is_initialized(): + return + world_size = torch.distributed.get_world_size() + rank = torch.distributed.get_rank() + if world_size == 1: + return + + def _send_and_wait(r): + if rank == r: + tensor = torch.tensor(0, device="cuda") + else: + tensor = torch.tensor(1, device="cuda") + torch.distributed.broadcast(tensor, r) + while tensor.item() == 1: + time.sleep(1) + + _send_and_wait(0) + # now sync on the main process + _send_and_wait(1) + + +def _encode(encoded_data, data): + # gets a byte representation for the data + encoded_bytes = pickle.dumps(data) + # convert this byte string into a byte tensor + storage = torch.ByteStorage.from_buffer(encoded_bytes) + tensor = torch.ByteTensor(storage).to("cuda") + # encoding: first byte is the size and then rest is the data + s = tensor.numel() + assert s <= 255, "Can't encode data greater than 255 bytes" + # put the encoded data in encoded_data + encoded_data[0] = s + encoded_data[1: (s + 1)] = tensor + + +def _decode(encoded_data): + size = encoded_data[0] + encoded_tensor = encoded_data[1: (size + 1)].to("cpu") + return pickle.loads(bytearray(encoded_tensor.tolist())) + + +# TODO try to use tensor in shared-memory instead of serializing to disk +# this involves getting the all_gather to work +def scatter_gather(data): + """ + This function gathers data from multiple processes, and returns them + in a list, as they were obtained from each process. + This function is useful for retrieving data from multiple processes, + when launching the code with torch.distributed.launch + Note: this function is slow and should not be used in tight loops, i.e., + do not use it in the training loop. + Arguments: + data: the object to be gathered from multiple processes. + It must be serializable + Returns: + result (list): a list with as many elements as there are processes, + where each element i in the list corresponds to the data that was + gathered from the process of rank i. + """ + # strategy: the main process creates a temporary directory, and communicates + # the location of the temporary directory to all other processes. + # each process will then serialize the data to the folder defined by + # the main process, and then the main process reads all of the serialized + # files and returns them in a list + if not torch.distributed.is_initialized(): + return [data] + synchronize() + # get rank of the current process + rank = torch.distributed.get_rank() + + # the data to communicate should be small + data_to_communicate = torch.empty(256, dtype=torch.uint8, device="cuda") + if rank == 0: + # manually creates a temporary directory, that needs to be cleaned + # afterwards + tmp_dir = tempfile.mkdtemp() + _encode(data_to_communicate, tmp_dir) + + synchronize() + # the main process (rank=0) communicates the data to all processes + torch.distributed.broadcast(data_to_communicate, 0) + + # get the data that was communicated + tmp_dir = _decode(data_to_communicate) + + # each process serializes to a different file + file_template = "file{}.pth" + tmp_file = os.path.join(tmp_dir, file_template.format(rank)) + torch.save(data, tmp_file) + + # synchronize before loading the data + synchronize() + + # only the master process returns the data + if rank == 0: + data_list = [] + world_size = torch.distributed.get_world_size() + for r in range(world_size): + file_path = os.path.join(tmp_dir, file_template.format(r)) + d = torch.load(file_path) + data_list.append(d) + # cleanup + os.remove(file_path) + # cleanup + os.rmdir(tmp_dir) + return data_list + + +def reduce_loss_dict(loss_dict): + """ + Reduce the loss dictionary from all processes so that process with rank + 0 has the averaged results. Returns a dict with the same fields as + loss_dict, after reduction. + """ + world_size = get_world_size() + if world_size < 2: + return loss_dict + with torch.no_grad(): + loss_names = [] + all_losses = [] + for k in sorted(loss_dict.keys()): + loss_names.append(k) + all_losses.append(loss_dict[k]) + all_losses = torch.stack(all_losses, dim=0) + torch.distributed.reduce(all_losses, dst=0) + if torch.distributed.get_rank() == 0: + # only main process gets accumulated, so only divide by + # world_size in this case + all_losses /= world_size + reduced_losses = {k: v for k, v in zip(loss_names, all_losses)} + return reduced_losses diff --git a/segmentation/mm_modules/fp16_utils/README.md b/segmentation/mm_modules/fp16_utils/README.md new file mode 100644 index 0000000..941de17 --- /dev/null +++ b/segmentation/mm_modules/fp16_utils/README.md @@ -0,0 +1,16 @@ +fp16_optimizer.py contains `FP16_Optimizer`, a Python class designed to wrap an existing Pytorch optimizer and automatically enable master parameters and loss scaling in a manner transparent to the user. To use `FP16_Optimizer`, only two lines of one's Python model need to change. + +#### [FP16_Optimizer API documentation](https://nvidia.github.io/apex/fp16_utils.html#automatic-management-of-master-params-loss-scaling) + +#### [Simple examples with FP16_Optimizer](https://github.com/NVIDIA/apex/tree/master/examples/FP16_Optimizer_simple) + +#### [Imagenet with FP16_Optimizer](https://github.com/NVIDIA/apex/tree/master/examples/imagenet) + +#### [word_language_model with FP16_Optimizer](https://github.com/NVIDIA/apex/tree/master/examples/word_language_model) + + +fp16_util.py contains a number of utilities to manually manage master parameters and loss scaling, if the user chooses. + +#### [Manual management documentation](https://nvidia.github.io/apex/fp16_utils.html#manual-master-parameter-management) + +The [Imagenet with FP16_Optimizer](https://github.com/NVIDIA/apex/tree/master/examples/imagenet) and [word_language_model with FP16_Optimizer](https://github.com/NVIDIA/apex/tree/master/examples/word_language_model) directories also contain `main.py` files that demonstrate manual management of master parameters and static loss scaling. These examples illustrate what sort of operations `FP16_Optimizer` is performing automatically. diff --git a/segmentation/mm_modules/fp16_utils/__init__.py b/segmentation/mm_modules/fp16_utils/__init__.py new file mode 100644 index 0000000..c7bb1f5 --- /dev/null +++ b/segmentation/mm_modules/fp16_utils/__init__.py @@ -0,0 +1,16 @@ +from .fp16util import ( + BN_convert_float, + network_to_half, + prep_param_lists, + model_grads_to_master_grads, + master_params_to_model_params, + tofp16, + to_python_float, + clip_grad_norm, + convert_module, + convert_network, + FP16Model, +) + +from .fp16_optimizer import FP16_Optimizer +from .loss_scaler import LossScaler, DynamicLossScaler diff --git a/segmentation/mm_modules/fp16_utils/fp16_optimizer.py b/segmentation/mm_modules/fp16_utils/fp16_optimizer.py new file mode 100644 index 0000000..fe999e0 --- /dev/null +++ b/segmentation/mm_modules/fp16_utils/fp16_optimizer.py @@ -0,0 +1,561 @@ +import torch +from torch import nn +from torch.autograd import Variable +from torch.nn.parameter import Parameter +from torch._utils import _flatten_dense_tensors, _unflatten_dense_tensors + +from .loss_scaler import DynamicLossScaler, LossScaler +from .fp16util import model_grads_to_master_grads, master_params_to_model_params, clip_grad_norm + +# TODO: Update overflow check + downscale to use Carl's fused kernel. +class FP16_Optimizer(object): + """ + :class:`FP16_Optimizer` is designed to wrap an existing PyTorch optimizer, + and manage static or dynamic loss scaling and master weights in a manner transparent to the user. + For standard use, only two lines must be changed: creating the :class:`FP16_Optimizer` instance, + and changing the call to ``backward``. + + Example:: + + model = torch.nn.Linear(D_in, D_out).cuda().half() + optimizer = torch.optim.SGD(model.parameters(), lr=1e-3) + # Name the FP16_Optimizer instance to replace the existing optimizer + # (recommended but not required): + optimizer = FP16_Optimizer(optimizer, static_loss_scale = 128.0) + ... + # loss.backward() becomes: + optimizer.backward(loss) + ... + + Example with dynamic loss scaling:: + + ... + optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True) + # optional arg to control dynamic loss scaling behavior + # dynamic_loss_args={'scale_window' : 500}) + # Usually, dynamic_loss_args is not necessary. + + Args: + init_optimizer (torch.optim.optimizer): Existing optimizer created with the parameters to optimize. Internally, :class:`FP16_Optimizer` replaces the passed optimizer's fp16 parameters, if any, with fp32 master parameters copied from the original ones. :class:`FP16_Optimizer` also stores references to the original fp16 parameters, and updates these fp16 parameters from the master fp32 copy at the end of each :attr:`step`. + static_loss_scale (float, optional, default=1.0): Loss scale used internally to scale gradients computed by the model. Any fp16 gradients will be copied to fp32, then downscaled before being applied to the fp32 master params, so ``static_loss_scale`` should not affect learning rate. + dynamic_loss_scale (bool, optional, default=False): Use dynamic loss scaling. If True, this will override any ``static_loss_scale`` option. + dynamic_loss_args (dict, optional, default=None): Dict of kwargs that will be forwarded to the internal :class:`DynamicLossScaler` instance's constructor. Keys of this dict must match kwargs accepted by :class:`DynamicLossScaler`'s constructor. If ``dynamic_loss_args`` is unspecified, :class:`DynamicLossScaler`'s defaults will be used. + verbose (bool, optional, default=True): By default, FP16_Optimizer's constructor prints out the parameters and parameter groups it is ingesting, as a sanity check. If this becomes annoying (e.g. for large models), it can be disabled by passing ``verbose=False``. ``verbose=False`` will not disable printing when the loss scale is readjusted during dynamic loss scaling. + + ``init_optimizer`` is expected to have been constructed in the ordinary way. + It is recommended (although not required) that the newly constructed :class:`FP16_Optimizer` instance be + named to replace ``init_optimizer``, for two reasons: + First, it means that references to the same name + later in the file will not have to change. + Second, :class:`FP16_Optimizer` reserves the right (as an implementation detail) to + modify ``init_optimizer``. If you do choose a unique name for the new + :class:`FP16_Optimizer` instance, you should only work with this new instance, + because the preexisting optimizer might no longer behave as expected. + + ``init_optimizer`` may be any Pytorch optimizer. + It may contain a mixture of fp16 and fp32 parameters organized into any number of + ``param_groups`` with different hyperparameters. The :class:`FP16_Optimizer` constructor will + ingest these ``param_groups`` and remember them. + + Calls to :: + + loss.backward() + + must be replaced with :: + + optimizer.backward(loss) + + because :class:`FP16_Optimizer` requires ownership of the backward pass to implement + loss scaling and copies to master gradients. + + .. note:: + Loss scaling, either static or dynamic, is orthogonal to learning rate, because gradients + are downscaled before being applied. This means that adjusting the loss scale, or using + dynamic loss scaling, should not require retuning the learning rate or any other + hyperparameters. + + + **Advanced options** + + **Closures**: :class:`FP16_Optimizer` can wrap a Pytorch optimizer that receives a closure. + See docstring for :attr:`step`. + + **Gradient clipping**: Use :attr:`clip_master_grads`. + + **Multiple losses**: If your model accumulates gradients from multiple losses, + this can be made more efficient by supplying ``update_master_grads=False`` + to :attr:`backward`. See docstring for :attr:`backward`. + + **Manually adjusting loss scale**: The current loss scale can be retrieved or set via :: + + print(optimizer.loss_scale) + optimizer.loss_scale = new_loss_scale + + For static loss scaling, manually adjusting the loss scale over time is a reasonable + thing to do. During later epochs, gradients may become smaller, and a + higher loss scale may be required, analogous to scheduling the learning rate. Dynamic loss + scaling is more subtle (see :class:`DynamicLossScaler`) and in this case, manually adjusting + the loss scale is not recommended. + + **Multi_GPU training**: If the wrapped ``init_optimizer`` was created from a model wrapped in + Pytorch DistributedDataParallel or Apex DistributedDataParallel, :class:`FP16_Optimizer` + should still work as intended. + """ + + def __init__(self, + init_optimizer, + static_loss_scale=1.0, + dynamic_loss_scale=False, + dynamic_loss_args=None, + verbose=True): + if not torch.cuda.is_available: + raise SystemError("Cannot use fp16 without CUDA.") + + self.verbose = verbose + + self.optimizer = init_optimizer + # init_state_dict sets up an alternative way to cast per-param state tensors. + # Stashing here in case https://github.com/pytorch/pytorch/issues/7733 makes it necessary. + # init_state_dict = init_optimizer.state_dict() + + self.fp16_groups = [] + self.fp32_from_fp16_groups = [] + self.fp32_from_fp32_groups = [] + for i, param_group in enumerate(self.optimizer.param_groups): + self.maybe_print("FP16_Optimizer processing param group {}:".format(i)) + fp16_params_this_group = [] + fp32_params_this_group = [] + fp32_from_fp16_params_this_group = [] + for i, param in enumerate(param_group['params']): + if param.requires_grad: + if param.type() == 'torch.cuda.HalfTensor': + self.maybe_print("FP16_Optimizer received torch.cuda.HalfTensor with {}" + .format(param.size())) + fp16_params_this_group.append(param) + master_param = param.detach().clone().float() + master_param.requires_grad = True + param_group['params'][i] = master_param + fp32_from_fp16_params_this_group.append(master_param) + # Reset existing state dict key to the new master param. + # We still need to recast per-param state tensors, if any, to FP32. + if param in self.optimizer.state: + self.optimizer.state[master_param] = self.optimizer.state.pop(param) + elif param.type() == 'torch.cuda.FloatTensor': + self.maybe_print("FP16_Optimizer received torch.cuda.FloatTensor with {}" + .format(param.size())) + fp32_params_this_group.append(param) + param_group['params'][i] = param + else: + raise TypeError("Wrapped parameters must be either " + "torch.cuda.FloatTensor or torch.cuda.HalfTensor. " + "Received {}".format(param.type())) + + self.fp16_groups.append(fp16_params_this_group) + self.fp32_from_fp16_groups.append(fp32_from_fp16_params_this_group) + self.fp32_from_fp32_groups.append(fp32_params_this_group) + + # Leverage state_dict() and load_state_dict() to recast preexisting per-param state tensors + self.optimizer.load_state_dict(self.optimizer.state_dict()) + # alternative way to cast per-param state tensors: + # self.optimizer.load_state_dict(init_state_dict) + + if dynamic_loss_scale: + self.dynamic_loss_scale = True + if dynamic_loss_args is not None: + self.loss_scaler = DynamicLossScaler(**dynamic_loss_args) + else: + self.loss_scaler = DynamicLossScaler() + else: + self.dynamic_loss_scale = False + self.loss_scaler = LossScaler(static_loss_scale) + + self.overflow = False + self.first_closure_call_this_step = True + + self.clip_grad_norm = clip_grad_norm + + def maybe_print(self, msg): + if self.verbose: + print(msg) + + def __getstate__(self): + raise RuntimeError("FP16_Optimizer should be serialized using state_dict().") + + def __setstate__(self, state): + raise RuntimeError("FP16_Optimizer should be deserialized using load_state_dict().") + + def zero_grad(self, set_grads_to_None=False): + """ + Zero fp32 and fp16 parameter grads. + """ + # In principle, only the .grad attributes of the model params need to be zeroed, + # because gradients are copied into the FP32 master params. However, we zero + # all gradients owned by the optimizer, just to be safe: + for group in self.optimizer.param_groups: + for p in group['params']: + if set_grads_to_None: + p.grad = None + else: + if p.grad is not None: + p.grad.detach_() + p.grad.zero_() + + # Zero fp16 gradients owned by the model: + for fp16_group in self.fp16_groups: + for param in fp16_group: + if set_grads_to_None: + param.grad = None + else: + if param.grad is not None: + param.grad.detach_() # as in torch.optim.optimizer.zero_grad() + param.grad.zero_() + + def _check_overflow(self): + params = [] + for group in self.fp16_groups: + for param in group: + params.append(param) + for group in self.fp32_from_fp32_groups: + for param in group: + params.append(param) + self.overflow = self.loss_scaler.has_overflow(params) + + def _update_scale(self, has_overflow=False): + self.loss_scaler.update_scale(has_overflow) + + def _master_params_to_model_params(self): + for fp16_group, fp32_from_fp16_group in zip(self.fp16_groups, self.fp32_from_fp16_groups): + master_params_to_model_params(fp16_group, fp32_from_fp16_group) + + # To consider: Integrate distributed with this wrapper by registering a hook on each variable + # that does the overflow check, gradient copy + downscale, and fp32 allreduce in a different stream. + def _model_grads_to_master_grads(self): + for fp16_group, fp32_from_fp16_group in zip(self.fp16_groups, self.fp32_from_fp16_groups): + model_grads_to_master_grads(fp16_group, fp32_from_fp16_group) + + def _downscale_master(self): + if self.loss_scale != 1.0: + for group in self.optimizer.param_groups: + for param in group['params']: + if param.grad is not None: + param.grad.data.mul_(1./self.loss_scale) + + def clip_master_grads(self, max_norm, norm_type=2): + """ + Clips fp32 master gradients via ``torch.nn.utils.clip_grad_norm``. + + Args: + max_norm (float or int): max norm of the gradients + norm_type (float or int): type of the used p-norm. Can be ``'inf'`` for + infinity norm. + + Returns: + Total norm of the current fp32 gradients (viewed as a single vector). + + .. warning:: + Returns -1 if the most recently computed fp16 gradients overflowed (that is, if ``self.overflow`` is ``True``). + """ + if not self.overflow: + fp32_params = [] + for param_group in self.optimizer.param_groups: + for param in param_group['params']: + fp32_params.append(param) + return self.clip_grad_norm(fp32_params, max_norm, norm_type) + else: + return -1 + + def state_dict(self): + """ + Returns a dict containing the current state of this :class:`FP16_Optimizer` instance. + This dict contains attributes of :class:`FP16_Optimizer`, as well as the state_dict + of the contained Pytorch optimizer. + Example:: + + checkpoint = {} + checkpoint['model'] = model.state_dict() + checkpoint['optimizer'] = optimizer.state_dict() + torch.save(checkpoint, "saved.pth") + """ + state_dict = {} + state_dict['loss_scaler'] = self.loss_scaler + state_dict['dynamic_loss_scale'] = self.dynamic_loss_scale + state_dict['overflow'] = self.overflow + state_dict['first_closure_call_this_step'] = self.first_closure_call_this_step + state_dict['optimizer_state_dict'] = self.optimizer.state_dict() + state_dict['fp32_from_fp16'] = self.fp32_from_fp16_groups + return state_dict + + def load_state_dict(self, state_dict): + """ + Loads a state_dict created by an earlier call to state_dict(). + If ``fp16_optimizer_instance`` was constructed from some ``init_optimizer``, + whose parameters in turn came from ``model``, it is expected that the user + will call ``model.load_state_dict()`` before + ``fp16_optimizer_instance.load_state_dict()`` is called. + + Example:: + + model = torch.nn.Linear(D_in, D_out).cuda().half() + optimizer = torch.optim.SGD(model.parameters(), lr=1e-3) + optimizer = FP16_Optimizer(optimizer, static_loss_scale = 128.0) + ... + checkpoint = torch.load("saved.pth") + model.load_state_dict(checkpoint['model']) + optimizer.load_state_dict(checkpoint['optimizer']) + """ + # I think it should actually be ok to reload the optimizer before the model. + self.loss_scaler = state_dict['loss_scaler'] + self.dynamic_loss_scale = state_dict['dynamic_loss_scale'] + self.overflow = state_dict['overflow'] + self.first_closure_call_this_step = state_dict['first_closure_call_this_step'] + self.optimizer.load_state_dict(state_dict['optimizer_state_dict']) + # At this point, the optimizer's references to the model's fp32 parameters are up to date. + # The optimizer's hyperparameters and internal buffers are also up to date. + # However, the fp32 master copies of the model's fp16 params stored by the optimizer are still + # out of date. There are two options. + # 1: Refresh the master params from the model's fp16 params. + # This requires less storage but incurs precision loss. + # 2: Save and restore the fp32 master copies separately. + # We choose option 2. + # + # Pytorch Optimizer.load_state_dict casts saved buffers (e.g. momentum) to the type and device + # of their associated parameters, because it's possible those buffers might not exist yet in + # the current optimizer instance. In our case, as long as the current FP16_Optimizer has been + # constructed in the same way as the one whose state_dict we are loading, the same master params + # are guaranteed to exist, so we can just copy_() from the saved master params. + for current_group, saved_group in zip(self.fp32_from_fp16_groups, state_dict['fp32_from_fp16']): + for current, saved in zip(current_group, saved_group): + current.data.copy_(saved.data) + + def step(self, closure=None): # could add clip option. + """ + If no closure is supplied, :attr:`step` should be called after + ``fp16_optimizer_obj.backward(loss)``. + :attr:`step` updates the fp32 master copy of parameters using the optimizer supplied to + :class:`FP16_Optimizer`'s constructor, then copies the updated fp32 params into the fp16 params + originally referenced by :class:`FP16_Optimizer`'s constructor, so the user may immediately run + another forward pass using their model. + + If a closure is supplied, :attr:`step` may be called without a prior call to + :attr:`backward(loss)`. + This control flow is identical to `ordinary Pytorch optimizer use`_ with closures. + However, the user should take care that any ``loss.backward()`` call within the closure + has been replaced by ``fp16_optimizer_obj.backward(loss)``. + + Args: + closure (optional): Closure that will be supplied to the underlying optimizer originally passed to :class:`FP16_Optimizer`'s constructor. closure should call :attr:`zero_grad()` on the :class:`FP16_Optimizer` object, compute the loss, call :attr:`backward(loss)`, and return the loss. + + Example with closure:: + + # optimizer is assumed to be an FP16_Optimizer object, previously constructed from an + # existing pytorch optimizer. + for input, target in dataset: + def closure(): + optimizer.zero_grad() + output = model(input) + loss = loss_fn(output, target) + # loss.backward() becomes: + optimizer.backward(loss) + return loss + optimizer.step(closure) + + .. warning:: + Currently, calling :attr:`step` with a closure is not compatible with dynamic loss scaling. + + .. _`ordinary Pytorch optimizer use`: + http://pytorch.org/docs/master/optim.html#optimizer-step-closure + """ + + scale = self.loss_scaler.loss_scale + self._update_scale(self.overflow) + + if self.overflow: + print("OVERFLOW! Skipping step. Attempted loss scale: {}, reducing to {}" + .format(scale, self.loss_scale)) + return + + if closure is not None: + retval = self._step_with_closure(closure) + else: + retval = self.optimizer.step() + + self._master_params_to_model_params() + + return retval + + def _step_with_closure(self, closure): + def wrapped_closure(): + # helpful for debugging + # print("Calling wrapped_closure, first_closure_call_this_step = {}" + # .format(self.first_closure_call_this_step)) + if self.first_closure_call_this_step: + # We expect that the fp16 params are initially fresh on entering self.step(), + # so _master_params_to_model_params() is unnecessary the first time wrapped_closure() + # is called within self.optimizer.step(). + self.first_closure_call_this_step = False + else: + # If self.optimizer.step() internally calls wrapped_closure more than once, + # it may update the fp32 params after each call. However, self.optimizer + # doesn't know about the fp16 params at all. If the fp32 params get updated, + # we can't rely on self.optimizer to refresh the fp16 params. We need + # to handle that manually: + self._master_params_to_model_params() + # Our API expects the user to give us ownership of the backward() call by + # replacing all calls to loss.backward() with optimizer.backward(loss). + # This requirement holds whether or not the call to backward() is made within a closure. + # If the user is properly calling optimizer.backward(loss) within "closure," + # calling closure() here will give the fp32 master params fresh gradients + # for the optimizer to play with, so all wrapped_closure needs to do is call + # closure() and return the loss. + temp_loss = closure() + while(self.overflow): + scale = self.loss_scaler.loss_scale + self._update_scale(self.overflow) + print("OVERFLOW within closure! Skipping step. Attempted loss scale: {}, " + "reducing to {}".format(scale, self.loss_scale)) + temp_loss = closure() + return temp_loss + + retval = self.optimizer.step(wrapped_closure) + + self.first_closure_call_this_step = True + + return retval + + def backward(self, loss, update_master_grads=True, retain_graph=False): + """ + :attr:`backward` performs the following conceptual steps: + + 1. fp32_loss = loss.float() (see first Note below) + 2. scaled_loss = fp32_loss*loss_scale + 3. scaled_loss.backward(), which accumulates scaled gradients into the ``.grad`` attributes of the model's leaves (which may be fp16, fp32, or a mixture, depending how your model was defined). + 4. fp16 grads are then copied to the master params' ``.grad`` attributes (see second Note), which are guaranteed to be fp32. + 5. Finally, master grads are divided by loss_scale. + + In this way, after :attr:`backward`, the master params have fresh gradients, + and :attr:`step` may be called. + + .. note:: + :attr:`backward` internally converts the loss to fp32 before applying the loss scale. + This provides some additional safety against overflow if the user has supplied an + fp16 loss value. + However, for maximum overflow safety, the user should + compute the loss criterion (MSE, cross entropy, etc) in fp32 before supplying it to + :attr:`backward`. + + .. warning:: + The gradients found in a model's leaves after the call to + :attr:`backward` should not be regarded as valid in general, + because it's possible + they have been scaled (and in the case of dynamic loss scaling, + the scale factor may change over time). + If the user wants to inspect gradients after a call to :attr:`backward`, + only the master gradients should be regarded as valid. These can be retrieved via + :attr:`inspect_master_grad_data()`. + + Args: + loss: The loss output by the user's model. loss may be either float or half (but see first Note above). + update_master_grads (bool, optional, default=True): Option to copy fp16 grads to fp32 grads on this call. By setting this to False, the user can delay the copy, which is useful to eliminate redundant fp16->fp32 grad copies if :attr:`backward` is being called on multiple losses in one iteration. If set to False, the user becomes responsible for calling :attr:`update_master_grads` before calling :attr:`step`. + retain_graph (bool, optional, default=False): Forwards the usual ``retain_graph=True`` option to the internal call to ``loss.backward``. If ``retain_graph`` is being used to accumulate gradient values from multiple backward passes before calling ``optimizer.step``, passing ``update_master_grads=False`` is also recommended (see Example below). + + Example:: + + # Ordinary operation: + optimizer.backward(loss) + + # Naive operation with multiple losses (technically valid, but less efficient): + # fp32 grads will be correct after the second call, but + # the first call incurs an unnecessary fp16->fp32 grad copy. + optimizer.backward(loss1) + optimizer.backward(loss2) + + # More efficient way to handle multiple losses: + # The fp16->fp32 grad copy is delayed until fp16 grads from all + # losses have been accumulated. + optimizer.backward(loss1, update_master_grads=False) + optimizer.backward(loss2, update_master_grads=False) + optimizer.update_master_grads() + """ + # To consider: try multiple backward passes using retain_grad=True to find + # a loss scale that works. After you find a loss scale that works, do a final dummy + # backward pass with retain_graph=False to tear down the graph. Doing this would avoid + # discarding the iteration, but probably wouldn't improve overall efficiency. + self.loss_scaler.backward(loss.float(), retain_graph=retain_graph) + if update_master_grads: + self.update_master_grads() + + def update_master_grads(self): + """ + Copy the ``.grad`` attribute from stored references to fp16 parameters to + the ``.grad`` attribute of the fp32 master parameters that are directly + updated by the optimizer. :attr:`update_master_grads` only needs to be called if + ``fp16_optimizer_obj.backward`` was called with ``update_master_grads=False``. + """ + if self.dynamic_loss_scale: + self._check_overflow() + if self.overflow: return + self._model_grads_to_master_grads() + self._downscale_master() + + def inspect_master_grad_data(self): + """ + When running with :class:`FP16_Optimizer`, + ``.grad`` attributes of a model's fp16 leaves should not be + regarded as truthful, because they might be scaled. + After a call to :attr:`fp16_optimizer_obj.backward(loss)`, if no overflow was encountered, + the fp32 master params' ``.grad`` + attributes will contain valid gradients properly divided by the loss scale. However, + because :class:`FP16_Optimizer` flattens some parameters, accessing them may be + nonintuitive. :attr:`inspect_master_grad_data` + allows those gradients to be viewed with shapes corresponding to their associated model leaves. + + Returns: + List of lists (one list for each parameter group). The list for each parameter group + is a list of the ``.grad.data`` attributes of the fp32 master params belonging to that group. + """ + if self.overflow: + print("Warning: calling FP16_Optimizer.inspect_master_grad_data while in an overflow state. " + "Gradients are currently invalid (may be inf, nan, or stale). Returning None.") + return None + else: + # The optimizer owns only references to master params. + master_grads_data = [] + for param_group in self.optimizer.param_groups: + master_grads_this_group = [] + for param in param_group['params']: + if param.grad is not None: + master_grads_this_group.append(param.grad.data) + else: + master_grads_this_group.append(None) + master_grads_data.append(master_grads_this_group) + return master_grads_data + + + # Promote loss scale so it can be retrieved or set via "fp16_optimizer_instance.loss_scale" + def _get_loss_scale(self): + return self.loss_scaler.loss_scale + + def _set_loss_scale(self, value): + self.loss_scaler.cur_scale = value + + loss_scale = property(_get_loss_scale, _set_loss_scale) + + # Promote state so it can be retrieved or set via "fp16_optimizer_instance.state" + def _get_state(self): + return self.optimizer.state + + def _set_state(self, value): + self.optimizer.state = value + + state = property(_get_state, _set_state) + + # Promote param_groups so it can be retrieved or set via "fp16_optimizer_instance.param_groups" + # (for example, to adjust the learning rate) + def _get_param_groups(self): + return self.optimizer.param_groups + + def _set_param_groups(self, value): + self.optimizer.param_groups = value + + param_groups = property(_get_param_groups, _set_param_groups) + diff --git a/segmentation/mm_modules/fp16_utils/fp16util.py b/segmentation/mm_modules/fp16_utils/fp16util.py new file mode 100644 index 0000000..66c13e4 --- /dev/null +++ b/segmentation/mm_modules/fp16_utils/fp16util.py @@ -0,0 +1,185 @@ +import torch +import torch.nn as nn +from torch.autograd import Variable +from torch._utils import _flatten_dense_tensors, _unflatten_dense_tensors + + +class tofp16(nn.Module): + """ + Utility module that implements:: + + def forward(self, input): + return input.half() + """ + + def __init__(self): + super(tofp16, self).__init__() + + def forward(self, input): + return input.half() + + +def BN_convert_float(module): + """ + Utility function for network_to_half(). + + Retained for legacy purposes. + """ + if isinstance(module, torch.nn.modules.batchnorm._BatchNorm) and module.affine is True: + module.float() + for child in module.children(): + BN_convert_float(child) + return module + + +def network_to_half(network): + """ + Convert model to half precision in a batchnorm-safe way. + + Retained for legacy purposes. It is recommended to use FP16Model. + """ + return nn.Sequential(tofp16(), BN_convert_float(network.half())) + + +def convert_module(module, dtype): + """ + Converts a module's immediate parameters and buffers to dtype. + """ + for param in module.parameters(recurse=False): + if param is not None: + if param.data.dtype.is_floating_point: + param.data = param.data.to(dtype=dtype) + if param._grad is not None and param._grad.data.dtype.is_floating_point: + param._grad.data = param._grad.data.to(dtype=dtype) + + for buf in module.buffers(recurse=False): + if buf is not None and buf.data.dtype.is_floating_point: + buf.data = buf.data.to(dtype=dtype) + + +def convert_network(network, dtype): + """ + Converts a network's parameters and buffers to dtype. + """ + for module in network.modules(): + if isinstance(module, torch.nn.modules.batchnorm._BatchNorm) and module.affine is True: + continue + convert_module(module, dtype) + return network + + +class FP16Model(nn.Module): + """ + Convert model to half precision in a batchnorm-safe way. + """ + + def __init__(self, network): + super(FP16Model, self).__init__() + self.network = convert_network(network, dtype=torch.half) + + def forward(self, *inputs): + inputs = tuple(t.half() for t in inputs) + return self.network(*inputs) + + +def backwards_debug_hook(grad): + raise RuntimeError("master_params recieved a gradient in the backward pass!") + +def prep_param_lists(model, flat_master=False): + """ + Creates a list of FP32 master parameters for a given model, as in + `Training Neural Networks with Mixed Precision: Real Examples`_. + + Args: + model (torch.nn.Module): Existing Pytorch model + flat_master (bool, optional, default=False): Flatten the master parameters into a single tensor, as a performance optimization. + Returns: + A tuple (``model_params``, ``master_params``). ``model_params`` is a list of the model's parameters for later use with :func:`model_grads_to_master_grads` and :func:`master_params_to_model_params`. ``master_params`` is a list of FP32 master gradients. If ``flat_master=True``, ``master_params`` will be a list with one element. + + Example:: + + model_params, master_params = prep_param_lists(model) + + .. warning:: + Currently, if ``flat_master=True``, all the model's parameters must be the same type. If the model has parameters of different types, use ``flat_master=False``, or use :class:`FP16_Optimizer`. + + .. _`Training Neural Networks with Mixed Precision: Real Examples`: + http://on-demand.gputechconf.com/gtc/2018/video/S81012/ + """ + model_params = [param for param in model.parameters() if param.requires_grad] + + if flat_master: + # Give the user some more useful error messages + try: + # flatten_dense_tensors returns a contiguous flat array. + # http://pytorch.org/docs/master/_modules/torch/_utils.html + master_params = _flatten_dense_tensors([param.data for param in model_params]).float() + except: + print("Error in prep_param_lists: model may contain a mixture of parameters " + "of different types. Use flat_master=False, or use F16_Optimizer.") + raise + master_params = torch.nn.Parameter(master_params) + master_params.requires_grad = True + # master_params.register_hook(backwards_debug_hook) + if master_params.grad is None: + master_params.grad = master_params.new(*master_params.size()) + return model_params, [master_params] + else: + master_params = [param.clone().float().detach() for param in model_params] + for param in master_params: + param.requires_grad = True + return model_params, master_params + + +def model_grads_to_master_grads(model_params, master_params, flat_master=False): + """ + Copy model gradients to master gradients. + + Args: + model_params: List of model parameters created by :func:`prep_param_lists`. + master_params: List of FP32 master parameters created by :func:`prep_param_lists`. If ``master_params`` was created with ``flat_master=True``, ``flat_master=True`` should also be supplied to :func:`model_grads_to_master_grads`. + """ + if flat_master: + # The flattening may incur one more deep copy than is necessary. + master_params[0].grad.data.copy_( + _flatten_dense_tensors([p.grad.data for p in model_params])) + else: + for model, master in zip(model_params, master_params): + if model.grad is not None: + if master.grad is None: + master.grad = Variable(master.data.new(*master.data.size())) + master.grad.data.copy_(model.grad.data) + else: + master.grad = None + + +def master_params_to_model_params(model_params, master_params, flat_master=False): + """ + Copy master parameters to model parameters. + + Args: + model_params: List of model parameters created by :func:`prep_param_lists`. + master_params: List of FP32 master parameters created by :func:`prep_param_lists`. If ``master_params`` was created with ``flat_master=True``, ``flat_master=True`` should also be supplied to :func:`master_params_to_model_params`. + """ + if flat_master: + for model, master in zip(model_params, + _unflatten_dense_tensors(master_params[0].data, model_params)): + model.data.copy_(master) + else: + for model, master in zip(model_params, master_params): + model.data.copy_(master.data) + +# Backward compatibility fixes + +def to_python_float(t): + if hasattr(t, 'item'): + return t.item() + else: + return t[0] + +TORCH_MAJOR = int(torch.__version__.split('.')[0]) +TORCH_MINOR = int(torch.__version__.split('.')[1]) +if TORCH_MAJOR == 0 and TORCH_MINOR <= 4: + clip_grad_norm = torch.nn.utils.clip_grad_norm +else: + clip_grad_norm = torch.nn.utils.clip_grad_norm_ diff --git a/segmentation/mm_modules/fp16_utils/loss_scaler.py b/segmentation/mm_modules/fp16_utils/loss_scaler.py new file mode 100644 index 0000000..b9f32fe --- /dev/null +++ b/segmentation/mm_modules/fp16_utils/loss_scaler.py @@ -0,0 +1,186 @@ +import torch + +# item() is a recent addition, so this helps with backward compatibility. +def to_python_float(t): + if hasattr(t, 'item'): + return t.item() + else: + return t[0] + +class LossScaler: + """ + Class that manages a static loss scale. This class is intended to interact with + :class:`FP16_Optimizer`, and should not be directly manipulated by the user. + + Use of :class:`LossScaler` is enabled via the ``static_loss_scale`` argument to + :class:`FP16_Optimizer`'s constructor. + + Args: + scale (float, optional, default=1.0): The loss scale. + """ + + def __init__(self, scale=1): + self.cur_scale = scale + + # `params` is a list / generator of torch.Variable + def has_overflow(self, params): + return False + + # `x` is a torch.Tensor + def _has_inf_or_nan(x): + return False + + def update_scale(self, overflow): + pass + + @property + def loss_scale(self): + return self.cur_scale + + def scale_gradient(self, module, grad_in, grad_out): + return tuple(self.loss_scale * g for g in grad_in) + + def backward(self, loss, retain_graph=False): + scaled_loss = loss*self.loss_scale + scaled_loss.backward(retain_graph=retain_graph) + +class DynamicLossScaler: + """ + Class that manages dynamic loss scaling. It is recommended to use :class:`DynamicLossScaler` + indirectly, by supplying ``dynamic_loss_scale=True`` to the constructor of + :class:`FP16_Optimizer`. However, it's important to understand how :class:`DynamicLossScaler` + operates, because the default options can be changed using the + the ``dynamic_loss_args`` argument to :class:`FP16_Optimizer`'s constructor. + + Loss scaling is designed to combat the problem of underflowing gradients encountered at long + times when training fp16 networks. Dynamic loss scaling begins by attempting a very high loss + scale. Ironically, this may result in OVERflowing gradients. If overflowing gradients are + encountered, :class:`DynamicLossScaler` informs :class:`FP16_Optimizer` that an overflow has + occurred. + :class:`FP16_Optimizer` then skips the update step for this particular iteration/minibatch, + and :class:`DynamicLossScaler` adjusts the loss scale to a lower value. + If a certain number of iterations occur without overflowing gradients detected, + :class:`DynamicLossScaler` increases the loss scale once more. + In this way :class:`DynamicLossScaler` attempts to "ride the edge" of + always using the highest loss scale possible without incurring overflow. + + Args: + init_scale (float, optional, default=2**32): Initial loss scale attempted by :class:`DynamicLossScaler.` + scale_factor (float, optional, default=2.0): Factor used when adjusting the loss scale. If an overflow is encountered, the loss scale is readjusted to loss scale/``scale_factor``. If ``scale_window`` consecutive iterations take place without an overflow, the loss scale is readjusted to loss_scale*``scale_factor``. + scale_window (int, optional, default=1000): Number of consecutive iterations without an overflow to wait before increasing the loss scale. + """ + + def __init__(self, + init_scale=2**32, + scale_factor=2., + scale_window=1000): + self.cur_scale = init_scale + self.cur_iter = 0 + self.last_overflow_iter = -1 + self.scale_factor = scale_factor + self.scale_window = scale_window + + # `params` is a list / generator of torch.Variable + def has_overflow(self, params): + for p in params: + if p.grad is not None and DynamicLossScaler._has_inf_or_nan(p.grad.data): + return True + + return False + + # `x` is a torch.Tensor + def _has_inf_or_nan(x): + try: + # if x is half, the .float() incurs an additional deep copy, but it's necessary if + # Pytorch's .sum() creates a one-element tensor of the same type as x + # (which is true for some recent version of pytorch). + cpu_sum = float(x.float().sum()) + # More efficient version that can be used if .sum() returns a Python scalar + # cpu_sum = float(x.sum()) + except RuntimeError as instance: + # We want to check if inst is actually an overflow exception. + # RuntimeError could come from a different error. + # If so, we still want the exception to propagate. + if "value cannot be converted" not in instance.args[0]: + raise + return True + else: + if cpu_sum == float('inf') or cpu_sum == -float('inf') or cpu_sum != cpu_sum: + return True + return False + + # `overflow` is boolean indicating whether the gradient overflowed + def update_scale(self, overflow): + if overflow: + # self.cur_scale /= self.scale_factor + self.cur_scale = max(self.cur_scale/self.scale_factor, 1) + self.last_overflow_iter = self.cur_iter + else: + if (self.cur_iter - self.last_overflow_iter) % self.scale_window == 0: + self.cur_scale *= self.scale_factor + self.cur_iter += 1 + + @property + def loss_scale(self): + return self.cur_scale + + def scale_gradient(self, module, grad_in, grad_out): + return tuple(self.loss_scale * g for g in grad_in) + + def backward(self, loss, retain_graph=False): + scaled_loss = loss*self.loss_scale + scaled_loss.backward(retain_graph=retain_graph) + +############################################################## +# Example usage below here -- assuming it's in a separate file +############################################################## +""" +TO-DO separate out into an example. +if __name__ == "__main__": + import torch + from torch.autograd import Variable + from dynamic_loss_scaler import DynamicLossScaler + + # N is batch size; D_in is input dimension; + # H is hidden dimension; D_out is output dimension. + N, D_in, H, D_out = 64, 1000, 100, 10 + + # Create random Tensors to hold inputs and outputs, and wrap them in Variables. + x = Variable(torch.randn(N, D_in), requires_grad=False) + y = Variable(torch.randn(N, D_out), requires_grad=False) + + w1 = Variable(torch.randn(D_in, H), requires_grad=True) + w2 = Variable(torch.randn(H, D_out), requires_grad=True) + parameters = [w1, w2] + + learning_rate = 1e-6 + optimizer = torch.optim.SGD(parameters, lr=learning_rate) + loss_scaler = DynamicLossScaler() + + for t in range(500): + y_pred = x.mm(w1).clamp(min=0).mm(w2) + loss = (y_pred - y).pow(2).sum() * loss_scaler.loss_scale + print('Iter {} loss scale: {}'.format(t, loss_scaler.loss_scale)) + print('Iter {} scaled loss: {}'.format(t, loss.data[0])) + print('Iter {} unscaled loss: {}'.format(t, loss.data[0] / loss_scaler.loss_scale)) + + # Run backprop + optimizer.zero_grad() + loss.backward() + + # Check for overflow + has_overflow = DynamicLossScaler.has_overflow(parameters) + + # If no overflow, unscale grad and update as usual + if not has_overflow: + for param in parameters: + param.grad.data.mul_(1. / loss_scaler.loss_scale) + optimizer.step() + # Otherwise, don't do anything -- ie, skip iteration + else: + print('OVERFLOW!') + + # Update loss scale for next iteration + loss_scaler.update_scale(has_overflow) + +""" diff --git a/segmentation/mm_modules/utils.py b/segmentation/mm_modules/utils.py new file mode 100644 index 0000000..c58b062 --- /dev/null +++ b/segmentation/mm_modules/utils.py @@ -0,0 +1,138 @@ +from __future__ import division +import torch +import torchvision +import numpy as np +import cv2 + +def postprocess(prediction, num_classes, conf_thre=0.7, nms_thre=0.45): + """ + Postprocess for the output of YOLO model + perform box transformation, specify the class for each detection, + and perform class-wise non-maximum suppression. + Args: + prediction (torch tensor): The shape is :math:`(N, B, 4)`. + :math:`N` is the number of predictions, + :math:`B` the number of boxes. The last axis consists of + :math:`xc, yc, w, h` where `xc` and `yc` represent a center + of a bounding box. + num_classes (int): + number of dataset classes. + conf_thre (float): + confidence threshold ranging from 0 to 1, + which is defined in the config file. + nms_thre (float): + IoU threshold of non-max suppression ranging from 0 to 1. + + Returns: + output (list of torch tensor): + + """ + box_corner = prediction.new(prediction.shape) + box_corner[:, :, 0] = prediction[:, :, 0] - prediction[:, :, 2] / 2 + box_corner[:, :, 1] = prediction[:, :, 1] - prediction[:, :, 3] / 2 + box_corner[:, :, 2] = prediction[:, :, 0] + prediction[:, :, 2] / 2 + box_corner[:, :, 3] = prediction[:, :, 1] + prediction[:, :, 3] / 2 + prediction[:, :, :4] = box_corner[:, :, :4] + + output = [None for _ in range(len(prediction))] + for i, image_pred in enumerate(prediction): + + # If none are remaining => process next image + if not image_pred.size(0): + continue + # Get score and class with highest confidence + class_conf, class_pred = torch.max( + image_pred[:, 5:5 + num_classes], 1, keepdim=True) + + conf_mask = (image_pred[:, 4] * class_conf.squeeze() >= conf_thre).squeeze() + # Detections ordered as (x1, y1, x2, y2, obj_conf, class_conf, class_pred) + detections = torch.cat( + (image_pred[:, :5], class_conf, class_pred.float()), 1) + detections = detections[conf_mask] + if not detections.size(0): + continue + + # Iterate through all predicted classes + unique_labels = detections[:, -1].unique() + + for c in unique_labels: + # Get the detections with the particular class + detections_class = detections[detections[:, -1] == c] + nms_out_index = torchvision.ops.nms( + detections_class[:, :4], detections_class[:, 4]*detections_class[:, 5], nms_thre) + detections_class = detections_class[nms_out_index] + if output[i] is None: + output[i] = detections_class + else: + output[i] = torch.cat((output[i], detections_class)) + + return output + + +def bboxes_iou(bboxes_a, bboxes_b, xyxy=True): + """Calculate the Intersection of Unions (IoUs) between bounding boxes. + IoU is calculated as a ratio of area of the intersection + and area of the union. + + Args: + bbox_a (array): An array whose shape is :math:`(N, 4)`. + :math:`N` is the number of bounding boxes. + The dtype should be :obj:`numpy.float32`. + bbox_b (array): An array similar to :obj:`bbox_a`, + whose shape is :math:`(K, 4)`. + The dtype should be :obj:`numpy.float32`. + Returns: + array: + An array whose shape is :math:`(N, K)`. \ + An element at index :math:`(n, k)` contains IoUs between \ + :math:`n` th bounding box in :obj:`bbox_a` and :math:`k` th bounding \ + box in :obj:`bbox_b`. + + from: https://github.com/chainer/chainercv + """ + if bboxes_a.shape[1] != 4 or bboxes_b.shape[1] != 4: + raise IndexError + + if xyxy: + tl = torch.max(bboxes_a[:, None, :2], bboxes_b[:, :2]) + br = torch.min(bboxes_a[:, None, 2:], bboxes_b[:, 2:]) + area_a = torch.prod(bboxes_a[:, 2:] - bboxes_a[:, :2], 1) + area_b = torch.prod(bboxes_b[:, 2:] - bboxes_b[:, :2], 1) + else: + tl = torch.max((bboxes_a[:, None, :2] - bboxes_a[:, None, 2:] / 2), + (bboxes_b[:, :2] - bboxes_b[:, 2:] / 2)) + br = torch.min((bboxes_a[:, None, :2] + bboxes_a[:, None, 2:] / 2), + (bboxes_b[:, :2] + bboxes_b[:, 2:] / 2)) + + area_a = torch.prod(bboxes_a[:, 2:], 1) + area_b = torch.prod(bboxes_b[:, 2:], 1) + en = (tl < br).type(tl.type()).prod(dim=2) + area_i = torch.prod(br - tl, 2) * en # * ((tl < br).all()) + return area_i / (area_a[:, None] + area_b - area_i) + + +def matrix_iou(a,b): + """ + return iou of a and b, numpy version for data augenmentation + """ + lt = np.maximum(a[:, np.newaxis, :2], b[:, :2]) + rb = np.minimum(a[:, np.newaxis, 2:], b[:, 2:]) + + area_i = np.prod(rb - lt, axis=2) * (lt < rb).all(axis=2) + area_a = np.prod(a[:, 2:] - a[:, :2], axis=1) + area_b = np.prod(b[:, 2:] - b[:, :2], axis=1) + return area_i / (area_a[:, np.newaxis] + area_b - area_i+1e-12) + +def visual(img, boxes, scores): + + COLORS = [(255, 0, 0), (0, 255, 0), (0, 0, 255)] + FONT = cv2.FONT_HERSHEY_SIMPLEX + for i in range(boxes.shape[0]): + + cv2.rectangle(img, (int(boxes[i][0]),int(boxes[i][1])),(int(boxes[i][2]),int(boxes[i][3])),COLORS[i%3],2) + cv2.putText(img, 'Object: %.2f'%scores[i],(int(boxes[i][0])-3,int(boxes[i][1])-5), FONT, + 0.4, (0,0,0),2) + + return img + + diff --git a/segmentation/mm_modules/vis_utils.py b/segmentation/mm_modules/vis_utils.py new file mode 100644 index 0000000..a322afc --- /dev/null +++ b/segmentation/mm_modules/vis_utils.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- + +import numpy as np +import os +import matplotlib + +matplotlib.use('AGG') + +import matplotlib.pyplot as plt +import torch +import cv2 +import math +from skimage import transform + +def make_vis(dataset, index, img, fuse_weights, fused_fs): + save_dir = 'vis_output/{}/{}'.format(dataset,index) + os.makedirs(save_dir, exist_ok=True) + + for i in range(len(fuse_weights)): + weights = fuse_weights[i].float().cpu().squeeze().numpy() + max_v = weights.max() + min_v = weights.min() + for j in range(3): + v = weights[j,:,:] + save_name = os.path.join(save_dir, 'level_{}_weight_{}.png'.format(i+1,j+1)) + add_heat(img, v, max_v, min_v, save=save_name) + + fused_f = fused_fs[i].float().cpu().squeeze().numpy() + max_f = fused_f.max() + min_f = fused_f.min() + save_f_name = os.path.join(save_dir, 'fused_feature_level_{}.png'.format(i+1)) + add_heat(img, fused_f, max_f, min_f, save=save_f_name) + +def make_pred_vis(dataset,index, img, class_names, bboxes, cls, scores): + save_preddir = 'vis_output/{}/pred/'.format(dataset) + os.makedirs(save_preddir, exist_ok=True) + + save_pred_name = os.path.join(save_preddir,'{}.png'.format(index)) + + bboxes = bboxes.numpy() + scores = scores.numpy() + cls_ids = cls.numpy() + + im = vis(img, bboxes, scores, cls_ids, class_names) + + cv2.imwrite(save_pred_name, im) + +def vis(img, boxes, scores, cls_ids, conf=0.5, class_names=None, color=None): + + colors = torch.FloatTensor([[1,0,1],[0,0,1],[0,1,1],[0,1,0],[1,1,0],[1,0,0]]); + def get_color(c, x, max_val): + ratio = float(x)/max_val * 5 + i = int(math.floor(ratio)) + j = int(math.ceil(ratio)) + ratio = ratio - i + r = (1-ratio) * colors[i][c] + ratio*colors[j][c] + return int(r*255) + + width = img.shape[1] + height = img.shape[0] + for i in range(len(boxes)): + box = boxes[i] + cls_conf = scores[i] + if cls_conf < conf: + continue + x1 = int(box[0]) + y1 = int(box[1]) + x2 = int(box[0]+box[2]) + y2 = int(box[1]+box[3]) + + + if color: + rgb = color + else: + rgb = (255, 0, 0) + if class_names is not None: + cls_conf = scores[i] + cls_id = int(cls_ids[i]) + class_name = class_names[cls_id] + classes = len(class_names) + offset = cls_id * 123456 % classes + red = get_color(2, offset, classes) + green = get_color(1, offset, classes) + blue = get_color(0, offset, classes) + if color is None: + rgb = (red, green, blue) + img = cv2.putText(img, '%s: %.2f'%(class_name,cls_conf), (x1,y1-5), cv2.FONT_HERSHEY_SIMPLEX, 0.3, rgb, 1) + img = cv2.rectangle(img, (x1,y1), (x2,y2), rgb, 1) + return img + +def add_heat(image, heat_map, max_v, min_v, alpha=0.4, save=None, cmap='jet', axis='off'): + height = image.shape[0] + width = image.shape[1] + + # resize heat map + heat_map_resized = transform.resize(heat_map, (height, width)) + + # normalize heat map + max_value = max_v + min_value = min_v + normalized_heat_map = (heat_map_resized - min_value) / (max_value - min_value) + + # display + plt.imshow(image) + plt.imshow(255 * normalized_heat_map, alpha=alpha, cmap=cmap) + plt.axis(axis) + + if save is not None: + plt.savefig(save, bbox_inches='tight', pad_inches=0) + + + + diff --git a/segmentation/mm_modules/voc_evaluator.py b/segmentation/mm_modules/voc_evaluator.py new file mode 100644 index 0000000..1fc5ae8 --- /dev/null +++ b/segmentation/mm_modules/voc_evaluator.py @@ -0,0 +1,204 @@ +import json +import tempfile +import sys +from tqdm import tqdm + +from pycocotools.cocoeval import COCOeval +from torch.autograd import Variable + +from dataset.vocdataset import * +from dataset.data_augment import ValTransform +from utils.utils import * +from utils import distributed_util +from utils.vis_utils import make_vis, make_pred_vis + +import time + +#DEBUG = True +DEBUG = False + +def _accumulate_predictions_from_multiple_gpus(predictions_per_gpu): + all_predictions = distributed_util.scatter_gather(predictions_per_gpu) + if not distributed_util.is_main_process(): + return + # merge the list of dicts + predictions = {} + for p in all_predictions: + predictions.update(p) + # convert a dict where the key is the index in a list + image_ids = list(sorted(predictions.keys())) + if len(image_ids) != image_ids[-1] + 1: + print('num_imgs: ',len(image_ids)) + print('last img_id: ',image_ids[-1]) + print( + "Number of images that were gathered from multiple processes is not " + "a contiguous set. Some images might be missing from the evaluation" + ) + + # convert to a list + predictions = [predictions[i] for i in image_ids] + return predictions + + +class VOCEvaluator(): + """ + COCO AP Evaluation class. + All the data in the val2017 dataset are processed \ + and evaluated by COCO API. + """ + def __init__(self, data_dir, img_size, confthre, nmsthre,vis=False): + """ + Args: + data_dir (str): dataset root directory + img_size (int): image size after preprocess. images are resized \ + to squares whose shape is (img_size, img_size). + confthre (float): + confidence threshold ranging from 0 to 1, \ + which is defined in the config file. + nmsthre (float): + IoU threshold of non-max supression ranging from 0 to 1. + """ + test_sets = [('2007', 'test'),] + self.dataset = VOCDetection( + root=data_dir, + image_sets = test_sets, + input_dim=img_size, + preproc = ValTransform(rgb_means=(0.485, 0.456, 0.406),std=(0.229, 0.224, 0.225)),) + self.num_images = len(self.dataset) + self.dataloader = torch.utils.data.DataLoader( + self.dataset, batch_size=1, shuffle=False, num_workers=0) + self.img_size = img_size + self.confthre = confthre + self.nmsthre = nmsthre + self.vis=vis + + def evaluate(self, model, half=False): + """ + COCO average precision (AP) Evaluation. Iterate inference on the test dataset + and the results are evaluated by COCO API. + Args: + model : model object + Returns: + ap50_95 (float) : calculated COCO AP for IoU=50:95 + ap50 (float) : calculated COCO AP for IoU=50 + """ + if isinstance(model, torch.nn.parallel.DistributedDataParallel): + model = model.module + model.eval() + cuda = torch.cuda.is_available() + if half: + Tensor = torch.cuda.HalfTensor if cuda else torch.HalfTensor + else: + Tensor = torch.cuda.FloatTensor if cuda else torch.FloatTensor + + ids = [] + data_dict = [] + dataiterator = iter(self.dataloader) + img_num = 0 + indices = list(range(self.num_images)) + dis_indices = indices[distributed_util.get_rank()::distributed_util.get_world_size()] + progress_bar = tqdm if distributed_util.is_main_process() else iter + num_classes = 20 + predictions = {} + + if distributed_util.is_main_process(): + inference_time=0 + nms_time=0 + n_samples=len(dis_indices) + + for i in progress_bar(dis_indices): + img, _, info_img, id_ = self.dataset[i] # load a batch + info_img = [float(info) for info in info_img] + ids.append(id_) + with torch.no_grad(): + img = Variable(img.type(Tensor).unsqueeze(0)) + + if distributed_util.is_main_process() and i > 9: + start=time.time() + + if self.vis: + outputs,fuse_weights,fused_f = model(img) + else: + outputs = model(img) + + if distributed_util.is_main_process() and i > 9: + infer_end=time.time() + inference_time += (infer_end-start) + + outputs = postprocess( + outputs, 20, self.confthre, self.nmsthre) + + + if distributed_util.is_main_process() and i > 9: + nms_end=time.time() + nms_time +=(nms_end-infer_end) + + if outputs[0] is None: + predictions[i] = (None, None, None) + continue + outputs = outputs[0].cpu().data + + bboxes = outputs[:, 0:4] + bboxes[:, 0::2] *= info_img[0] / self.img_size[0] + bboxes[:, 1::2] *= info_img[1] / self.img_size[1] + cls = outputs[:, 6] + scores = outputs[:, 4]* outputs[:,5] + predictions[i] = (bboxes, cls, scores) + + if self.vis: + o_img,_,_,_ = self.dataset.pull_item(i) + make_vis('VOC', i, o_img, fuse_weights, fused_f) + class_names = self.dataset._classes + + bbox = bboxes.clone() + bbox[:, 2] = bbox[:,2] - bbox[:,0] + bbox[:, 3] = bbox[:,3] - bbox[:,1] + + make_pred_vis('VOC', i, o_img, class_names, bbox, cls, scores) + + if DEBUG and distributed_util.is_main_process(): + o_img,_,_,_ = self.dataset.pull_item(i) + class_names = self.dataset._classes + bbox = bboxes.clone() + bbox[:, 2] = bbox[:,2] - bbox[:,0] + bbox[:, 3] = bbox[:,3] - bbox[:,1] + make_pred_vis('VOC', i, o_img, class_names, bbox, cls, scores) + + distributed_util.synchronize() + predictions = _accumulate_predictions_from_multiple_gpus(predictions) + if not distributed_util.is_main_process(): + return 0, 0 + + + print('Main process Evaluating...') + + a_infer_time = 1000*inference_time / (n_samples-10) + a_nms_time= 1000*nms_time / (n_samples-10) + + print('Average forward time: %.2f ms, Average NMS time: %.2f ms, Average inference time: %.2f ms' %(a_infer_time, \ + a_nms_time, (a_infer_time+a_nms_time))) + + all_boxes = [[[] for _ in range(self.num_images)] + for _ in range(num_classes)] + for img_num in range(self.num_images): + bboxes, cls, scores = predictions[img_num] + if bboxes is None: + for j in range(num_classes): + all_boxes[j][img_num] = np.empty([0,5],dtype=np.float32) + continue + for j in range(num_classes): + mask_c = (cls == j) + if sum(mask_c) ==0: + all_boxes[j][img_num] = np.empty([0,5],dtype=np.float32) + continue + + c_dets = torch.cat((bboxes, scores.unsqueeze(1)),dim=1) + all_boxes[j][img_num] = c_dets[mask_c].numpy() + + sys.stdout.write('im_eval: {:d}/{:d} \r'.format(img_num+1, self.num_images)) + sys.stdout.flush() + + with tempfile.TemporaryDirectory() as tempdir: + mAP50, mAP70 = self.dataset.evaluate_detections(all_boxes, tempdir) + return mAP50,mAP70 + diff --git a/segmentation/mmcv_custom/__init__.py b/segmentation/mmcv_custom/__init__.py new file mode 100644 index 0000000..7e0e39b --- /dev/null +++ b/segmentation/mmcv_custom/__init__.py @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- + +from .checkpoint import load_checkpoint + +__all__ = ['load_checkpoint'] diff --git a/segmentation/mmcv_custom/checkpoint.py b/segmentation/mmcv_custom/checkpoint.py new file mode 100644 index 0000000..51322c1 --- /dev/null +++ b/segmentation/mmcv_custom/checkpoint.py @@ -0,0 +1,500 @@ +# Copyright (c) Open-MMLab. All rights reserved. +import io +import os +import os.path as osp +import pkgutil +import time +import warnings +from collections import OrderedDict +from importlib import import_module +from tempfile import TemporaryDirectory + +import torch +import torchvision +from torch.optim import Optimizer +from torch.utils import model_zoo +from torch.nn import functional as F + +import mmcv +from mmcv.fileio import FileClient +from mmcv.fileio import load as load_file +from mmcv.parallel import is_module_wrapper +from mmcv.utils import mkdir_or_exist +from mmcv.runner import get_dist_info + +ENV_MMCV_HOME = 'MMCV_HOME' +ENV_XDG_CACHE_HOME = 'XDG_CACHE_HOME' +DEFAULT_CACHE_DIR = '~/.cache' + + +def _get_mmcv_home(): + mmcv_home = os.path.expanduser( + os.getenv( + ENV_MMCV_HOME, + os.path.join( + os.getenv(ENV_XDG_CACHE_HOME, DEFAULT_CACHE_DIR), 'mmcv'))) + + mkdir_or_exist(mmcv_home) + return mmcv_home + + +def load_state_dict(module, state_dict, strict=False, logger=None): + """Load state_dict to a module. + + This method is modified from :meth:`torch.nn.Module.load_state_dict`. + Default value for ``strict`` is set to ``False`` and the message for + param mismatch will be shown even if strict is False. + + Args: + module (Module): Module that receives the state_dict. + state_dict (OrderedDict): Weights. + strict (bool): whether to strictly enforce that the keys + in :attr:`state_dict` match the keys returned by this module's + :meth:`~torch.nn.Module.state_dict` function. Default: ``False``. + logger (:obj:`logging.Logger`, optional): Logger to log the error + message. If not specified, print function will be used. + """ + unexpected_keys = [] + all_missing_keys = [] + err_msg = [] + + metadata = getattr(state_dict, '_metadata', None) + state_dict = state_dict.copy() + if metadata is not None: + state_dict._metadata = metadata + + # use _load_from_state_dict to enable checkpoint version control + def load(module, prefix=''): + # recursively check parallel module in case that the model has a + # complicated structure, e.g., nn.Module(nn.Module(DDP)) + if is_module_wrapper(module): + module = module.module + local_metadata = {} if metadata is None else metadata.get( + prefix[:-1], {}) + module._load_from_state_dict(state_dict, prefix, local_metadata, True, + all_missing_keys, unexpected_keys, + err_msg) + for name, child in module._modules.items(): + if child is not None: + load(child, prefix + name + '.') + + load(module) + load = None # break load->load reference cycle + + # ignore "num_batches_tracked" of BN layers + missing_keys = [ + key for key in all_missing_keys if 'num_batches_tracked' not in key + ] + + if unexpected_keys: + err_msg.append('unexpected key in source ' + f'state_dict: {", ".join(unexpected_keys)}\n') + if missing_keys: + err_msg.append( + f'missing keys in source state_dict: {", ".join(missing_keys)}\n') + + rank, _ = get_dist_info() + if len(err_msg) > 0 and rank == 0: + err_msg.insert( + 0, 'The model and loaded state dict do not match exactly\n') + err_msg = '\n'.join(err_msg) + if strict: + raise RuntimeError(err_msg) + elif logger is not None: + logger.warning(err_msg) + else: + print(err_msg) + + +def load_url_dist(url, model_dir=None): + """In distributed setting, this function only download checkpoint at local + rank 0.""" + rank, world_size = get_dist_info() + rank = int(os.environ.get('LOCAL_RANK', rank)) + if rank == 0: + checkpoint = model_zoo.load_url(url, model_dir=model_dir) + if world_size > 1: + torch.distributed.barrier() + if rank > 0: + checkpoint = model_zoo.load_url(url, model_dir=model_dir) + return checkpoint + + +def load_pavimodel_dist(model_path, map_location=None): + """In distributed setting, this function only download checkpoint at local + rank 0.""" + try: + from pavi import modelcloud + except ImportError: + raise ImportError( + 'Please install pavi to load checkpoint from modelcloud.') + rank, world_size = get_dist_info() + rank = int(os.environ.get('LOCAL_RANK', rank)) + if rank == 0: + model = modelcloud.get(model_path) + with TemporaryDirectory() as tmp_dir: + downloaded_file = osp.join(tmp_dir, model.name) + model.download(downloaded_file) + checkpoint = torch.load(downloaded_file, map_location=map_location) + if world_size > 1: + torch.distributed.barrier() + if rank > 0: + model = modelcloud.get(model_path) + with TemporaryDirectory() as tmp_dir: + downloaded_file = osp.join(tmp_dir, model.name) + model.download(downloaded_file) + checkpoint = torch.load( + downloaded_file, map_location=map_location) + return checkpoint + + +def load_fileclient_dist(filename, backend, map_location): + """In distributed setting, this function only download checkpoint at local + rank 0.""" + rank, world_size = get_dist_info() + rank = int(os.environ.get('LOCAL_RANK', rank)) + allowed_backends = ['ceph'] + if backend not in allowed_backends: + raise ValueError(f'Load from Backend {backend} is not supported.') + if rank == 0: + fileclient = FileClient(backend=backend) + buffer = io.BytesIO(fileclient.get(filename)) + checkpoint = torch.load(buffer, map_location=map_location) + if world_size > 1: + torch.distributed.barrier() + if rank > 0: + fileclient = FileClient(backend=backend) + buffer = io.BytesIO(fileclient.get(filename)) + checkpoint = torch.load(buffer, map_location=map_location) + return checkpoint + + +def get_torchvision_models(): + model_urls = dict() + for _, name, ispkg in pkgutil.walk_packages(torchvision.models.__path__): + if ispkg: + continue + _zoo = import_module(f'torchvision.models.{name}') + if hasattr(_zoo, 'model_urls'): + _urls = getattr(_zoo, 'model_urls') + model_urls.update(_urls) + return model_urls + + +def get_external_models(): + mmcv_home = _get_mmcv_home() + default_json_path = osp.join(mmcv.__path__[0], 'model_zoo/open_mmlab.json') + default_urls = load_file(default_json_path) + assert isinstance(default_urls, dict) + external_json_path = osp.join(mmcv_home, 'open_mmlab.json') + if osp.exists(external_json_path): + external_urls = load_file(external_json_path) + assert isinstance(external_urls, dict) + default_urls.update(external_urls) + + return default_urls + + +def get_mmcls_models(): + mmcls_json_path = osp.join(mmcv.__path__[0], 'model_zoo/mmcls.json') + mmcls_urls = load_file(mmcls_json_path) + + return mmcls_urls + + +def get_deprecated_model_names(): + deprecate_json_path = osp.join(mmcv.__path__[0], + 'model_zoo/deprecated.json') + deprecate_urls = load_file(deprecate_json_path) + assert isinstance(deprecate_urls, dict) + + return deprecate_urls + + +def _process_mmcls_checkpoint(checkpoint): + state_dict = checkpoint['state_dict'] + new_state_dict = OrderedDict() + for k, v in state_dict.items(): + if k.startswith('backbone.'): + new_state_dict[k[9:]] = v + new_checkpoint = dict(state_dict=new_state_dict) + + return new_checkpoint + + +def _load_checkpoint(filename, map_location=None): + """Load checkpoint from somewhere (modelzoo, file, url). + + Args: + filename (str): Accept local filepath, URL, ``torchvision://xxx``, + ``open-mmlab://xxx``. Please refer to ``docs/model_zoo.md`` for + details. + map_location (str | None): Same as :func:`torch.load`. Default: None. + + Returns: + dict | OrderedDict: The loaded checkpoint. It can be either an + OrderedDict storing model weights or a dict containing other + information, which depends on the checkpoint. + """ + if filename.startswith('modelzoo://'): + warnings.warn('The URL scheme of "modelzoo://" is deprecated, please ' + 'use "torchvision://" instead') + model_urls = get_torchvision_models() + model_name = filename[11:] + checkpoint = load_url_dist(model_urls[model_name]) + elif filename.startswith('torchvision://'): + model_urls = get_torchvision_models() + model_name = filename[14:] + checkpoint = load_url_dist(model_urls[model_name]) + elif filename.startswith('open-mmlab://'): + model_urls = get_external_models() + model_name = filename[13:] + deprecated_urls = get_deprecated_model_names() + if model_name in deprecated_urls: + warnings.warn(f'open-mmlab://{model_name} is deprecated in favor ' + f'of open-mmlab://{deprecated_urls[model_name]}') + model_name = deprecated_urls[model_name] + model_url = model_urls[model_name] + # check if is url + if model_url.startswith(('http://', 'https://')): + checkpoint = load_url_dist(model_url) + else: + filename = osp.join(_get_mmcv_home(), model_url) + if not osp.isfile(filename): + raise IOError(f'{filename} is not a checkpoint file') + checkpoint = torch.load(filename, map_location=map_location) + elif filename.startswith('mmcls://'): + model_urls = get_mmcls_models() + model_name = filename[8:] + checkpoint = load_url_dist(model_urls[model_name]) + checkpoint = _process_mmcls_checkpoint(checkpoint) + elif filename.startswith(('http://', 'https://')): + checkpoint = load_url_dist(filename) + elif filename.startswith('pavi://'): + model_path = filename[7:] + checkpoint = load_pavimodel_dist(model_path, map_location=map_location) + elif filename.startswith('s3://'): + checkpoint = load_fileclient_dist( + filename, backend='ceph', map_location=map_location) + else: + if not osp.isfile(filename): + raise IOError(f'{filename} is not a checkpoint file') + checkpoint = torch.load(filename, map_location=map_location) + return checkpoint + + +def load_checkpoint(model, + filename, + map_location='cpu', + strict=False, + logger=None): + """Load checkpoint from a file or URI. + + Args: + model (Module): Module to load checkpoint. + filename (str): Accept local filepath, URL, ``torchvision://xxx``, + ``open-mmlab://xxx``. Please refer to ``docs/model_zoo.md`` for + details. + map_location (str): Same as :func:`torch.load`. + strict (bool): Whether to allow different params for the model and + checkpoint. + logger (:mod:`logging.Logger` or None): The logger for error message. + + Returns: + dict or OrderedDict: The loaded checkpoint. + """ + checkpoint = _load_checkpoint(filename, map_location) + # OrderedDict is a subclass of dict + if not isinstance(checkpoint, dict): + raise RuntimeError( + f'No state_dict found in checkpoint file {filename}') + # get state_dict from checkpoint + if 'state_dict' in checkpoint: + state_dict = checkpoint['state_dict'] + elif 'model' in checkpoint: + state_dict = checkpoint['model'] + else: + state_dict = checkpoint + # strip prefix of state_dict + if list(state_dict.keys())[0].startswith('module.'): + state_dict = {k[7:]: v for k, v in state_dict.items()} + + # for MoBY, load model of online branch + if sorted(list(state_dict.keys()))[0].startswith('encoder'): + state_dict = {k.replace('encoder.', ''): v for k, v in state_dict.items() if k.startswith('encoder.')} + + # reshape absolute position embedding + if state_dict.get('absolute_pos_embed') is not None: + absolute_pos_embed = state_dict['absolute_pos_embed'] + N1, L, C1 = absolute_pos_embed.size() + N2, C2, H, W = model.absolute_pos_embed.size() + if N1 != N2 or C1 != C2 or L != H*W: + logger.warning("Error in loading absolute_pos_embed, pass") + else: + state_dict['absolute_pos_embed'] = absolute_pos_embed.view(N2, H, W, C2).permute(0, 3, 1, 2) + + # interpolate position bias table if needed + relative_position_bias_table_keys = [k for k in state_dict.keys() if "relative_position_bias_table" in k] + for table_key in relative_position_bias_table_keys: + table_pretrained = state_dict[table_key] + table_current = model.state_dict()[table_key] + L1, nH1 = table_pretrained.size() + L2, nH2 = table_current.size() + if nH1 != nH2: + logger.warning(f"Error in loading {table_key}, pass") + else: + if L1 != L2: + S1 = int(L1 ** 0.5) + S2 = int(L2 ** 0.5) + table_pretrained_resized = F.interpolate( + table_pretrained.permute(1, 0).view(1, nH1, S1, S1), + size=(S2, S2), mode='bicubic') + state_dict[table_key] = table_pretrained_resized.view(nH2, L2).permute(1, 0) + + # load state_dict + load_state_dict(model, state_dict, strict, logger) + return checkpoint + + +def weights_to_cpu(state_dict): + """Copy a model state_dict to cpu. + + Args: + state_dict (OrderedDict): Model weights on GPU. + + Returns: + OrderedDict: Model weights on GPU. + """ + state_dict_cpu = OrderedDict() + for key, val in state_dict.items(): + state_dict_cpu[key] = val.cpu() + return state_dict_cpu + + +def _save_to_state_dict(module, destination, prefix, keep_vars): + """Saves module state to `destination` dictionary. + + This method is modified from :meth:`torch.nn.Module._save_to_state_dict`. + + Args: + module (nn.Module): The module to generate state_dict. + destination (dict): A dict where state will be stored. + prefix (str): The prefix for parameters and buffers used in this + module. + """ + for name, param in module._parameters.items(): + if param is not None: + destination[prefix + name] = param if keep_vars else param.detach() + for name, buf in module._buffers.items(): + # remove check of _non_persistent_buffers_set to allow nn.BatchNorm2d + if buf is not None: + destination[prefix + name] = buf if keep_vars else buf.detach() + + +def get_state_dict(module, destination=None, prefix='', keep_vars=False): + """Returns a dictionary containing a whole state of the module. + + Both parameters and persistent buffers (e.g. running averages) are + included. Keys are corresponding parameter and buffer names. + + This method is modified from :meth:`torch.nn.Module.state_dict` to + recursively check parallel module in case that the model has a complicated + structure, e.g., nn.Module(nn.Module(DDP)). + + Args: + module (nn.Module): The module to generate state_dict. + destination (OrderedDict): Returned dict for the state of the + module. + prefix (str): Prefix of the key. + keep_vars (bool): Whether to keep the variable property of the + parameters. Default: False. + + Returns: + dict: A dictionary containing a whole state of the module. + """ + # recursively check parallel module in case that the model has a + # complicated structure, e.g., nn.Module(nn.Module(DDP)) + if is_module_wrapper(module): + module = module.module + + # below is the same as torch.nn.Module.state_dict() + if destination is None: + destination = OrderedDict() + destination._metadata = OrderedDict() + destination._metadata[prefix[:-1]] = local_metadata = dict( + version=module._version) + _save_to_state_dict(module, destination, prefix, keep_vars) + for name, child in module._modules.items(): + if child is not None: + get_state_dict( + child, destination, prefix + name + '.', keep_vars=keep_vars) + for hook in module._state_dict_hooks.values(): + hook_result = hook(module, destination, prefix, local_metadata) + if hook_result is not None: + destination = hook_result + return destination + + +def save_checkpoint(model, filename, optimizer=None, meta=None): + """Save checkpoint to file. + + The checkpoint will have 3 fields: ``meta``, ``state_dict`` and + ``optimizer``. By default ``meta`` will contain version and time info. + + Args: + model (Module): Module whose params are to be saved. + filename (str): Checkpoint filename. + optimizer (:obj:`Optimizer`, optional): Optimizer to be saved. + meta (dict, optional): Metadata to be saved in checkpoint. + """ + if meta is None: + meta = {} + elif not isinstance(meta, dict): + raise TypeError(f'meta must be a dict or None, but got {type(meta)}') + meta.update(mmcv_version=mmcv.__version__, time=time.asctime()) + + if is_module_wrapper(model): + model = model.module + + if hasattr(model, 'CLASSES') and model.CLASSES is not None: + # save class name to the meta + meta.update(CLASSES=model.CLASSES) + + checkpoint = { + 'meta': meta, + 'state_dict': weights_to_cpu(get_state_dict(model)) + } + # save optimizer state dict in the checkpoint + if isinstance(optimizer, Optimizer): + checkpoint['optimizer'] = optimizer.state_dict() + elif isinstance(optimizer, dict): + checkpoint['optimizer'] = {} + for name, optim in optimizer.items(): + checkpoint['optimizer'][name] = optim.state_dict() + + if filename.startswith('pavi://'): + try: + from pavi import modelcloud + from pavi.exception import NodeNotFoundError + except ImportError: + raise ImportError( + 'Please install pavi to load checkpoint from modelcloud.') + model_path = filename[7:] + root = modelcloud.Folder() + model_dir, model_name = osp.split(model_path) + try: + model = modelcloud.get(model_dir) + except NodeNotFoundError: + model = root.create_training_model(model_dir) + with TemporaryDirectory() as tmp_dir: + checkpoint_file = osp.join(tmp_dir, model_name) + with open(checkpoint_file, 'wb') as f: + torch.save(checkpoint, f) + f.flush() + model.create_file(checkpoint_file, name=model_name) + else: + mmcv.mkdir_or_exist(osp.dirname(filename)) + # immediately flush buffer + with open(filename, 'wb') as f: + torch.save(checkpoint, f) + f.flush() diff --git a/segmentation/mmcv_custom/runner/__init__.py b/segmentation/mmcv_custom/runner/__init__.py new file mode 100644 index 0000000..c701cb0 --- /dev/null +++ b/segmentation/mmcv_custom/runner/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) Open-MMLab. All rights reserved. +from .checkpoint import save_checkpoint +from .epoch_based_runner import EpochBasedRunnerAmp + + +__all__ = [ + 'EpochBasedRunnerAmp', 'save_checkpoint' +] diff --git a/segmentation/mmcv_custom/runner/checkpoint.py b/segmentation/mmcv_custom/runner/checkpoint.py new file mode 100644 index 0000000..b04167e --- /dev/null +++ b/segmentation/mmcv_custom/runner/checkpoint.py @@ -0,0 +1,85 @@ +# Copyright (c) Open-MMLab. All rights reserved. +import os.path as osp +import time +from tempfile import TemporaryDirectory + +import torch +from torch.optim import Optimizer + +import mmcv +from mmcv.parallel import is_module_wrapper +from mmcv.runner.checkpoint import weights_to_cpu, get_state_dict + +try: + import apex +except: + print('apex is not installed') + + +def save_checkpoint(model, filename, optimizer=None, meta=None): + """Save checkpoint to file. + + The checkpoint will have 4 fields: ``meta``, ``state_dict`` and + ``optimizer``, ``amp``. By default ``meta`` will contain version + and time info. + + Args: + model (Module): Module whose params are to be saved. + filename (str): Checkpoint filename. + optimizer (:obj:`Optimizer`, optional): Optimizer to be saved. + meta (dict, optional): Metadata to be saved in checkpoint. + """ + if meta is None: + meta = {} + elif not isinstance(meta, dict): + raise TypeError(f'meta must be a dict or None, but got {type(meta)}') + meta.update(mmcv_version=mmcv.__version__, time=time.asctime()) + + if is_module_wrapper(model): + model = model.module + + if hasattr(model, 'CLASSES') and model.CLASSES is not None: + # save class name to the meta + meta.update(CLASSES=model.CLASSES) + + checkpoint = { + 'meta': meta, + 'state_dict': weights_to_cpu(get_state_dict(model)) + } + # save optimizer state dict in the checkpoint + if isinstance(optimizer, Optimizer): + checkpoint['optimizer'] = optimizer.state_dict() + elif isinstance(optimizer, dict): + checkpoint['optimizer'] = {} + for name, optim in optimizer.items(): + checkpoint['optimizer'][name] = optim.state_dict() + + # save amp state dict in the checkpoint + checkpoint['amp'] = apex.amp.state_dict() + + if filename.startswith('pavi://'): + try: + from pavi import modelcloud + from pavi.exception import NodeNotFoundError + except ImportError: + raise ImportError( + 'Please install pavi to load checkpoint from modelcloud.') + model_path = filename[7:] + root = modelcloud.Folder() + model_dir, model_name = osp.split(model_path) + try: + model = modelcloud.get(model_dir) + except NodeNotFoundError: + model = root.create_training_model(model_dir) + with TemporaryDirectory() as tmp_dir: + checkpoint_file = osp.join(tmp_dir, model_name) + with open(checkpoint_file, 'wb') as f: + torch.save(checkpoint, f) + f.flush() + model.create_file(checkpoint_file, name=model_name) + else: + mmcv.mkdir_or_exist(osp.dirname(filename)) + # immediately flush buffer + with open(filename, 'wb') as f: + torch.save(checkpoint, f) + f.flush() diff --git a/segmentation/mmcv_custom/runner/epoch_based_runner.py b/segmentation/mmcv_custom/runner/epoch_based_runner.py new file mode 100644 index 0000000..7cdf3fa --- /dev/null +++ b/segmentation/mmcv_custom/runner/epoch_based_runner.py @@ -0,0 +1,104 @@ +# Copyright (c) Open-MMLab. All rights reserved. +import os.path as osp +import platform +import shutil + +import torch +from torch.optim import Optimizer + +import mmcv +from mmcv.runner import RUNNERS, EpochBasedRunner +from .checkpoint import save_checkpoint + +try: + import apex +except: + print('apex is not installed') + + +@RUNNERS.register_module() +class EpochBasedRunnerAmp(EpochBasedRunner): + """Epoch-based Runner with AMP support. + + This runner train models epoch by epoch. + """ + + def save_checkpoint(self, + out_dir, + filename_tmpl='epoch_{}.pth', + save_optimizer=True, + meta=None, + create_symlink=True): + """Save the checkpoint. + + Args: + out_dir (str): The directory that checkpoints are saved. + filename_tmpl (str, optional): The checkpoint filename template, + which contains a placeholder for the epoch number. + Defaults to 'epoch_{}.pth'. + save_optimizer (bool, optional): Whether to save the optimizer to + the checkpoint. Defaults to True. + meta (dict, optional): The meta information to be saved in the + checkpoint. Defaults to None. + create_symlink (bool, optional): Whether to create a symlink + "latest.pth" to point to the latest checkpoint. + Defaults to True. + """ + if meta is None: + meta = dict(epoch=self.epoch + 1, iter=self.iter) + elif isinstance(meta, dict): + meta.update(epoch=self.epoch + 1, iter=self.iter) + else: + raise TypeError( + f'meta should be a dict or None, but got {type(meta)}') + if self.meta is not None: + meta.update(self.meta) + + filename = filename_tmpl.format(self.epoch + 1) + filepath = osp.join(out_dir, filename) + optimizer = self.optimizer if save_optimizer else None + save_checkpoint(self.model, filepath, optimizer=optimizer, meta=meta) + # in some environments, `os.symlink` is not supported, you may need to + # set `create_symlink` to False + if create_symlink: + dst_file = osp.join(out_dir, 'latest.pth') + if platform.system() != 'Windows': + mmcv.symlink(filename, dst_file) + else: + shutil.copy(filepath, dst_file) + + def resume(self, + checkpoint, + resume_optimizer=True, + map_location='default'): + if map_location == 'default': + if torch.cuda.is_available(): + device_id = torch.cuda.current_device() + checkpoint = self.load_checkpoint( + checkpoint, + map_location=lambda storage, loc: storage.cuda(device_id)) + else: + checkpoint = self.load_checkpoint(checkpoint) + else: + checkpoint = self.load_checkpoint( + checkpoint, map_location=map_location) + + self._epoch = checkpoint['meta']['epoch'] + self._iter = checkpoint['meta']['iter'] + if 'optimizer' in checkpoint and resume_optimizer: + if isinstance(self.optimizer, Optimizer): + self.optimizer.load_state_dict(checkpoint['optimizer']) + elif isinstance(self.optimizer, dict): + for k in self.optimizer.keys(): + self.optimizer[k].load_state_dict( + checkpoint['optimizer'][k]) + else: + raise TypeError( + 'Optimizer should be dict or torch.optim.Optimizer ' + f'but got {type(self.optimizer)}') + + if 'amp' in checkpoint: + apex.amp.load_state_dict(checkpoint['amp']) + self.logger.info('load amp state dict') + + self.logger.info('resumed epoch %d, iter %d', self.epoch, self.iter) diff --git a/segmentation/mmseg/__init__.py b/segmentation/mmseg/__init__.py new file mode 100644 index 0000000..e44ad9b --- /dev/null +++ b/segmentation/mmseg/__init__.py @@ -0,0 +1,30 @@ +import mmcv + +from .version import __version__, version_info + +MMCV_MIN = '1.3.1' +MMCV_MAX = '1.4.0' + + +def digit_version(version_str): + digit_version = [] + for x in version_str.split('.'): + if x.isdigit(): + digit_version.append(int(x)) + elif x.find('rc') != -1: + patch_version = x.split('rc') + digit_version.append(int(patch_version[0]) - 1) + digit_version.append(int(patch_version[1])) + return digit_version + + +mmcv_min_version = digit_version(MMCV_MIN) +mmcv_max_version = digit_version(MMCV_MAX) +mmcv_version = digit_version(mmcv.__version__) + + +# assert (mmcv_min_version <= mmcv_version <= mmcv_max_version), \ +# f'MMCV=={mmcv.__version__} is used but incompatible. ' \ +# f'Please install mmcv>={mmcv_min_version}, <={mmcv_max_version}.' + +__all__ = ['__version__', 'version_info'] diff --git a/segmentation/mmseg/apis/__init__.py b/segmentation/mmseg/apis/__init__.py new file mode 100644 index 0000000..170724b --- /dev/null +++ b/segmentation/mmseg/apis/__init__.py @@ -0,0 +1,9 @@ +from .inference import inference_segmentor, init_segmentor, show_result_pyplot +from .test import multi_gpu_test, single_gpu_test +from .train import get_root_logger, set_random_seed, train_segmentor + +__all__ = [ + 'get_root_logger', 'set_random_seed', 'train_segmentor', 'init_segmentor', + 'inference_segmentor', 'multi_gpu_test', 'single_gpu_test', + 'show_result_pyplot' +] diff --git a/segmentation/mmseg/apis/inference.py b/segmentation/mmseg/apis/inference.py new file mode 100644 index 0000000..bf875cb --- /dev/null +++ b/segmentation/mmseg/apis/inference.py @@ -0,0 +1,135 @@ +import matplotlib.pyplot as plt +import mmcv +import torch +from mmcv.parallel import collate, scatter +from mmcv.runner import load_checkpoint + +from mmseg.datasets.pipelines import Compose +from mmseg.models import build_segmentor + + +def init_segmentor(config, checkpoint=None, device='cuda:0'): + """Initialize a segmentor from config file. + + Args: + config (str or :obj:`mmcv.Config`): Config file path or the config + object. + checkpoint (str, optional): Checkpoint path. If left as None, the model + will not load any weights. + device (str, optional) CPU/CUDA device option. Default 'cuda:0'. + Use 'cpu' for loading model on CPU. + Returns: + nn.Module: The constructed segmentor. + """ + if isinstance(config, str): + config = mmcv.Config.fromfile(config) + elif not isinstance(config, mmcv.Config): + raise TypeError('config must be a filename or Config object, ' + 'but got {}'.format(type(config))) + config.model.pretrained = None + config.model.train_cfg = None + model = build_segmentor(config.model, test_cfg=config.get('test_cfg')) + if checkpoint is not None: + checkpoint = load_checkpoint(model, checkpoint, map_location='cpu') + model.CLASSES = checkpoint['meta']['CLASSES'] + model.PALETTE = checkpoint['meta']['PALETTE'] + model.cfg = config # save the config in the model for convenience + model.to(device) + model.eval() + return model + + +class LoadImage: + """A simple pipeline to load image.""" + + def __call__(self, results): + """Call function to load images into results. + + Args: + results (dict): A result dict contains the file name + of the image to be read. + + Returns: + dict: ``results`` will be returned containing loaded image. + """ + + if isinstance(results['img'], str): + results['filename'] = results['img'] + results['ori_filename'] = results['img'] + else: + results['filename'] = None + results['ori_filename'] = None + img = mmcv.imread(results['img']) + results['img'] = img + results['img_shape'] = img.shape + results['ori_shape'] = img.shape + return results + + +def inference_segmentor(model, img): + """Inference image(s) with the segmentor. + + Args: + model (nn.Module): The loaded segmentor. + imgs (str/ndarray or list[str/ndarray]): Either image files or loaded + images. + + Returns: + (list[Tensor]): The segmentation result. + """ + cfg = model.cfg + device = next(model.parameters()).device # model device + # build the data pipeline + test_pipeline = [LoadImage()] + cfg.data.test.pipeline[1:] + test_pipeline = Compose(test_pipeline) + # prepare data + data = dict(img=img) + data = test_pipeline(data) + data = collate([data], samples_per_gpu=1) + if next(model.parameters()).is_cuda: + # scatter to specified GPU + data = scatter(data, [device])[0] + else: + data['img_metas'] = [i.data[0] for i in data['img_metas']] + + # forward the model + with torch.no_grad(): + result = model(return_loss=False, rescale=True, **data) + return result + + +def show_result_pyplot(model, + img, + result, + palette=None, + fig_size=(15, 10), + opacity=0.5, + title='', + block=True): + """Visualize the segmentation results on the image. + + Args: + model (nn.Module): The loaded segmentor. + img (str or np.ndarray): Image filename or loaded image. + result (list): The segmentation result. + palette (list[list[int]]] | None): The palette of segmentation + map. If None is given, random palette will be generated. + Default: None + fig_size (tuple): Figure size of the pyplot figure. + opacity(float): Opacity of painted segmentation map. + Default 0.5. + Must be in (0, 1] range. + title (str): The title of pyplot figure. + Default is ''. + block (bool): Whether to block the pyplot figure. + Default is True. + """ + if hasattr(model, 'module'): + model = model.module + img = model.show_result( + img, result, palette=palette, show=False, opacity=opacity) + plt.figure(figsize=fig_size) + plt.imshow(mmcv.bgr2rgb(img)) + plt.title(title) + plt.tight_layout() + plt.show(block=block) diff --git a/segmentation/mmseg/apis/test.py b/segmentation/mmseg/apis/test.py new file mode 100644 index 0000000..9728de4 --- /dev/null +++ b/segmentation/mmseg/apis/test.py @@ -0,0 +1,238 @@ +import os.path as osp +import pickle +import shutil +import tempfile + +import mmcv +import numpy as np +import torch +import torch.distributed as dist +from mmcv.image import tensor2imgs +from mmcv.runner import get_dist_info + + +def np2tmp(array, temp_file_name=None): + """Save ndarray to local numpy file. + + Args: + array (ndarray): Ndarray to save. + temp_file_name (str): Numpy file name. If 'temp_file_name=None', this + function will generate a file name with tempfile.NamedTemporaryFile + to save ndarray. Default: None. + + Returns: + str: The numpy file name. + """ + + if temp_file_name is None: + temp_file_name = tempfile.NamedTemporaryFile( + suffix='.npy', delete=False).name + np.save(temp_file_name, array) + return temp_file_name + + +def single_gpu_test(model, + data_loader, + show=False, + out_dir=None, + efficient_test=False, + opacity=0.5): + """Test with single GPU. + + Args: + model (nn.Module): Model to be tested. + data_loader (utils.data.Dataloader): Pytorch data loader. + show (bool): Whether show results during inference. Default: False. + out_dir (str, optional): If specified, the results will be dumped into + the directory to save output results. + efficient_test (bool): Whether save the results as local numpy files to + save CPU memory during evaluation. Default: False. + opacity(float): Opacity of painted segmentation map. + Default 0.5. + Must be in (0, 1] range. + Returns: + list: The prediction results. + """ + + model.eval() + results = [] + dataset = data_loader.dataset + prog_bar = mmcv.ProgressBar(len(dataset)) + for i, data in enumerate(data_loader): + with torch.no_grad(): + result = model(return_loss=False, **data) + + if show or out_dir: + img_tensor = data['img'][0] + img_metas = data['img_metas'][0].data[0] + imgs = tensor2imgs(img_tensor, **img_metas[0]['img_norm_cfg']) + assert len(imgs) == len(img_metas) + + for img, img_meta in zip(imgs, img_metas): + h, w, _ = img_meta['img_shape'] + img_show = img[:h, :w, :] + + ori_h, ori_w = img_meta['ori_shape'][:-1] + img_show = mmcv.imresize(img_show, (ori_w, ori_h)) + + if out_dir: + out_file = osp.join(out_dir, img_meta['ori_filename']) + else: + out_file = None + + model.module.show_result( + img_show, + result, + palette=dataset.PALETTE, + show=show, + out_file=out_file, + opacity=opacity) + + if isinstance(result, list): + if efficient_test: + result = [np2tmp(_) for _ in result] + results.extend(result) + else: + if efficient_test: + result = np2tmp(result) + results.append(result) + + batch_size = len(result) + for _ in range(batch_size): + prog_bar.update() + return results + + +def multi_gpu_test(model, + data_loader, + tmpdir=None, + gpu_collect=False, + efficient_test=False): + """Test model with multiple gpus. + + This method tests model with multiple gpus and collects the results + under two different modes: gpu and cpu modes. By setting 'gpu_collect=True' + it encodes results to gpu tensors and use gpu communication for results + collection. On cpu mode it saves the results on different gpus to 'tmpdir' + and collects them by the rank 0 worker. + + Args: + model (nn.Module): Model to be tested. + data_loader (utils.data.Dataloader): Pytorch data loader. + tmpdir (str): Path of directory to save the temporary results from + different gpus under cpu mode. + gpu_collect (bool): Option to use either gpu or cpu to collect results. + efficient_test (bool): Whether save the results as local numpy files to + save CPU memory during evaluation. Default: False. + + Returns: + list: The prediction results. + """ + + model.eval() + results = [] + dataset = data_loader.dataset + rank, world_size = get_dist_info() + if rank == 0: + prog_bar = mmcv.ProgressBar(len(dataset)) + for i, data in enumerate(data_loader): + with torch.no_grad(): + result = model(return_loss=False, rescale=True, **data) + + if isinstance(result, list): + if efficient_test: + result = [np2tmp(_) for _ in result] + results.extend(result) + else: + if efficient_test: + result = np2tmp(result) + results.append(result) + + if rank == 0: + batch_size = len(result) + for _ in range(batch_size * world_size): + prog_bar.update() + + # collect results from all ranks + if gpu_collect: + results = collect_results_gpu(results, len(dataset)) + else: + results = collect_results_cpu(results, len(dataset), tmpdir) + return results + + +def collect_results_cpu(result_part, size, tmpdir=None): + """Collect results with CPU.""" + rank, world_size = get_dist_info() + # create a tmp dir if it is not specified + if tmpdir is None: + MAX_LEN = 512 + # 32 is whitespace + dir_tensor = torch.full((MAX_LEN, ), + 32, + dtype=torch.uint8, + device='cuda') + if rank == 0: + tmpdir = tempfile.mkdtemp() + tmpdir = torch.tensor( + bytearray(tmpdir.encode()), dtype=torch.uint8, device='cuda') + dir_tensor[:len(tmpdir)] = tmpdir + dist.broadcast(dir_tensor, 0) + tmpdir = dir_tensor.cpu().numpy().tobytes().decode().rstrip() + else: + mmcv.mkdir_or_exist(tmpdir) + # dump the part result to the dir + mmcv.dump(result_part, osp.join(tmpdir, 'part_{}.pkl'.format(rank))) + dist.barrier() + # collect all parts + if rank != 0: + return None + else: + # load results of all parts from tmp dir + part_list = [] + for i in range(world_size): + part_file = osp.join(tmpdir, 'part_{}.pkl'.format(i)) + part_list.append(mmcv.load(part_file)) + # sort the results + ordered_results = [] + for res in zip(*part_list): + ordered_results.extend(list(res)) + # the dataloader may pad some samples + ordered_results = ordered_results[:size] + # remove tmp dir + shutil.rmtree(tmpdir) + return ordered_results + + +def collect_results_gpu(result_part, size): + """Collect results with GPU.""" + rank, world_size = get_dist_info() + # dump result part to tensor with pickle + part_tensor = torch.tensor( + bytearray(pickle.dumps(result_part)), dtype=torch.uint8, device='cuda') + # gather all result part tensor shape + shape_tensor = torch.tensor(part_tensor.shape, device='cuda') + shape_list = [shape_tensor.clone() for _ in range(world_size)] + dist.all_gather(shape_list, shape_tensor) + # padding result part tensor to max length + shape_max = torch.tensor(shape_list).max() + part_send = torch.zeros(shape_max, dtype=torch.uint8, device='cuda') + part_send[:shape_tensor[0]] = part_tensor + part_recv_list = [ + part_tensor.new_zeros(shape_max) for _ in range(world_size) + ] + # gather all result part + dist.all_gather(part_recv_list, part_send) + + if rank == 0: + part_list = [] + for recv, shape in zip(part_recv_list, shape_list): + part_list.append( + pickle.loads(recv[:shape[0]].cpu().numpy().tobytes())) + # sort the results + ordered_results = [] + for res in zip(*part_list): + ordered_results.extend(list(res)) + # the dataloader may pad some samples + ordered_results = ordered_results[:size] + return ordered_results diff --git a/segmentation/mmseg/apis/train.py b/segmentation/mmseg/apis/train.py new file mode 100644 index 0000000..5f526df --- /dev/null +++ b/segmentation/mmseg/apis/train.py @@ -0,0 +1,116 @@ +import random +import warnings + +import numpy as np +import torch +from mmcv.parallel import MMDataParallel, MMDistributedDataParallel +from mmcv.runner import build_optimizer, build_runner + +from mmseg.core import DistEvalHook, EvalHook +from mmseg.datasets import build_dataloader, build_dataset +from mmseg.utils import get_root_logger + + +def set_random_seed(seed, deterministic=False): + """Set random seed. + + Args: + seed (int): Seed to be used. + deterministic (bool): Whether to set the deterministic option for + CUDNN backend, i.e., set `torch.backends.cudnn.deterministic` + to True and `torch.backends.cudnn.benchmark` to False. + Default: False. + """ + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + if deterministic: + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + + +def train_segmentor(model, + dataset, + cfg, + distributed=False, + validate=False, + timestamp=None, + meta=None): + """Launch segmentor training.""" + logger = get_root_logger(cfg.log_level) + + # prepare data loaders + dataset = dataset if isinstance(dataset, (list, tuple)) else [dataset] + data_loaders = [ + build_dataloader( + ds, + cfg.data.samples_per_gpu, + cfg.data.workers_per_gpu, + # cfg.gpus will be ignored if distributed + len(cfg.gpu_ids), + dist=distributed, + seed=cfg.seed, + drop_last=True) for ds in dataset + ] + + # put model on gpus + if distributed: + find_unused_parameters = cfg.get('find_unused_parameters', False) + # Sets the `find_unused_parameters` parameter in + # torch.nn.parallel.DistributedDataParallel + model = MMDistributedDataParallel( + model.cuda(), + device_ids=[torch.cuda.current_device()], + broadcast_buffers=False, + find_unused_parameters=find_unused_parameters) + else: + model = MMDataParallel( + model.cuda(cfg.gpu_ids[0]), device_ids=cfg.gpu_ids) + + # build runner + optimizer = build_optimizer(model, cfg.optimizer) + + if cfg.get('runner') is None: + cfg.runner = {'type': 'IterBasedRunner', 'max_iters': cfg.total_iters} + warnings.warn( + 'config is now expected to have a `runner` section, ' + 'please set `runner` in your config.', UserWarning) + + runner = build_runner( + cfg.runner, + default_args=dict( + model=model, + batch_processor=None, + optimizer=optimizer, + work_dir=cfg.work_dir, + logger=logger, + meta=meta)) + + # register hooks + runner.register_training_hooks(cfg.lr_config, cfg.optimizer_config, + cfg.checkpoint_config, cfg.log_config, + cfg.get('momentum_config', None)) + + # an ugly walkaround to make the .log and .log.json filenames the same + runner.timestamp = timestamp + + # register eval hooks + if validate: + val_dataset = build_dataset(cfg.data.val, dict(test_mode=True)) + val_dataloader = build_dataloader( + val_dataset, + samples_per_gpu=1, + workers_per_gpu=cfg.data.workers_per_gpu, + dist=distributed, + shuffle=False) + eval_cfg = cfg.get('evaluation', {}) + eval_cfg['by_epoch'] = cfg.runner['type'] != 'IterBasedRunner' + eval_hook = DistEvalHook if distributed else EvalHook + runner.register_hook(eval_hook(val_dataloader, **eval_cfg)) + + if cfg.resume_from: + runner.resume(cfg.resume_from) + elif cfg.load_from: + runner.load_checkpoint(cfg.load_from) + runner.run(data_loaders, cfg.workflow) diff --git a/segmentation/mmseg/core/__init__.py b/segmentation/mmseg/core/__init__.py new file mode 100644 index 0000000..9656055 --- /dev/null +++ b/segmentation/mmseg/core/__init__.py @@ -0,0 +1,3 @@ +from .evaluation import * # noqa: F401, F403 +from .seg import * # noqa: F401, F403 +from .utils import * # noqa: F401, F403 diff --git a/segmentation/mmseg/core/evaluation/__init__.py b/segmentation/mmseg/core/evaluation/__init__.py new file mode 100644 index 0000000..f7cc4b2 --- /dev/null +++ b/segmentation/mmseg/core/evaluation/__init__.py @@ -0,0 +1,8 @@ +from .class_names import get_classes, get_palette +from .eval_hooks import DistEvalHook, EvalHook +from .metrics import eval_metrics, mean_dice, mean_fscore, mean_iou + +__all__ = [ + 'EvalHook', 'DistEvalHook', 'mean_dice', 'mean_iou', 'mean_fscore', + 'eval_metrics', 'get_classes', 'get_palette' +] diff --git a/segmentation/mmseg/core/evaluation/class_names.py b/segmentation/mmseg/core/evaluation/class_names.py new file mode 100644 index 0000000..0d8e66d --- /dev/null +++ b/segmentation/mmseg/core/evaluation/class_names.py @@ -0,0 +1,152 @@ +import mmcv + + +def cityscapes_classes(): + """Cityscapes class names for external use.""" + return [ + 'road', 'sidewalk', 'building', 'wall', 'fence', 'pole', + 'traffic light', 'traffic sign', 'vegetation', 'terrain', 'sky', + 'person', 'rider', 'car', 'truck', 'bus', 'train', 'motorcycle', + 'bicycle' + ] + + +def ade_classes(): + """ADE20K class names for external use.""" + return [ + 'wall', 'building', 'sky', 'floor', 'tree', 'ceiling', 'road', 'bed ', + 'windowpane', 'grass', 'cabinet', 'sidewalk', 'person', 'earth', + 'door', 'table', 'mountain', 'plant', 'curtain', 'chair', 'car', + 'water', 'painting', 'sofa', 'shelf', 'house', 'sea', 'mirror', 'rug', + 'field', 'armchair', 'seat', 'fence', 'desk', 'rock', 'wardrobe', + 'lamp', 'bathtub', 'railing', 'cushion', 'base', 'box', 'column', + 'signboard', 'chest of drawers', 'counter', 'sand', 'sink', + 'skyscraper', 'fireplace', 'refrigerator', 'grandstand', 'path', + 'stairs', 'runway', 'case', 'pool table', 'pillow', 'screen door', + 'stairway', 'river', 'bridge', 'bookcase', 'blind', 'coffee table', + 'toilet', 'flower', 'book', 'hill', 'bench', 'countertop', 'stove', + 'palm', 'kitchen island', 'computer', 'swivel chair', 'boat', 'bar', + 'arcade machine', 'hovel', 'bus', 'towel', 'light', 'truck', 'tower', + 'chandelier', 'awning', 'streetlight', 'booth', 'television receiver', + 'airplane', 'dirt track', 'apparel', 'pole', 'land', 'bannister', + 'escalator', 'ottoman', 'bottle', 'buffet', 'poster', 'stage', 'van', + 'ship', 'fountain', 'conveyer belt', 'canopy', 'washer', 'plaything', + 'swimming pool', 'stool', 'barrel', 'basket', 'waterfall', 'tent', + 'bag', 'minibike', 'cradle', 'oven', 'ball', 'food', 'step', 'tank', + 'trade name', 'microwave', 'pot', 'animal', 'bicycle', 'lake', + 'dishwasher', 'screen', 'blanket', 'sculpture', 'hood', 'sconce', + 'vase', 'traffic light', 'tray', 'ashcan', 'fan', 'pier', 'crt screen', + 'plate', 'monitor', 'bulletin board', 'shower', 'radiator', 'glass', + 'clock', 'flag' + ] + + +def voc_classes(): + """Pascal VOC class names for external use.""" + return [ + 'background', 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', + 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', + 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', + 'tvmonitor' + ] + + +def cityscapes_palette(): + """Cityscapes palette for external use.""" + return [[128, 64, 128], [244, 35, 232], [70, 70, 70], [102, 102, 156], + [190, 153, 153], [153, 153, 153], [250, 170, 30], [220, 220, 0], + [107, 142, 35], [152, 251, 152], [70, 130, 180], [220, 20, 60], + [255, 0, 0], [0, 0, 142], [0, 0, 70], [0, 60, 100], [0, 80, 100], + [0, 0, 230], [119, 11, 32]] + + +def ade_palette(): + """ADE20K palette for external use.""" + return [[120, 120, 120], [180, 120, 120], [6, 230, 230], [80, 50, 50], + [4, 200, 3], [120, 120, 80], [140, 140, 140], [204, 5, 255], + [230, 230, 230], [4, 250, 7], [224, 5, 255], [235, 255, 7], + [150, 5, 61], [120, 120, 70], [8, 255, 51], [255, 6, 82], + [143, 255, 140], [204, 255, 4], [255, 51, 7], [204, 70, 3], + [0, 102, 200], [61, 230, 250], [255, 6, 51], [11, 102, 255], + [255, 7, 71], [255, 9, 224], [9, 7, 230], [220, 220, 220], + [255, 9, 92], [112, 9, 255], [8, 255, 214], [7, 255, 224], + [255, 184, 6], [10, 255, 71], [255, 41, 10], [7, 255, 255], + [224, 255, 8], [102, 8, 255], [255, 61, 6], [255, 194, 7], + [255, 122, 8], [0, 255, 20], [255, 8, 41], [255, 5, 153], + [6, 51, 255], [235, 12, 255], [160, 150, 20], [0, 163, 255], + [140, 140, 140], [250, 10, 15], [20, 255, 0], [31, 255, 0], + [255, 31, 0], [255, 224, 0], [153, 255, 0], [0, 0, 255], + [255, 71, 0], [0, 235, 255], [0, 173, 255], [31, 0, 255], + [11, 200, 200], [255, 82, 0], [0, 255, 245], [0, 61, 255], + [0, 255, 112], [0, 255, 133], [255, 0, 0], [255, 163, 0], + [255, 102, 0], [194, 255, 0], [0, 143, 255], [51, 255, 0], + [0, 82, 255], [0, 255, 41], [0, 255, 173], [10, 0, 255], + [173, 255, 0], [0, 255, 153], [255, 92, 0], [255, 0, 255], + [255, 0, 245], [255, 0, 102], [255, 173, 0], [255, 0, 20], + [255, 184, 184], [0, 31, 255], [0, 255, 61], [0, 71, 255], + [255, 0, 204], [0, 255, 194], [0, 255, 82], [0, 10, 255], + [0, 112, 255], [51, 0, 255], [0, 194, 255], [0, 122, 255], + [0, 255, 163], [255, 153, 0], [0, 255, 10], [255, 112, 0], + [143, 255, 0], [82, 0, 255], [163, 255, 0], [255, 235, 0], + [8, 184, 170], [133, 0, 255], [0, 255, 92], [184, 0, 255], + [255, 0, 31], [0, 184, 255], [0, 214, 255], [255, 0, 112], + [92, 255, 0], [0, 224, 255], [112, 224, 255], [70, 184, 160], + [163, 0, 255], [153, 0, 255], [71, 255, 0], [255, 0, 163], + [255, 204, 0], [255, 0, 143], [0, 255, 235], [133, 255, 0], + [255, 0, 235], [245, 0, 255], [255, 0, 122], [255, 245, 0], + [10, 190, 212], [214, 255, 0], [0, 204, 255], [20, 0, 255], + [255, 255, 0], [0, 153, 255], [0, 41, 255], [0, 255, 204], + [41, 0, 255], [41, 255, 0], [173, 0, 255], [0, 245, 255], + [71, 0, 255], [122, 0, 255], [0, 255, 184], [0, 92, 255], + [184, 255, 0], [0, 133, 255], [255, 214, 0], [25, 194, 194], + [102, 255, 0], [92, 0, 255]] + + +def voc_palette(): + """Pascal VOC palette for external use.""" + return [[0, 0, 0], [128, 0, 0], [0, 128, 0], [128, 128, 0], [0, 0, 128], + [128, 0, 128], [0, 128, 128], [128, 128, 128], [64, 0, 0], + [192, 0, 0], [64, 128, 0], [192, 128, 0], [64, 0, 128], + [192, 0, 128], [64, 128, 128], [192, 128, 128], [0, 64, 0], + [128, 64, 0], [0, 192, 0], [128, 192, 0], [0, 64, 128]] + + +dataset_aliases = { + 'cityscapes': ['cityscapes'], + 'ade': ['ade', 'ade20k'], + 'voc': ['voc', 'pascal_voc', 'voc12', 'voc12aug'] +} + + +def get_classes(dataset): + """Get class names of a dataset.""" + alias2name = {} + for name, aliases in dataset_aliases.items(): + for alias in aliases: + alias2name[alias] = name + + if mmcv.is_str(dataset): + if dataset in alias2name: + labels = eval(alias2name[dataset] + '_classes()') + else: + raise ValueError(f'Unrecognized dataset: {dataset}') + else: + raise TypeError(f'dataset must a str, but got {type(dataset)}') + return labels + + +def get_palette(dataset): + """Get class palette (RGB) of a dataset.""" + alias2name = {} + for name, aliases in dataset_aliases.items(): + for alias in aliases: + alias2name[alias] = name + + if mmcv.is_str(dataset): + if dataset in alias2name: + labels = eval(alias2name[dataset] + '_palette()') + else: + raise ValueError(f'Unrecognized dataset: {dataset}') + else: + raise TypeError(f'dataset must a str, but got {type(dataset)}') + return labels diff --git a/segmentation/mmseg/core/evaluation/eval_hooks.py b/segmentation/mmseg/core/evaluation/eval_hooks.py new file mode 100644 index 0000000..34c44c7 --- /dev/null +++ b/segmentation/mmseg/core/evaluation/eval_hooks.py @@ -0,0 +1,109 @@ +import os.path as osp + +from mmcv.runner import DistEvalHook as _DistEvalHook +from mmcv.runner import EvalHook as _EvalHook + + +class EvalHook(_EvalHook): + """Single GPU EvalHook, with efficient test support. + + Args: + by_epoch (bool): Determine perform evaluation by epoch or by iteration. + If set to True, it will perform by epoch. Otherwise, by iteration. + Default: False. + efficient_test (bool): Whether save the results as local numpy files to + save CPU memory during evaluation. Default: False. + Returns: + list: The prediction results. + """ + + greater_keys = ['mIoU', 'mAcc', 'aAcc'] + + def __init__(self, *args, by_epoch=False, efficient_test=False, **kwargs): + super().__init__(*args, by_epoch=by_epoch, **kwargs) + self.efficient_test = efficient_test + + def after_train_iter(self, runner): + """After train epoch hook. + + Override default ``single_gpu_test``. + """ + if self.by_epoch or not self.every_n_iters(runner, self.interval): + return + from mmseg.apis import single_gpu_test + runner.log_buffer.clear() + results = single_gpu_test( + runner.model, + self.dataloader, + show=False, + efficient_test=self.efficient_test) + self.evaluate(runner, results) + + def after_train_epoch(self, runner): + """After train epoch hook. + + Override default ``single_gpu_test``. + """ + if not self.by_epoch or not self.every_n_epochs(runner, self.interval): + return + from mmseg.apis import single_gpu_test + runner.log_buffer.clear() + results = single_gpu_test(runner.model, self.dataloader, show=False) + self.evaluate(runner, results) + + +class DistEvalHook(_DistEvalHook): + """Distributed EvalHook, with efficient test support. + + Args: + by_epoch (bool): Determine perform evaluation by epoch or by iteration. + If set to True, it will perform by epoch. Otherwise, by iteration. + Default: False. + efficient_test (bool): Whether save the results as local numpy files to + save CPU memory during evaluation. Default: False. + Returns: + list: The prediction results. + """ + + greater_keys = ['mIoU', 'mAcc', 'aAcc'] + + def __init__(self, *args, by_epoch=False, efficient_test=False, **kwargs): + super().__init__(*args, by_epoch=by_epoch, **kwargs) + self.efficient_test = efficient_test + + def after_train_iter(self, runner): + """After train epoch hook. + + Override default ``multi_gpu_test``. + """ + if self.by_epoch or not self.every_n_iters(runner, self.interval): + return + from mmseg.apis import multi_gpu_test + runner.log_buffer.clear() + results = multi_gpu_test( + runner.model, + self.dataloader, + tmpdir=osp.join(runner.work_dir, '.eval_hook'), + gpu_collect=self.gpu_collect, + efficient_test=self.efficient_test) + if runner.rank == 0: + print('\n') + self.evaluate(runner, results) + + def after_train_epoch(self, runner): + """After train epoch hook. + + Override default ``multi_gpu_test``. + """ + if not self.by_epoch or not self.every_n_epochs(runner, self.interval): + return + from mmseg.apis import multi_gpu_test + runner.log_buffer.clear() + results = multi_gpu_test( + runner.model, + self.dataloader, + tmpdir=osp.join(runner.work_dir, '.eval_hook'), + gpu_collect=self.gpu_collect) + if runner.rank == 0: + print('\n') + self.evaluate(runner, results) diff --git a/segmentation/mmseg/core/evaluation/metrics.py b/segmentation/mmseg/core/evaluation/metrics.py new file mode 100644 index 0000000..a216afe --- /dev/null +++ b/segmentation/mmseg/core/evaluation/metrics.py @@ -0,0 +1,326 @@ +from collections import OrderedDict + +import mmcv +import numpy as np +import torch + + +def f_score(precision, recall, beta=1): + """calcuate the f-score value. + + Args: + precision (float | torch.Tensor): The precision value. + recall (float | torch.Tensor): The recall value. + beta (int): Determines the weight of recall in the combined score. + Default: False. + + Returns: + [torch.tensor]: The f-score value. + """ + score = (1 + beta**2) * (precision * recall) / ( + (beta**2 * precision) + recall) + return score + + +def intersect_and_union(pred_label, + label, + num_classes, + ignore_index, + label_map=dict(), + reduce_zero_label=False): + """Calculate intersection and Union. + + Args: + pred_label (ndarray | str): Prediction segmentation map + or predict result filename. + label (ndarray | str): Ground truth segmentation map + or label filename. + num_classes (int): Number of categories. + ignore_index (int): Index that will be ignored in evaluation. + label_map (dict): Mapping old labels to new labels. The parameter will + work only when label is str. Default: dict(). + reduce_zero_label (bool): Wether ignore zero label. The parameter will + work only when label is str. Default: False. + + Returns: + torch.Tensor: The intersection of prediction and ground truth + histogram on all classes. + torch.Tensor: The union of prediction and ground truth histogram on + all classes. + torch.Tensor: The prediction histogram on all classes. + torch.Tensor: The ground truth histogram on all classes. + """ + + if isinstance(pred_label, str): + pred_label = torch.from_numpy(np.load(pred_label)) + else: + pred_label = torch.from_numpy((pred_label)) + + if isinstance(label, str): + label = torch.from_numpy( + mmcv.imread(label, flag='unchanged', backend='pillow')) + else: + label = torch.from_numpy(label) + + if label_map is not None: + for old_id, new_id in label_map.items(): + label[label == old_id] = new_id + if reduce_zero_label: + label[label == 0] = 255 + label = label - 1 + label[label == 254] = 255 + + mask = (label != ignore_index) + pred_label = pred_label[mask] + label = label[mask] + + intersect = pred_label[pred_label == label] + area_intersect = torch.histc( + intersect.float(), bins=(num_classes), min=0, max=num_classes - 1) + area_pred_label = torch.histc( + pred_label.float(), bins=(num_classes), min=0, max=num_classes - 1) + area_label = torch.histc( + label.float(), bins=(num_classes), min=0, max=num_classes - 1) + area_union = area_pred_label + area_label - area_intersect + return area_intersect, area_union, area_pred_label, area_label + + +def total_intersect_and_union(results, + gt_seg_maps, + num_classes, + ignore_index, + label_map=dict(), + reduce_zero_label=False): + """Calculate Total Intersection and Union. + + Args: + results (list[ndarray] | list[str]): List of prediction segmentation + maps or list of prediction result filenames. + gt_seg_maps (list[ndarray] | list[str]): list of ground truth + segmentation maps or list of label filenames. + num_classes (int): Number of categories. + ignore_index (int): Index that will be ignored in evaluation. + label_map (dict): Mapping old labels to new labels. Default: dict(). + reduce_zero_label (bool): Wether ignore zero label. Default: False. + + Returns: + ndarray: The intersection of prediction and ground truth histogram + on all classes. + ndarray: The union of prediction and ground truth histogram on all + classes. + ndarray: The prediction histogram on all classes. + ndarray: The ground truth histogram on all classes. + """ + num_imgs = len(results) + assert len(gt_seg_maps) == num_imgs + total_area_intersect = torch.zeros((num_classes, ), dtype=torch.float64) + total_area_union = torch.zeros((num_classes, ), dtype=torch.float64) + total_area_pred_label = torch.zeros((num_classes, ), dtype=torch.float64) + total_area_label = torch.zeros((num_classes, ), dtype=torch.float64) + for i in range(num_imgs): + area_intersect, area_union, area_pred_label, area_label = \ + intersect_and_union( + results[i], gt_seg_maps[i], num_classes, ignore_index, + label_map, reduce_zero_label) + total_area_intersect += area_intersect + total_area_union += area_union + total_area_pred_label += area_pred_label + total_area_label += area_label + return total_area_intersect, total_area_union, total_area_pred_label, \ + total_area_label + + +def mean_iou(results, + gt_seg_maps, + num_classes, + ignore_index, + nan_to_num=None, + label_map=dict(), + reduce_zero_label=False): + """Calculate Mean Intersection and Union (mIoU) + + Args: + results (list[ndarray] | list[str]): List of prediction segmentation + maps or list of prediction result filenames. + gt_seg_maps (list[ndarray] | list[str]): list of ground truth + segmentation maps or list of label filenames. + num_classes (int): Number of categories. + ignore_index (int): Index that will be ignored in evaluation. + nan_to_num (int, optional): If specified, NaN values will be replaced + by the numbers defined by the user. Default: None. + label_map (dict): Mapping old labels to new labels. Default: dict(). + reduce_zero_label (bool): Wether ignore zero label. Default: False. + + Returns: + dict[str, float | ndarray]: + float: Overall accuracy on all images. + ndarray: Per category accuracy, shape (num_classes, ). + ndarray: Per category IoU, shape (num_classes, ). + """ + iou_result = eval_metrics( + results=results, + gt_seg_maps=gt_seg_maps, + num_classes=num_classes, + ignore_index=ignore_index, + metrics=['mIoU'], + nan_to_num=nan_to_num, + label_map=label_map, + reduce_zero_label=reduce_zero_label) + return iou_result + + +def mean_dice(results, + gt_seg_maps, + num_classes, + ignore_index, + nan_to_num=None, + label_map=dict(), + reduce_zero_label=False): + """Calculate Mean Dice (mDice) + + Args: + results (list[ndarray] | list[str]): List of prediction segmentation + maps or list of prediction result filenames. + gt_seg_maps (list[ndarray] | list[str]): list of ground truth + segmentation maps or list of label filenames. + num_classes (int): Number of categories. + ignore_index (int): Index that will be ignored in evaluation. + nan_to_num (int, optional): If specified, NaN values will be replaced + by the numbers defined by the user. Default: None. + label_map (dict): Mapping old labels to new labels. Default: dict(). + reduce_zero_label (bool): Wether ignore zero label. Default: False. + + Returns: + dict[str, float | ndarray]: Default metrics. + float: Overall accuracy on all images. + ndarray: Per category accuracy, shape (num_classes, ). + ndarray: Per category dice, shape (num_classes, ). + """ + + dice_result = eval_metrics( + results=results, + gt_seg_maps=gt_seg_maps, + num_classes=num_classes, + ignore_index=ignore_index, + metrics=['mDice'], + nan_to_num=nan_to_num, + label_map=label_map, + reduce_zero_label=reduce_zero_label) + return dice_result + + +def mean_fscore(results, + gt_seg_maps, + num_classes, + ignore_index, + nan_to_num=None, + label_map=dict(), + reduce_zero_label=False, + beta=1): + """Calculate Mean Intersection and Union (mIoU) + + Args: + results (list[ndarray] | list[str]): List of prediction segmentation + maps or list of prediction result filenames. + gt_seg_maps (list[ndarray] | list[str]): list of ground truth + segmentation maps or list of label filenames. + num_classes (int): Number of categories. + ignore_index (int): Index that will be ignored in evaluation. + nan_to_num (int, optional): If specified, NaN values will be replaced + by the numbers defined by the user. Default: None. + label_map (dict): Mapping old labels to new labels. Default: dict(). + reduce_zero_label (bool): Wether ignore zero label. Default: False. + beta (int): Determines the weight of recall in the combined score. + Default: False. + + + Returns: + dict[str, float | ndarray]: Default metrics. + float: Overall accuracy on all images. + ndarray: Per category recall, shape (num_classes, ). + ndarray: Per category precision, shape (num_classes, ). + ndarray: Per category f-score, shape (num_classes, ). + """ + fscore_result = eval_metrics( + results=results, + gt_seg_maps=gt_seg_maps, + num_classes=num_classes, + ignore_index=ignore_index, + metrics=['mFscore'], + nan_to_num=nan_to_num, + label_map=label_map, + reduce_zero_label=reduce_zero_label, + beta=beta) + return fscore_result + + +def eval_metrics(results, + gt_seg_maps, + num_classes, + ignore_index, + metrics=['mIoU'], + nan_to_num=None, + label_map=dict(), + reduce_zero_label=False, + beta=1): + """Calculate evaluation metrics + Args: + results (list[ndarray] | list[str]): List of prediction segmentation + maps or list of prediction result filenames. + gt_seg_maps (list[ndarray] | list[str]): list of ground truth + segmentation maps or list of label filenames. + num_classes (int): Number of categories. + ignore_index (int): Index that will be ignored in evaluation. + metrics (list[str] | str): Metrics to be evaluated, 'mIoU' and 'mDice'. + nan_to_num (int, optional): If specified, NaN values will be replaced + by the numbers defined by the user. Default: None. + label_map (dict): Mapping old labels to new labels. Default: dict(). + reduce_zero_label (bool): Wether ignore zero label. Default: False. + Returns: + float: Overall accuracy on all images. + ndarray: Per category accuracy, shape (num_classes, ). + ndarray: Per category evaluation metrics, shape (num_classes, ). + """ + if isinstance(metrics, str): + metrics = [metrics] + allowed_metrics = ['mIoU', 'mDice', 'mFscore'] + if not set(metrics).issubset(set(allowed_metrics)): + raise KeyError('metrics {} is not supported'.format(metrics)) + + total_area_intersect, total_area_union, total_area_pred_label, \ + total_area_label = total_intersect_and_union( + results, gt_seg_maps, num_classes, ignore_index, label_map, + reduce_zero_label) + all_acc = total_area_intersect.sum() / total_area_label.sum() + ret_metrics = OrderedDict({'aAcc': all_acc}) + for metric in metrics: + if metric == 'mIoU': + iou = total_area_intersect / total_area_union + acc = total_area_intersect / total_area_label + ret_metrics['IoU'] = iou + ret_metrics['Acc'] = acc + elif metric == 'mDice': + dice = 2 * total_area_intersect / ( + total_area_pred_label + total_area_label) + acc = total_area_intersect / total_area_label + ret_metrics['Dice'] = dice + ret_metrics['Acc'] = acc + elif metric == 'mFscore': + precision = total_area_intersect / total_area_pred_label + recall = total_area_intersect / total_area_label + f_value = torch.tensor( + [f_score(x[0], x[1], beta) for x in zip(precision, recall)]) + ret_metrics['Fscore'] = f_value + ret_metrics['Precision'] = precision + ret_metrics['Recall'] = recall + + ret_metrics = { + metric: value.numpy() + for metric, value in ret_metrics.items() + } + if nan_to_num is not None: + ret_metrics = OrderedDict({ + metric: np.nan_to_num(metric_value, nan=nan_to_num) + for metric, metric_value in ret_metrics.items() + }) + return ret_metrics diff --git a/segmentation/mmseg/core/seg/__init__.py b/segmentation/mmseg/core/seg/__init__.py new file mode 100644 index 0000000..93bc129 --- /dev/null +++ b/segmentation/mmseg/core/seg/__init__.py @@ -0,0 +1,4 @@ +from .builder import build_pixel_sampler +from .sampler import BasePixelSampler, OHEMPixelSampler + +__all__ = ['build_pixel_sampler', 'BasePixelSampler', 'OHEMPixelSampler'] diff --git a/segmentation/mmseg/core/seg/builder.py b/segmentation/mmseg/core/seg/builder.py new file mode 100644 index 0000000..f5a117c --- /dev/null +++ b/segmentation/mmseg/core/seg/builder.py @@ -0,0 +1,8 @@ +from mmcv.utils import Registry, build_from_cfg + +PIXEL_SAMPLERS = Registry('pixel sampler') + + +def build_pixel_sampler(cfg, **default_args): + """Build pixel sampler for segmentation map.""" + return build_from_cfg(cfg, PIXEL_SAMPLERS, default_args) diff --git a/segmentation/mmseg/core/seg/sampler/__init__.py b/segmentation/mmseg/core/seg/sampler/__init__.py new file mode 100644 index 0000000..332b242 --- /dev/null +++ b/segmentation/mmseg/core/seg/sampler/__init__.py @@ -0,0 +1,4 @@ +from .base_pixel_sampler import BasePixelSampler +from .ohem_pixel_sampler import OHEMPixelSampler + +__all__ = ['BasePixelSampler', 'OHEMPixelSampler'] diff --git a/segmentation/mmseg/core/seg/sampler/base_pixel_sampler.py b/segmentation/mmseg/core/seg/sampler/base_pixel_sampler.py new file mode 100644 index 0000000..b75b156 --- /dev/null +++ b/segmentation/mmseg/core/seg/sampler/base_pixel_sampler.py @@ -0,0 +1,12 @@ +from abc import ABCMeta, abstractmethod + + +class BasePixelSampler(metaclass=ABCMeta): + """Base class of pixel sampler.""" + + def __init__(self, **kwargs): + pass + + @abstractmethod + def sample(self, seg_logit, seg_label): + """Placeholder for sample function.""" diff --git a/segmentation/mmseg/core/seg/sampler/ohem_pixel_sampler.py b/segmentation/mmseg/core/seg/sampler/ohem_pixel_sampler.py new file mode 100644 index 0000000..88bb10d --- /dev/null +++ b/segmentation/mmseg/core/seg/sampler/ohem_pixel_sampler.py @@ -0,0 +1,76 @@ +import torch +import torch.nn.functional as F + +from ..builder import PIXEL_SAMPLERS +from .base_pixel_sampler import BasePixelSampler + + +@PIXEL_SAMPLERS.register_module() +class OHEMPixelSampler(BasePixelSampler): + """Online Hard Example Mining Sampler for segmentation. + + Args: + context (nn.Module): The context of sampler, subclass of + :obj:`BaseDecodeHead`. + thresh (float, optional): The threshold for hard example selection. + Below which, are prediction with low confidence. If not + specified, the hard examples will be pixels of top ``min_kept`` + loss. Default: None. + min_kept (int, optional): The minimum number of predictions to keep. + Default: 100000. + """ + + def __init__(self, context, thresh=None, min_kept=100000): + super(OHEMPixelSampler, self).__init__() + self.context = context + assert min_kept > 1 + self.thresh = thresh + self.min_kept = min_kept + + def sample(self, seg_logit, seg_label): + """Sample pixels that have high loss or with low prediction confidence. + + Args: + seg_logit (torch.Tensor): segmentation logits, shape (N, C, H, W) + seg_label (torch.Tensor): segmentation label, shape (N, 1, H, W) + + Returns: + torch.Tensor: segmentation weight, shape (N, H, W) + """ + with torch.no_grad(): + assert seg_logit.shape[2:] == seg_label.shape[2:] + assert seg_label.shape[1] == 1 + seg_label = seg_label.squeeze(1).long() + batch_kept = self.min_kept * seg_label.size(0) + valid_mask = seg_label != self.context.ignore_index + seg_weight = seg_logit.new_zeros(size=seg_label.size()) + valid_seg_weight = seg_weight[valid_mask] + if self.thresh is not None: + seg_prob = F.softmax(seg_logit, dim=1) + + tmp_seg_label = seg_label.clone().unsqueeze(1) + tmp_seg_label[tmp_seg_label == self.context.ignore_index] = 0 + seg_prob = seg_prob.gather(1, tmp_seg_label).squeeze(1) + sort_prob, sort_indices = seg_prob[valid_mask].sort() + + if sort_prob.numel() > 0: + min_threshold = sort_prob[min(batch_kept, + sort_prob.numel() - 1)] + else: + min_threshold = 0.0 + threshold = max(min_threshold, self.thresh) + valid_seg_weight[seg_prob[valid_mask] < threshold] = 1. + else: + losses = self.context.loss_decode( + seg_logit, + seg_label, + weight=None, + ignore_index=self.context.ignore_index, + reduction_override='none') + # faster than topk according to https://github.com/pytorch/pytorch/issues/22812 # noqa + _, sort_indices = losses[valid_mask].sort(descending=True) + valid_seg_weight[sort_indices[:batch_kept]] = 1. + + seg_weight[valid_mask] = valid_seg_weight + + return seg_weight diff --git a/segmentation/mmseg/core/utils/__init__.py b/segmentation/mmseg/core/utils/__init__.py new file mode 100644 index 0000000..f2678b3 --- /dev/null +++ b/segmentation/mmseg/core/utils/__init__.py @@ -0,0 +1,3 @@ +from .misc import add_prefix + +__all__ = ['add_prefix'] diff --git a/segmentation/mmseg/core/utils/misc.py b/segmentation/mmseg/core/utils/misc.py new file mode 100644 index 0000000..eb862a8 --- /dev/null +++ b/segmentation/mmseg/core/utils/misc.py @@ -0,0 +1,17 @@ +def add_prefix(inputs, prefix): + """Add prefix for dict. + + Args: + inputs (dict): The input dict with str keys. + prefix (str): The prefix to add. + + Returns: + + dict: The dict with keys updated with ``prefix``. + """ + + outputs = dict() + for name, value in inputs.items(): + outputs[f'{prefix}.{name}'] = value + + return outputs diff --git a/segmentation/mmseg/datasets/__init__.py b/segmentation/mmseg/datasets/__init__.py new file mode 100644 index 0000000..ebeaef4 --- /dev/null +++ b/segmentation/mmseg/datasets/__init__.py @@ -0,0 +1,19 @@ +from .ade import ADE20KDataset +from .builder import DATASETS, PIPELINES, build_dataloader, build_dataset +from .chase_db1 import ChaseDB1Dataset +from .cityscapes import CityscapesDataset +from .custom import CustomDataset +from .dataset_wrappers import ConcatDataset, RepeatDataset +from .drive import DRIVEDataset +from .hrf import HRFDataset +from .pascal_context import PascalContextDataset, PascalContextDataset59 +from .stare import STAREDataset +from .voc import PascalVOCDataset + +__all__ = [ + 'CustomDataset', 'build_dataloader', 'ConcatDataset', 'RepeatDataset', + 'DATASETS', 'build_dataset', 'PIPELINES', 'CityscapesDataset', + 'PascalVOCDataset', 'ADE20KDataset', 'PascalContextDataset', + 'PascalContextDataset59', 'ChaseDB1Dataset', 'DRIVEDataset', 'HRFDataset', + 'STAREDataset' +] diff --git a/segmentation/mmseg/datasets/ade.py b/segmentation/mmseg/datasets/ade.py new file mode 100644 index 0000000..5daf7e3 --- /dev/null +++ b/segmentation/mmseg/datasets/ade.py @@ -0,0 +1,163 @@ +import os.path as osp +import tempfile + +import mmcv +import numpy as np +from PIL import Image + +from .builder import DATASETS +from .custom import CustomDataset + + +@DATASETS.register_module() +class ADE20KDataset(CustomDataset): + """ADE20K dataset. + + In segmentation map annotation for ADE20K, 0 stands for background, which + is not included in 150 categories. ``reduce_zero_label`` is fixed to True. + The ``img_suffix`` is fixed to '.jpg' and ``seg_map_suffix`` is fixed to + '.png'. + """ + CLASSES = ( + 'wall', 'building', 'sky', 'floor', 'tree', 'ceiling', 'road', 'bed ', + 'windowpane', 'grass', 'cabinet', 'sidewalk', 'person', 'earth', + 'door', 'table', 'mountain', 'plant', 'curtain', 'chair', 'car', + 'water', 'painting', 'sofa', 'shelf', 'house', 'sea', 'mirror', 'rug', + 'field', 'armchair', 'seat', 'fence', 'desk', 'rock', 'wardrobe', + 'lamp', 'bathtub', 'railing', 'cushion', 'base', 'box', 'column', + 'signboard', 'chest of drawers', 'counter', 'sand', 'sink', + 'skyscraper', 'fireplace', 'refrigerator', 'grandstand', 'path', + 'stairs', 'runway', 'case', 'pool table', 'pillow', 'screen door', + 'stairway', 'river', 'bridge', 'bookcase', 'blind', 'coffee table', + 'toilet', 'flower', 'book', 'hill', 'bench', 'countertop', 'stove', + 'palm', 'kitchen island', 'computer', 'swivel chair', 'boat', 'bar', + 'arcade machine', 'hovel', 'bus', 'towel', 'light', 'truck', 'tower', + 'chandelier', 'awning', 'streetlight', 'booth', 'television receiver', + 'airplane', 'dirt track', 'apparel', 'pole', 'land', 'bannister', + 'escalator', 'ottoman', 'bottle', 'buffet', 'poster', 'stage', 'van', + 'ship', 'fountain', 'conveyer belt', 'canopy', 'washer', 'plaything', + 'swimming pool', 'stool', 'barrel', 'basket', 'waterfall', 'tent', + 'bag', 'minibike', 'cradle', 'oven', 'ball', 'food', 'step', 'tank', + 'trade name', 'microwave', 'pot', 'animal', 'bicycle', 'lake', + 'dishwasher', 'screen', 'blanket', 'sculpture', 'hood', 'sconce', + 'vase', 'traffic light', 'tray', 'ashcan', 'fan', 'pier', 'crt screen', + 'plate', 'monitor', 'bulletin board', 'shower', 'radiator', 'glass', + 'clock', 'flag') + + PALETTE = [[120, 120, 120], [180, 120, 120], [6, 230, 230], [80, 50, 50], + [4, 200, 3], [120, 120, 80], [140, 140, 140], [204, 5, 255], + [230, 230, 230], [4, 250, 7], [224, 5, 255], [235, 255, 7], + [150, 5, 61], [120, 120, 70], [8, 255, 51], [255, 6, 82], + [143, 255, 140], [204, 255, 4], [255, 51, 7], [204, 70, 3], + [0, 102, 200], [61, 230, 250], [255, 6, 51], [11, 102, 255], + [255, 7, 71], [255, 9, 224], [9, 7, 230], [220, 220, 220], + [255, 9, 92], [112, 9, 255], [8, 255, 214], [7, 255, 224], + [255, 184, 6], [10, 255, 71], [255, 41, 10], [7, 255, 255], + [224, 255, 8], [102, 8, 255], [255, 61, 6], [255, 194, 7], + [255, 122, 8], [0, 255, 20], [255, 8, 41], [255, 5, 153], + [6, 51, 255], [235, 12, 255], [160, 150, 20], [0, 163, 255], + [140, 140, 140], [250, 10, 15], [20, 255, 0], [31, 255, 0], + [255, 31, 0], [255, 224, 0], [153, 255, 0], [0, 0, 255], + [255, 71, 0], [0, 235, 255], [0, 173, 255], [31, 0, 255], + [11, 200, 200], [255, 82, 0], [0, 255, 245], [0, 61, 255], + [0, 255, 112], [0, 255, 133], [255, 0, 0], [255, 163, 0], + [255, 102, 0], [194, 255, 0], [0, 143, 255], [51, 255, 0], + [0, 82, 255], [0, 255, 41], [0, 255, 173], [10, 0, 255], + [173, 255, 0], [0, 255, 153], [255, 92, 0], [255, 0, 255], + [255, 0, 245], [255, 0, 102], [255, 173, 0], [255, 0, 20], + [255, 184, 184], [0, 31, 255], [0, 255, 61], [0, 71, 255], + [255, 0, 204], [0, 255, 194], [0, 255, 82], [0, 10, 255], + [0, 112, 255], [51, 0, 255], [0, 194, 255], [0, 122, 255], + [0, 255, 163], [255, 153, 0], [0, 255, 10], [255, 112, 0], + [143, 255, 0], [82, 0, 255], [163, 255, 0], [255, 235, 0], + [8, 184, 170], [133, 0, 255], [0, 255, 92], [184, 0, 255], + [255, 0, 31], [0, 184, 255], [0, 214, 255], [255, 0, 112], + [92, 255, 0], [0, 224, 255], [112, 224, 255], [70, 184, 160], + [163, 0, 255], [153, 0, 255], [71, 255, 0], [255, 0, 163], + [255, 204, 0], [255, 0, 143], [0, 255, 235], [133, 255, 0], + [255, 0, 235], [245, 0, 255], [255, 0, 122], [255, 245, 0], + [10, 190, 212], [214, 255, 0], [0, 204, 255], [20, 0, 255], + [255, 255, 0], [0, 153, 255], [0, 41, 255], [0, 255, 204], + [41, 0, 255], [41, 255, 0], [173, 0, 255], [0, 245, 255], + [71, 0, 255], [122, 0, 255], [0, 255, 184], [0, 92, 255], + [184, 255, 0], [0, 133, 255], [255, 214, 0], [25, 194, 194], + [102, 255, 0], [92, 0, 255]] + + def __init__(self, **kwargs): + super(ADE20KDataset, self).__init__( + img_suffix='.jpg', + seg_map_suffix='.png', + reduce_zero_label=True, + **kwargs) + + def results2img(self, results, imgfile_prefix, to_label_id): + """Write the segmentation results to images. + + Args: + results (list[list | tuple | ndarray]): Testing results of the + dataset. + imgfile_prefix (str): The filename prefix of the png files. + If the prefix is "somepath/xxx", + the png files will be named "somepath/xxx.png". + to_label_id (bool): whether convert output to label_id for + submission + + Returns: + list[str: str]: result txt files which contains corresponding + semantic segmentation images. + """ + mmcv.mkdir_or_exist(imgfile_prefix) + result_files = [] + prog_bar = mmcv.ProgressBar(len(self)) + for idx in range(len(self)): + result = results[idx] + + filename = self.img_infos[idx]['filename'] + basename = osp.splitext(osp.basename(filename))[0] + + png_filename = osp.join(imgfile_prefix, f'{basename}.png') + + # The index range of official requirement is from 0 to 150. + # But the index range of output is from 0 to 149. + # That is because we set reduce_zero_label=True. + result = result + 1 + + output = Image.fromarray(result.astype(np.uint8)) + output.save(png_filename) + result_files.append(png_filename) + + prog_bar.update() + + return result_files + + def format_results(self, results, imgfile_prefix=None, to_label_id=True): + """Format the results into dir (standard format for ade20k evaluation). + + Args: + results (list): Testing results of the dataset. + imgfile_prefix (str | None): The prefix of images files. It + includes the file path and the prefix of filename, e.g., + "a/b/prefix". If not specified, a temp file will be created. + Default: None. + to_label_id (bool): whether convert output to label_id for + submission. Default: False + + Returns: + tuple: (result_files, tmp_dir), result_files is a list containing + the image paths, tmp_dir is the temporal directory created + for saving json/png files when img_prefix is not specified. + """ + + assert isinstance(results, list), 'results must be a list' + assert len(results) == len(self), ( + 'The length of results is not equal to the dataset len: ' + f'{len(results)} != {len(self)}') + + if imgfile_prefix is None: + tmp_dir = tempfile.TemporaryDirectory() + imgfile_prefix = tmp_dir.name + else: + tmp_dir = None + + result_files = self.results2img(results, imgfile_prefix, to_label_id) + return result_files, tmp_dir diff --git a/segmentation/mmseg/datasets/builder.py b/segmentation/mmseg/datasets/builder.py new file mode 100644 index 0000000..f7a9926 --- /dev/null +++ b/segmentation/mmseg/datasets/builder.py @@ -0,0 +1,169 @@ +import copy +import platform +import random +from functools import partial + +import numpy as np +from mmcv.parallel import collate +from mmcv.runner import get_dist_info +from mmcv.utils import Registry, build_from_cfg +from mmcv.utils.parrots_wrapper import DataLoader, PoolDataLoader +from torch.utils.data import DistributedSampler + +if platform.system() != 'Windows': + # https://github.com/pytorch/pytorch/issues/973 + import resource + rlimit = resource.getrlimit(resource.RLIMIT_NOFILE) + hard_limit = rlimit[1] + soft_limit = min(4096, hard_limit) + resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, hard_limit)) + +DATASETS = Registry('dataset') +PIPELINES = Registry('pipeline') + + +def _concat_dataset(cfg, default_args=None): + """Build :obj:`ConcatDataset by.""" + from .dataset_wrappers import ConcatDataset + img_dir = cfg['img_dir'] + ann_dir = cfg.get('ann_dir', None) + split = cfg.get('split', None) + num_img_dir = len(img_dir) if isinstance(img_dir, (list, tuple)) else 1 + if ann_dir is not None: + num_ann_dir = len(ann_dir) if isinstance(ann_dir, (list, tuple)) else 1 + else: + num_ann_dir = 0 + if split is not None: + num_split = len(split) if isinstance(split, (list, tuple)) else 1 + else: + num_split = 0 + if num_img_dir > 1: + assert num_img_dir == num_ann_dir or num_ann_dir == 0 + assert num_img_dir == num_split or num_split == 0 + else: + assert num_split == num_ann_dir or num_ann_dir <= 1 + num_dset = max(num_split, num_img_dir) + + datasets = [] + for i in range(num_dset): + data_cfg = copy.deepcopy(cfg) + if isinstance(img_dir, (list, tuple)): + data_cfg['img_dir'] = img_dir[i] + if isinstance(ann_dir, (list, tuple)): + data_cfg['ann_dir'] = ann_dir[i] + if isinstance(split, (list, tuple)): + data_cfg['split'] = split[i] + datasets.append(build_dataset(data_cfg, default_args)) + + return ConcatDataset(datasets) + + +def build_dataset(cfg, default_args=None): + """Build datasets.""" + from .dataset_wrappers import ConcatDataset, RepeatDataset + if isinstance(cfg, (list, tuple)): + dataset = ConcatDataset([build_dataset(c, default_args) for c in cfg]) + elif cfg['type'] == 'RepeatDataset': + dataset = RepeatDataset( + build_dataset(cfg['dataset'], default_args), cfg['times']) + elif isinstance(cfg.get('img_dir'), (list, tuple)) or isinstance( + cfg.get('split', None), (list, tuple)): + dataset = _concat_dataset(cfg, default_args) + else: + dataset = build_from_cfg(cfg, DATASETS, default_args) + + return dataset + + +def build_dataloader(dataset, + samples_per_gpu, + workers_per_gpu, + num_gpus=1, + dist=True, + shuffle=True, + seed=None, + drop_last=False, + pin_memory=True, + dataloader_type='PoolDataLoader', + **kwargs): + """Build PyTorch DataLoader. + + In distributed training, each GPU/process has a dataloader. + In non-distributed training, there is only one dataloader for all GPUs. + + Args: + dataset (Dataset): A PyTorch dataset. + samples_per_gpu (int): Number of training samples on each GPU, i.e., + batch size of each GPU. + workers_per_gpu (int): How many subprocesses to use for data loading + for each GPU. + num_gpus (int): Number of GPUs. Only used in non-distributed training. + dist (bool): Distributed training/test or not. Default: True. + shuffle (bool): Whether to shuffle the data at every epoch. + Default: True. + seed (int | None): Seed to be used. Default: None. + drop_last (bool): Whether to drop the last incomplete batch in epoch. + Default: False + pin_memory (bool): Whether to use pin_memory in DataLoader. + Default: True + dataloader_type (str): Type of dataloader. Default: 'PoolDataLoader' + kwargs: any keyword argument to be used to initialize DataLoader + + Returns: + DataLoader: A PyTorch dataloader. + """ + rank, world_size = get_dist_info() + if dist: + sampler = DistributedSampler( + dataset, world_size, rank, shuffle=shuffle) + shuffle = False + batch_size = samples_per_gpu + num_workers = workers_per_gpu + else: + sampler = None + batch_size = num_gpus * samples_per_gpu + num_workers = num_gpus * workers_per_gpu + + init_fn = partial( + worker_init_fn, num_workers=num_workers, rank=rank, + seed=seed) if seed is not None else None + + assert dataloader_type in ( + 'DataLoader', + 'PoolDataLoader'), f'unsupported dataloader {dataloader_type}' + + if dataloader_type == 'PoolDataLoader': + dataloader = PoolDataLoader + elif dataloader_type == 'DataLoader': + dataloader = DataLoader + + data_loader = dataloader( + dataset, + batch_size=batch_size, + sampler=sampler, + num_workers=num_workers, + collate_fn=partial(collate, samples_per_gpu=samples_per_gpu), + pin_memory=pin_memory, + shuffle=shuffle, + worker_init_fn=init_fn, + drop_last=drop_last, + **kwargs) + + return data_loader + + +def worker_init_fn(worker_id, num_workers, rank, seed): + """Worker init func for dataloader. + + The seed of each worker equals to num_worker * rank + worker_id + user_seed + + Args: + worker_id (int): Worker id. + num_workers (int): Number of workers. + rank (int): The rank of current process. + seed (int): The random seed to use. + """ + + worker_seed = num_workers * rank + worker_id + seed + np.random.seed(worker_seed) + random.seed(worker_seed) diff --git a/segmentation/mmseg/datasets/chase_db1.py b/segmentation/mmseg/datasets/chase_db1.py new file mode 100644 index 0000000..8bc29be --- /dev/null +++ b/segmentation/mmseg/datasets/chase_db1.py @@ -0,0 +1,27 @@ +import os.path as osp + +from .builder import DATASETS +from .custom import CustomDataset + + +@DATASETS.register_module() +class ChaseDB1Dataset(CustomDataset): + """Chase_db1 dataset. + + In segmentation map annotation for Chase_db1, 0 stands for background, + which is included in 2 categories. ``reduce_zero_label`` is fixed to False. + The ``img_suffix`` is fixed to '.png' and ``seg_map_suffix`` is fixed to + '_1stHO.png'. + """ + + CLASSES = ('background', 'vessel') + + PALETTE = [[120, 120, 120], [6, 230, 230]] + + def __init__(self, **kwargs): + super(ChaseDB1Dataset, self).__init__( + img_suffix='.png', + seg_map_suffix='_1stHO.png', + reduce_zero_label=False, + **kwargs) + assert osp.exists(self.img_dir) diff --git a/segmentation/mmseg/datasets/cityscapes.py b/segmentation/mmseg/datasets/cityscapes.py new file mode 100644 index 0000000..fa9958a --- /dev/null +++ b/segmentation/mmseg/datasets/cityscapes.py @@ -0,0 +1,217 @@ +import os.path as osp +import tempfile + +import mmcv +import numpy as np +from mmcv.utils import print_log +from PIL import Image + +from .builder import DATASETS +from .custom import CustomDataset + + +@DATASETS.register_module() +class CityscapesDataset(CustomDataset): + """Cityscapes dataset. + + The ``img_suffix`` is fixed to '_leftImg8bit.png' and ``seg_map_suffix`` is + fixed to '_gtFine_labelTrainIds.png' for Cityscapes dataset. + """ + + CLASSES = ('road', 'sidewalk', 'building', 'wall', 'fence', 'pole', + 'traffic light', 'traffic sign', 'vegetation', 'terrain', 'sky', + 'person', 'rider', 'car', 'truck', 'bus', 'train', 'motorcycle', + 'bicycle') + + PALETTE = [[128, 64, 128], [244, 35, 232], [70, 70, 70], [102, 102, 156], + [190, 153, 153], [153, 153, 153], [250, 170, 30], [220, 220, 0], + [107, 142, 35], [152, 251, 152], [70, 130, 180], [220, 20, 60], + [255, 0, 0], [0, 0, 142], [0, 0, 70], [0, 60, 100], + [0, 80, 100], [0, 0, 230], [119, 11, 32]] + + def __init__(self, **kwargs): + super(CityscapesDataset, self).__init__( + img_suffix='_leftImg8bit.png', + seg_map_suffix='_gtFine_labelTrainIds.png', + **kwargs) + + @staticmethod + def _convert_to_label_id(result): + """Convert trainId to id for cityscapes.""" + if isinstance(result, str): + result = np.load(result) + import cityscapesscripts.helpers.labels as CSLabels + result_copy = result.copy() + for trainId, label in CSLabels.trainId2label.items(): + result_copy[result == trainId] = label.id + + return result_copy + + def results2img(self, results, imgfile_prefix, to_label_id): + """Write the segmentation results to images. + + Args: + results (list[list | tuple | ndarray]): Testing results of the + dataset. + imgfile_prefix (str): The filename prefix of the png files. + If the prefix is "somepath/xxx", + the png files will be named "somepath/xxx.png". + to_label_id (bool): whether convert output to label_id for + submission + + Returns: + list[str: str]: result txt files which contains corresponding + semantic segmentation images. + """ + mmcv.mkdir_or_exist(imgfile_prefix) + result_files = [] + prog_bar = mmcv.ProgressBar(len(self)) + for idx in range(len(self)): + result = results[idx] + if to_label_id: + result = self._convert_to_label_id(result) + filename = self.img_infos[idx]['filename'] + basename = osp.splitext(osp.basename(filename))[0] + + png_filename = osp.join(imgfile_prefix, f'{basename}.png') + + output = Image.fromarray(result.astype(np.uint8)).convert('P') + import cityscapesscripts.helpers.labels as CSLabels + palette = np.zeros((len(CSLabels.id2label), 3), dtype=np.uint8) + for label_id, label in CSLabels.id2label.items(): + palette[label_id] = label.color + + output.putpalette(palette) + output.save(png_filename) + result_files.append(png_filename) + prog_bar.update() + + return result_files + + def format_results(self, results, imgfile_prefix=None, to_label_id=True): + """Format the results into dir (standard format for Cityscapes + evaluation). + + Args: + results (list): Testing results of the dataset. + imgfile_prefix (str | None): The prefix of images files. It + includes the file path and the prefix of filename, e.g., + "a/b/prefix". If not specified, a temp file will be created. + Default: None. + to_label_id (bool): whether convert output to label_id for + submission. Default: False + + Returns: + tuple: (result_files, tmp_dir), result_files is a list containing + the image paths, tmp_dir is the temporal directory created + for saving json/png files when img_prefix is not specified. + """ + + assert isinstance(results, list), 'results must be a list' + assert len(results) == len(self), ( + 'The length of results is not equal to the dataset len: ' + f'{len(results)} != {len(self)}') + + if imgfile_prefix is None: + tmp_dir = tempfile.TemporaryDirectory() + imgfile_prefix = tmp_dir.name + else: + tmp_dir = None + result_files = self.results2img(results, imgfile_prefix, to_label_id) + + return result_files, tmp_dir + + def evaluate(self, + results, + metric='mIoU', + logger=None, + imgfile_prefix=None, + efficient_test=False): + """Evaluation in Cityscapes/default protocol. + + Args: + results (list): Testing results of the dataset. + metric (str | list[str]): Metrics to be evaluated. + logger (logging.Logger | None | str): Logger used for printing + related information during evaluation. Default: None. + imgfile_prefix (str | None): The prefix of output image file, + for cityscapes evaluation only. It includes the file path and + the prefix of filename, e.g., "a/b/prefix". + If results are evaluated with cityscapes protocol, it would be + the prefix of output png files. The output files would be + png images under folder "a/b/prefix/xxx.png", where "xxx" is + the image name of cityscapes. If not specified, a temp file + will be created for evaluation. + Default: None. + + Returns: + dict[str, float]: Cityscapes/default metrics. + """ + + eval_results = dict() + metrics = metric.copy() if isinstance(metric, list) else [metric] + if 'cityscapes' in metrics: + eval_results.update( + self._evaluate_cityscapes(results, logger, imgfile_prefix)) + metrics.remove('cityscapes') + if len(metrics) > 0: + eval_results.update( + super(CityscapesDataset, + self).evaluate(results, metrics, logger, efficient_test)) + + return eval_results + + def _evaluate_cityscapes(self, results, logger, imgfile_prefix): + """Evaluation in Cityscapes protocol. + + Args: + results (list): Testing results of the dataset. + logger (logging.Logger | str | None): Logger used for printing + related information during evaluation. Default: None. + imgfile_prefix (str | None): The prefix of output image file + + Returns: + dict[str: float]: Cityscapes evaluation results. + """ + try: + import cityscapesscripts.evaluation.evalPixelLevelSemanticLabeling as CSEval # noqa + except ImportError: + raise ImportError('Please run "pip install cityscapesscripts" to ' + 'install cityscapesscripts first.') + msg = 'Evaluating in Cityscapes style' + if logger is None: + msg = '\n' + msg + print_log(msg, logger=logger) + + result_files, tmp_dir = self.format_results(results, imgfile_prefix) + + if tmp_dir is None: + result_dir = imgfile_prefix + else: + result_dir = tmp_dir.name + + eval_results = dict() + print_log(f'Evaluating results under {result_dir} ...', logger=logger) + + CSEval.args.evalInstLevelScore = True + CSEval.args.predictionPath = osp.abspath(result_dir) + CSEval.args.evalPixelAccuracy = True + CSEval.args.JSONOutput = False + + seg_map_list = [] + pred_list = [] + + # when evaluating with official cityscapesscripts, + # **_gtFine_labelIds.png is used + for seg_map in mmcv.scandir( + self.ann_dir, 'gtFine_labelIds.png', recursive=True): + seg_map_list.append(osp.join(self.ann_dir, seg_map)) + pred_list.append(CSEval.getPrediction(CSEval.args, seg_map)) + + eval_results.update( + CSEval.evaluateImgLists(pred_list, seg_map_list, CSEval.args)) + + if tmp_dir is not None: + tmp_dir.cleanup() + + return eval_results diff --git a/segmentation/mmseg/datasets/custom.py b/segmentation/mmseg/datasets/custom.py new file mode 100644 index 0000000..9c88235 --- /dev/null +++ b/segmentation/mmseg/datasets/custom.py @@ -0,0 +1,400 @@ +import os +import os.path as osp +from collections import OrderedDict +from functools import reduce + +import mmcv +import numpy as np +from mmcv.utils import print_log +from prettytable import PrettyTable +from torch.utils.data import Dataset + +from mmseg.core import eval_metrics +from mmseg.utils import get_root_logger +from .builder import DATASETS +from .pipelines import Compose + + +@DATASETS.register_module() +class CustomDataset(Dataset): + """Custom dataset for semantic segmentation. An example of file structure + is as followed. + + .. code-block:: none + + ├── data + │ ├── my_dataset + │ │ ├── img_dir + │ │ │ ├── train + │ │ │ │ ├── xxx{img_suffix} + │ │ │ │ ├── yyy{img_suffix} + │ │ │ │ ├── zzz{img_suffix} + │ │ │ ├── val + │ │ ├── ann_dir + │ │ │ ├── train + │ │ │ │ ├── xxx{seg_map_suffix} + │ │ │ │ ├── yyy{seg_map_suffix} + │ │ │ │ ├── zzz{seg_map_suffix} + │ │ │ ├── val + + The img/gt_semantic_seg pair of CustomDataset should be of the same + except suffix. A valid img/gt_semantic_seg filename pair should be like + ``xxx{img_suffix}`` and ``xxx{seg_map_suffix}`` (extension is also included + in the suffix). If split is given, then ``xxx`` is specified in txt file. + Otherwise, all files in ``img_dir/``and ``ann_dir`` will be loaded. + Please refer to ``docs/tutorials/new_dataset.md`` for more details. + + + Args: + pipeline (list[dict]): Processing pipeline + img_dir (str): Path to image directory + img_suffix (str): Suffix of images. Default: '.jpg' + ann_dir (str, optional): Path to annotation directory. Default: None + seg_map_suffix (str): Suffix of segmentation maps. Default: '.png' + split (str, optional): Split txt file. If split is specified, only + file with suffix in the splits will be loaded. Otherwise, all + images in img_dir/ann_dir will be loaded. Default: None + data_root (str, optional): Data root for img_dir/ann_dir. Default: + None. + test_mode (bool): If test_mode=True, gt wouldn't be loaded. + ignore_index (int): The label index to be ignored. Default: 255 + reduce_zero_label (bool): Whether to mark label zero as ignored. + Default: False + classes (str | Sequence[str], optional): Specify classes to load. + If is None, ``cls.CLASSES`` will be used. Default: None. + palette (Sequence[Sequence[int]]] | np.ndarray | None): + The palette of segmentation map. If None is given, and + self.PALETTE is None, random palette will be generated. + Default: None + """ + + CLASSES = None + + PALETTE = None + + def __init__(self, + pipeline, + img_dir, + img_suffix='.jpg', + ann_dir=None, + seg_map_suffix='.png', + split=None, + data_root=None, + test_mode=False, + ignore_index=255, + reduce_zero_label=False, + classes=None, + palette=None): + self.pipeline = Compose(pipeline) + self.img_dir = img_dir + self.img_suffix = img_suffix + self.ann_dir = ann_dir + self.seg_map_suffix = seg_map_suffix + self.split = split + self.data_root = data_root + self.test_mode = test_mode + self.ignore_index = ignore_index + self.reduce_zero_label = reduce_zero_label + self.label_map = None + self.CLASSES, self.PALETTE = self.get_classes_and_palette( + classes, palette) + + # join paths if data_root is specified + if self.data_root is not None: + if not osp.isabs(self.img_dir): + self.img_dir = osp.join(self.data_root, self.img_dir) + if not (self.ann_dir is None or osp.isabs(self.ann_dir)): + self.ann_dir = osp.join(self.data_root, self.ann_dir) + if not (self.split is None or osp.isabs(self.split)): + self.split = osp.join(self.data_root, self.split) + + # load annotations + self.img_infos = self.load_annotations(self.img_dir, self.img_suffix, + self.ann_dir, + self.seg_map_suffix, self.split) + + def __len__(self): + """Total number of samples of data.""" + return len(self.img_infos) + + def load_annotations(self, img_dir, img_suffix, ann_dir, seg_map_suffix, + split): + """Load annotation from directory. + + Args: + img_dir (str): Path to image directory + img_suffix (str): Suffix of images. + ann_dir (str|None): Path to annotation directory. + seg_map_suffix (str|None): Suffix of segmentation maps. + split (str|None): Split txt file. If split is specified, only file + with suffix in the splits will be loaded. Otherwise, all images + in img_dir/ann_dir will be loaded. Default: None + + Returns: + list[dict]: All image info of dataset. + """ + + img_infos = [] + if split is not None: + with open(split) as f: + for line in f: + img_name = line.strip() + img_info = dict(filename=img_name + img_suffix) + if ann_dir is not None: + seg_map = img_name + seg_map_suffix + img_info['ann'] = dict(seg_map=seg_map) + img_infos.append(img_info) + else: + for img in mmcv.scandir(img_dir, img_suffix, recursive=True): + img_info = dict(filename=img) + if ann_dir is not None: + seg_map = img.replace(img_suffix, seg_map_suffix) + img_info['ann'] = dict(seg_map=seg_map) + img_infos.append(img_info) + + print_log(f'Loaded {len(img_infos)} images', logger=get_root_logger()) + return img_infos + + def get_ann_info(self, idx): + """Get annotation by index. + + Args: + idx (int): Index of data. + + Returns: + dict: Annotation info of specified index. + """ + + return self.img_infos[idx]['ann'] + + def pre_pipeline(self, results): + """Prepare results dict for pipeline.""" + results['seg_fields'] = [] + results['img_prefix'] = self.img_dir + results['seg_prefix'] = self.ann_dir + if self.custom_classes: + results['label_map'] = self.label_map + + def __getitem__(self, idx): + """Get training/test data after pipeline. + + Args: + idx (int): Index of data. + + Returns: + dict: Training/test data (with annotation if `test_mode` is set + False). + """ + + if self.test_mode: + return self.prepare_test_img(idx) + else: + return self.prepare_train_img(idx) + + def prepare_train_img(self, idx): + """Get training data and annotations after pipeline. + + Args: + idx (int): Index of data. + + Returns: + dict: Training data and annotation after pipeline with new keys + introduced by pipeline. + """ + + img_info = self.img_infos[idx] + ann_info = self.get_ann_info(idx) + results = dict(img_info=img_info, ann_info=ann_info) + self.pre_pipeline(results) + return self.pipeline(results) + + def prepare_test_img(self, idx): + """Get testing data after pipeline. + + Args: + idx (int): Index of data. + + Returns: + dict: Testing data after pipeline with new keys introduced by + pipeline. + """ + + img_info = self.img_infos[idx] + results = dict(img_info=img_info) + self.pre_pipeline(results) + return self.pipeline(results) + + def format_results(self, results, **kwargs): + """Place holder to format result to dataset specific output.""" + + def get_gt_seg_maps(self, efficient_test=False): + """Get ground truth segmentation maps for evaluation.""" + gt_seg_maps = [] + for img_info in self.img_infos: + seg_map = osp.join(self.ann_dir, img_info['ann']['seg_map']) + if efficient_test: + gt_seg_map = seg_map + else: + gt_seg_map = mmcv.imread( + seg_map, flag='unchanged', backend='pillow') + gt_seg_maps.append(gt_seg_map) + return gt_seg_maps + + def get_classes_and_palette(self, classes=None, palette=None): + """Get class names of current dataset. + + Args: + classes (Sequence[str] | str | None): If classes is None, use + default CLASSES defined by builtin dataset. If classes is a + string, take it as a file name. The file contains the name of + classes where each line contains one class name. If classes is + a tuple or list, override the CLASSES defined by the dataset. + palette (Sequence[Sequence[int]]] | np.ndarray | None): + The palette of segmentation map. If None is given, random + palette will be generated. Default: None + """ + if classes is None: + self.custom_classes = False + return self.CLASSES, self.PALETTE + + self.custom_classes = True + if isinstance(classes, str): + # take it as a file path + class_names = mmcv.list_from_file(classes) + elif isinstance(classes, (tuple, list)): + class_names = classes + else: + raise ValueError(f'Unsupported type {type(classes)} of classes.') + + if self.CLASSES: + if not set(classes).issubset(self.CLASSES): + raise ValueError('classes is not a subset of CLASSES.') + + # dictionary, its keys are the old label ids and its values + # are the new label ids. + # used for changing pixel labels in load_annotations. + self.label_map = {} + for i, c in enumerate(self.CLASSES): + if c not in class_names: + self.label_map[i] = -1 + else: + self.label_map[i] = classes.index(c) + + palette = self.get_palette_for_custom_classes(class_names, palette) + + return class_names, palette + + def get_palette_for_custom_classes(self, class_names, palette=None): + + if self.label_map is not None: + # return subset of palette + palette = [] + for old_id, new_id in sorted( + self.label_map.items(), key=lambda x: x[1]): + if new_id != -1: + palette.append(self.PALETTE[old_id]) + palette = type(self.PALETTE)(palette) + + elif palette is None: + if self.PALETTE is None: + palette = np.random.randint(0, 255, size=(len(class_names), 3)) + else: + palette = self.PALETTE + + return palette + + def evaluate(self, + results, + metric='mIoU', + logger=None, + efficient_test=False, + **kwargs): + """Evaluate the dataset. + + Args: + results (list): Testing results of the dataset. + metric (str | list[str]): Metrics to be evaluated. 'mIoU', + 'mDice' and 'mFscore' are supported. + logger (logging.Logger | None | str): Logger used for printing + related information during evaluation. Default: None. + + Returns: + dict[str, float]: Default metrics. + """ + + if isinstance(metric, str): + metric = [metric] + allowed_metrics = ['mIoU', 'mDice', 'mFscore'] + if not set(metric).issubset(set(allowed_metrics)): + raise KeyError('metric {} is not supported'.format(metric)) + eval_results = {} + gt_seg_maps = self.get_gt_seg_maps(efficient_test) + if self.CLASSES is None: + num_classes = len( + reduce(np.union1d, [np.unique(_) for _ in gt_seg_maps])) + else: + num_classes = len(self.CLASSES) + ret_metrics = eval_metrics( + results, + gt_seg_maps, + num_classes, + self.ignore_index, + metric, + label_map=self.label_map, + reduce_zero_label=self.reduce_zero_label) + + if self.CLASSES is None: + class_names = tuple(range(num_classes)) + else: + class_names = self.CLASSES + + # summary table + ret_metrics_summary = OrderedDict({ + ret_metric: np.round(np.nanmean(ret_metric_value) * 100, 2) + for ret_metric, ret_metric_value in ret_metrics.items() + }) + + # each class table + ret_metrics.pop('aAcc', None) + ret_metrics_class = OrderedDict({ + ret_metric: np.round(ret_metric_value * 100, 2) + for ret_metric, ret_metric_value in ret_metrics.items() + }) + ret_metrics_class.update({'Class': class_names}) + ret_metrics_class.move_to_end('Class', last=False) + + # for logger + class_table_data = PrettyTable() + for key, val in ret_metrics_class.items(): + class_table_data.add_column(key, val) + + summary_table_data = PrettyTable() + for key, val in ret_metrics_summary.items(): + if key == 'aAcc': + summary_table_data.add_column(key, [val]) + else: + summary_table_data.add_column('m' + key, [val]) + + print_log('per class results:', logger) + print_log('\n' + class_table_data.get_string(), logger=logger) + print_log('Summary:', logger) + print_log('\n' + summary_table_data.get_string(), logger=logger) + + # each metric dict + for key, value in ret_metrics_summary.items(): + if key == 'aAcc': + eval_results[key] = value / 100.0 + else: + eval_results['m' + key] = value / 100.0 + + ret_metrics_class.pop('Class', None) + for key, value in ret_metrics_class.items(): + eval_results.update({ + key + '.' + str(name): value[idx] / 100.0 + for idx, name in enumerate(class_names) + }) + + if mmcv.is_list_of(results, str): + for file_name in results: + os.remove(file_name) + return eval_results diff --git a/segmentation/mmseg/datasets/dataset_wrappers.py b/segmentation/mmseg/datasets/dataset_wrappers.py new file mode 100644 index 0000000..d6a5e95 --- /dev/null +++ b/segmentation/mmseg/datasets/dataset_wrappers.py @@ -0,0 +1,50 @@ +from torch.utils.data.dataset import ConcatDataset as _ConcatDataset + +from .builder import DATASETS + + +@DATASETS.register_module() +class ConcatDataset(_ConcatDataset): + """A wrapper of concatenated dataset. + + Same as :obj:`torch.utils.data.dataset.ConcatDataset`, but + concat the group flag for image aspect ratio. + + Args: + datasets (list[:obj:`Dataset`]): A list of datasets. + """ + + def __init__(self, datasets): + super(ConcatDataset, self).__init__(datasets) + self.CLASSES = datasets[0].CLASSES + self.PALETTE = datasets[0].PALETTE + + +@DATASETS.register_module() +class RepeatDataset(object): + """A wrapper of repeated dataset. + + The length of repeated dataset will be `times` larger than the original + dataset. This is useful when the data loading time is long but the dataset + is small. Using RepeatDataset can reduce the data loading time between + epochs. + + Args: + dataset (:obj:`Dataset`): The dataset to be repeated. + times (int): Repeat times. + """ + + def __init__(self, dataset, times): + self.dataset = dataset + self.times = times + self.CLASSES = dataset.CLASSES + self.PALETTE = dataset.PALETTE + self._ori_len = len(self.dataset) + + def __getitem__(self, idx): + """Get item from original dataset.""" + return self.dataset[idx % self._ori_len] + + def __len__(self): + """The length is multiplied by ``times``""" + return self.times * self._ori_len diff --git a/segmentation/mmseg/datasets/drive.py b/segmentation/mmseg/datasets/drive.py new file mode 100644 index 0000000..3cbfda8 --- /dev/null +++ b/segmentation/mmseg/datasets/drive.py @@ -0,0 +1,27 @@ +import os.path as osp + +from .builder import DATASETS +from .custom import CustomDataset + + +@DATASETS.register_module() +class DRIVEDataset(CustomDataset): + """DRIVE dataset. + + In segmentation map annotation for DRIVE, 0 stands for background, which is + included in 2 categories. ``reduce_zero_label`` is fixed to False. The + ``img_suffix`` is fixed to '.png' and ``seg_map_suffix`` is fixed to + '_manual1.png'. + """ + + CLASSES = ('background', 'vessel') + + PALETTE = [[120, 120, 120], [6, 230, 230]] + + def __init__(self, **kwargs): + super(DRIVEDataset, self).__init__( + img_suffix='.png', + seg_map_suffix='_manual1.png', + reduce_zero_label=False, + **kwargs) + assert osp.exists(self.img_dir) diff --git a/segmentation/mmseg/datasets/hrf.py b/segmentation/mmseg/datasets/hrf.py new file mode 100644 index 0000000..923203b --- /dev/null +++ b/segmentation/mmseg/datasets/hrf.py @@ -0,0 +1,27 @@ +import os.path as osp + +from .builder import DATASETS +from .custom import CustomDataset + + +@DATASETS.register_module() +class HRFDataset(CustomDataset): + """HRF dataset. + + In segmentation map annotation for HRF, 0 stands for background, which is + included in 2 categories. ``reduce_zero_label`` is fixed to False. The + ``img_suffix`` is fixed to '.png' and ``seg_map_suffix`` is fixed to + '.png'. + """ + + CLASSES = ('background', 'vessel') + + PALETTE = [[120, 120, 120], [6, 230, 230]] + + def __init__(self, **kwargs): + super(HRFDataset, self).__init__( + img_suffix='.png', + seg_map_suffix='.png', + reduce_zero_label=False, + **kwargs) + assert osp.exists(self.img_dir) diff --git a/segmentation/mmseg/datasets/pascal_context.py b/segmentation/mmseg/datasets/pascal_context.py new file mode 100644 index 0000000..541a63c --- /dev/null +++ b/segmentation/mmseg/datasets/pascal_context.py @@ -0,0 +1,103 @@ +import os.path as osp + +from .builder import DATASETS +from .custom import CustomDataset + + +@DATASETS.register_module() +class PascalContextDataset(CustomDataset): + """PascalContext dataset. + + In segmentation map annotation for PascalContext, 0 stands for background, + which is included in 60 categories. ``reduce_zero_label`` is fixed to + False. The ``img_suffix`` is fixed to '.jpg' and ``seg_map_suffix`` is + fixed to '.png'. + + Args: + split (str): Split txt file for PascalContext. + """ + + CLASSES = ('background', 'aeroplane', 'bag', 'bed', 'bedclothes', 'bench', + 'bicycle', 'bird', 'boat', 'book', 'bottle', 'building', 'bus', + 'cabinet', 'car', 'cat', 'ceiling', 'chair', 'cloth', + 'computer', 'cow', 'cup', 'curtain', 'dog', 'door', 'fence', + 'floor', 'flower', 'food', 'grass', 'ground', 'horse', + 'keyboard', 'light', 'motorbike', 'mountain', 'mouse', 'person', + 'plate', 'platform', 'pottedplant', 'road', 'rock', 'sheep', + 'shelves', 'sidewalk', 'sign', 'sky', 'snow', 'sofa', 'table', + 'track', 'train', 'tree', 'truck', 'tvmonitor', 'wall', 'water', + 'window', 'wood') + + PALETTE = [[120, 120, 120], [180, 120, 120], [6, 230, 230], [80, 50, 50], + [4, 200, 3], [120, 120, 80], [140, 140, 140], [204, 5, 255], + [230, 230, 230], [4, 250, 7], [224, 5, 255], [235, 255, 7], + [150, 5, 61], [120, 120, 70], [8, 255, 51], [255, 6, 82], + [143, 255, 140], [204, 255, 4], [255, 51, 7], [204, 70, 3], + [0, 102, 200], [61, 230, 250], [255, 6, 51], [11, 102, 255], + [255, 7, 71], [255, 9, 224], [9, 7, 230], [220, 220, 220], + [255, 9, 92], [112, 9, 255], [8, 255, 214], [7, 255, 224], + [255, 184, 6], [10, 255, 71], [255, 41, 10], [7, 255, 255], + [224, 255, 8], [102, 8, 255], [255, 61, 6], [255, 194, 7], + [255, 122, 8], [0, 255, 20], [255, 8, 41], [255, 5, 153], + [6, 51, 255], [235, 12, 255], [160, 150, 20], [0, 163, 255], + [140, 140, 140], [250, 10, 15], [20, 255, 0], [31, 255, 0], + [255, 31, 0], [255, 224, 0], [153, 255, 0], [0, 0, 255], + [255, 71, 0], [0, 235, 255], [0, 173, 255], [31, 0, 255]] + + def __init__(self, split, **kwargs): + super(PascalContextDataset, self).__init__( + img_suffix='.jpg', + seg_map_suffix='.png', + split=split, + reduce_zero_label=False, + **kwargs) + assert osp.exists(self.img_dir) and self.split is not None + + +@DATASETS.register_module() +class PascalContextDataset59(CustomDataset): + """PascalContext dataset. + + In segmentation map annotation for PascalContext, 0 stands for background, + which is included in 60 categories. ``reduce_zero_label`` is fixed to + False. The ``img_suffix`` is fixed to '.jpg' and ``seg_map_suffix`` is + fixed to '.png'. + + Args: + split (str): Split txt file for PascalContext. + """ + + CLASSES = ('aeroplane', 'bag', 'bed', 'bedclothes', 'bench', 'bicycle', + 'bird', 'boat', 'book', 'bottle', 'building', 'bus', 'cabinet', + 'car', 'cat', 'ceiling', 'chair', 'cloth', 'computer', 'cow', + 'cup', 'curtain', 'dog', 'door', 'fence', 'floor', 'flower', + 'food', 'grass', 'ground', 'horse', 'keyboard', 'light', + 'motorbike', 'mountain', 'mouse', 'person', 'plate', 'platform', + 'pottedplant', 'road', 'rock', 'sheep', 'shelves', 'sidewalk', + 'sign', 'sky', 'snow', 'sofa', 'table', 'track', 'train', + 'tree', 'truck', 'tvmonitor', 'wall', 'water', 'window', 'wood') + + PALETTE = [[180, 120, 120], [6, 230, 230], [80, 50, 50], [4, 200, 3], + [120, 120, 80], [140, 140, 140], [204, 5, 255], [230, 230, 230], + [4, 250, 7], [224, 5, 255], [235, 255, 7], [150, 5, 61], + [120, 120, 70], [8, 255, 51], [255, 6, 82], [143, 255, 140], + [204, 255, 4], [255, 51, 7], [204, 70, 3], [0, 102, 200], + [61, 230, 250], [255, 6, 51], [11, 102, 255], [255, 7, 71], + [255, 9, 224], [9, 7, 230], [220, 220, 220], [255, 9, 92], + [112, 9, 255], [8, 255, 214], [7, 255, 224], [255, 184, 6], + [10, 255, 71], [255, 41, 10], [7, 255, 255], [224, 255, 8], + [102, 8, 255], [255, 61, 6], [255, 194, 7], [255, 122, 8], + [0, 255, 20], [255, 8, 41], [255, 5, 153], [6, 51, 255], + [235, 12, 255], [160, 150, 20], [0, 163, 255], [140, 140, 140], + [250, 10, 15], [20, 255, 0], [31, 255, 0], [255, 31, 0], + [255, 224, 0], [153, 255, 0], [0, 0, 255], [255, 71, 0], + [0, 235, 255], [0, 173, 255], [31, 0, 255]] + + def __init__(self, split, **kwargs): + super(PascalContextDataset59, self).__init__( + img_suffix='.jpg', + seg_map_suffix='.png', + split=split, + reduce_zero_label=True, + **kwargs) + assert osp.exists(self.img_dir) and self.split is not None diff --git a/segmentation/mmseg/datasets/pipelines/__init__.py b/segmentation/mmseg/datasets/pipelines/__init__.py new file mode 100644 index 0000000..8b9046b --- /dev/null +++ b/segmentation/mmseg/datasets/pipelines/__init__.py @@ -0,0 +1,16 @@ +from .compose import Compose +from .formating import (Collect, ImageToTensor, ToDataContainer, ToTensor, + Transpose, to_tensor) +from .loading import LoadAnnotations, LoadImageFromFile +from .test_time_aug import MultiScaleFlipAug +from .transforms import (CLAHE, AdjustGamma, Normalize, Pad, + PhotoMetricDistortion, RandomCrop, RandomFlip, + RandomRotate, Rerange, Resize, RGB2Gray, SegRescale) + +__all__ = [ + 'Compose', 'to_tensor', 'ToTensor', 'ImageToTensor', 'ToDataContainer', + 'Transpose', 'Collect', 'LoadAnnotations', 'LoadImageFromFile', + 'MultiScaleFlipAug', 'Resize', 'RandomFlip', 'Pad', 'RandomCrop', + 'Normalize', 'SegRescale', 'PhotoMetricDistortion', 'RandomRotate', + 'AdjustGamma', 'CLAHE', 'Rerange', 'RGB2Gray' +] diff --git a/segmentation/mmseg/datasets/pipelines/compose.py b/segmentation/mmseg/datasets/pipelines/compose.py new file mode 100644 index 0000000..ca48f1c --- /dev/null +++ b/segmentation/mmseg/datasets/pipelines/compose.py @@ -0,0 +1,51 @@ +import collections + +from mmcv.utils import build_from_cfg + +from ..builder import PIPELINES + + +@PIPELINES.register_module() +class Compose(object): + """Compose multiple transforms sequentially. + + Args: + transforms (Sequence[dict | callable]): Sequence of transform object or + config dict to be composed. + """ + + def __init__(self, transforms): + assert isinstance(transforms, collections.abc.Sequence) + self.transforms = [] + for transform in transforms: + if isinstance(transform, dict): + transform = build_from_cfg(transform, PIPELINES) + self.transforms.append(transform) + elif callable(transform): + self.transforms.append(transform) + else: + raise TypeError('transform must be callable or a dict') + + def __call__(self, data): + """Call function to apply transforms sequentially. + + Args: + data (dict): A result dict contains the data to transform. + + Returns: + dict: Transformed data. + """ + + for t in self.transforms: + data = t(data) + if data is None: + return None + return data + + def __repr__(self): + format_string = self.__class__.__name__ + '(' + for t in self.transforms: + format_string += '\n' + format_string += f' {t}' + format_string += '\n)' + return format_string diff --git a/segmentation/mmseg/datasets/pipelines/formating.py b/segmentation/mmseg/datasets/pipelines/formating.py new file mode 100644 index 0000000..34061c1 --- /dev/null +++ b/segmentation/mmseg/datasets/pipelines/formating.py @@ -0,0 +1,288 @@ +from collections.abc import Sequence + +import mmcv +import numpy as np +import torch +from mmcv.parallel import DataContainer as DC + +from ..builder import PIPELINES + + +def to_tensor(data): + """Convert objects of various python types to :obj:`torch.Tensor`. + + Supported types are: :class:`numpy.ndarray`, :class:`torch.Tensor`, + :class:`Sequence`, :class:`int` and :class:`float`. + + Args: + data (torch.Tensor | numpy.ndarray | Sequence | int | float): Data to + be converted. + """ + + if isinstance(data, torch.Tensor): + return data + elif isinstance(data, np.ndarray): + return torch.from_numpy(data) + elif isinstance(data, Sequence) and not mmcv.is_str(data): + return torch.tensor(data) + elif isinstance(data, int): + return torch.LongTensor([data]) + elif isinstance(data, float): + return torch.FloatTensor([data]) + else: + raise TypeError(f'type {type(data)} cannot be converted to tensor.') + + +@PIPELINES.register_module() +class ToTensor(object): + """Convert some results to :obj:`torch.Tensor` by given keys. + + Args: + keys (Sequence[str]): Keys that need to be converted to Tensor. + """ + + def __init__(self, keys): + self.keys = keys + + def __call__(self, results): + """Call function to convert data in results to :obj:`torch.Tensor`. + + Args: + results (dict): Result dict contains the data to convert. + + Returns: + dict: The result dict contains the data converted + to :obj:`torch.Tensor`. + """ + + for key in self.keys: + results[key] = to_tensor(results[key]) + return results + + def __repr__(self): + return self.__class__.__name__ + f'(keys={self.keys})' + + +@PIPELINES.register_module() +class ImageToTensor(object): + """Convert image to :obj:`torch.Tensor` by given keys. + + The dimension order of input image is (H, W, C). The pipeline will convert + it to (C, H, W). If only 2 dimension (H, W) is given, the output would be + (1, H, W). + + Args: + keys (Sequence[str]): Key of images to be converted to Tensor. + """ + + def __init__(self, keys): + self.keys = keys + + def __call__(self, results): + """Call function to convert image in results to :obj:`torch.Tensor` and + transpose the channel order. + + Args: + results (dict): Result dict contains the image data to convert. + + Returns: + dict: The result dict contains the image converted + to :obj:`torch.Tensor` and transposed to (C, H, W) order. + """ + + for key in self.keys: + img = results[key] + if len(img.shape) < 3: + img = np.expand_dims(img, -1) + results[key] = to_tensor(img.transpose(2, 0, 1)) + return results + + def __repr__(self): + return self.__class__.__name__ + f'(keys={self.keys})' + + +@PIPELINES.register_module() +class Transpose(object): + """Transpose some results by given keys. + + Args: + keys (Sequence[str]): Keys of results to be transposed. + order (Sequence[int]): Order of transpose. + """ + + def __init__(self, keys, order): + self.keys = keys + self.order = order + + def __call__(self, results): + """Call function to convert image in results to :obj:`torch.Tensor` and + transpose the channel order. + + Args: + results (dict): Result dict contains the image data to convert. + + Returns: + dict: The result dict contains the image converted + to :obj:`torch.Tensor` and transposed to (C, H, W) order. + """ + + for key in self.keys: + results[key] = results[key].transpose(self.order) + return results + + def __repr__(self): + return self.__class__.__name__ + \ + f'(keys={self.keys}, order={self.order})' + + +@PIPELINES.register_module() +class ToDataContainer(object): + """Convert results to :obj:`mmcv.DataContainer` by given fields. + + Args: + fields (Sequence[dict]): Each field is a dict like + ``dict(key='xxx', **kwargs)``. The ``key`` in result will + be converted to :obj:`mmcv.DataContainer` with ``**kwargs``. + Default: ``(dict(key='img', stack=True), + dict(key='gt_semantic_seg'))``. + """ + + def __init__(self, + fields=(dict(key='img', + stack=True), dict(key='gt_semantic_seg'))): + self.fields = fields + + def __call__(self, results): + """Call function to convert data in results to + :obj:`mmcv.DataContainer`. + + Args: + results (dict): Result dict contains the data to convert. + + Returns: + dict: The result dict contains the data converted to + :obj:`mmcv.DataContainer`. + """ + + for field in self.fields: + field = field.copy() + key = field.pop('key') + results[key] = DC(results[key], **field) + return results + + def __repr__(self): + return self.__class__.__name__ + f'(fields={self.fields})' + + +@PIPELINES.register_module() +class DefaultFormatBundle(object): + """Default formatting bundle. + + It simplifies the pipeline of formatting common fields, including "img" + and "gt_semantic_seg". These fields are formatted as follows. + + - img: (1)transpose, (2)to tensor, (3)to DataContainer (stack=True) + - gt_semantic_seg: (1)unsqueeze dim-0 (2)to tensor, + (3)to DataContainer (stack=True) + """ + + def __call__(self, results): + """Call function to transform and format common fields in results. + + Args: + results (dict): Result dict contains the data to convert. + + Returns: + dict: The result dict contains the data that is formatted with + default bundle. + """ + + if 'img' in results: + img = results['img'] + if len(img.shape) < 3: + img = np.expand_dims(img, -1) + img = np.ascontiguousarray(img.transpose(2, 0, 1)) + results['img'] = DC(to_tensor(img), stack=True) + if 'gt_semantic_seg' in results: + # convert to long + results['gt_semantic_seg'] = DC( + to_tensor(results['gt_semantic_seg'][None, + ...].astype(np.int64)), + stack=True) + return results + + def __repr__(self): + return self.__class__.__name__ + + +@PIPELINES.register_module() +class Collect(object): + """Collect data from the loader relevant to the specific task. + + This is usually the last stage of the data loader pipeline. Typically keys + is set to some subset of "img", "gt_semantic_seg". + + The "img_meta" item is always populated. The contents of the "img_meta" + dictionary depends on "meta_keys". By default this includes: + + - "img_shape": shape of the image input to the network as a tuple + (h, w, c). Note that images may be zero padded on the bottom/right + if the batch tensor is larger than this shape. + + - "scale_factor": a float indicating the preprocessing scale + + - "flip": a boolean indicating if image flip transform was used + + - "filename": path to the image file + + - "ori_shape": original shape of the image as a tuple (h, w, c) + + - "pad_shape": image shape after padding + + - "img_norm_cfg": a dict of normalization information: + - mean - per channel mean subtraction + - std - per channel std divisor + - to_rgb - bool indicating if bgr was converted to rgb + + Args: + keys (Sequence[str]): Keys of results to be collected in ``data``. + meta_keys (Sequence[str], optional): Meta keys to be converted to + ``mmcv.DataContainer`` and collected in ``data[img_metas]``. + Default: ``('filename', 'ori_filename', 'ori_shape', 'img_shape', + 'pad_shape', 'scale_factor', 'flip', 'flip_direction', + 'img_norm_cfg')`` + """ + + def __init__(self, + keys, + meta_keys=('filename', 'ori_filename', 'ori_shape', + 'img_shape', 'pad_shape', 'scale_factor', 'flip', + 'flip_direction', 'img_norm_cfg')): + self.keys = keys + self.meta_keys = meta_keys + + def __call__(self, results): + """Call function to collect keys in results. The keys in ``meta_keys`` + will be converted to :obj:mmcv.DataContainer. + + Args: + results (dict): Result dict contains the data to collect. + + Returns: + dict: The result dict contains the following keys + - keys in``self.keys`` + - ``img_metas`` + """ + + data = {} + img_meta = {} + for key in self.meta_keys: + img_meta[key] = results[key] + data['img_metas'] = DC(img_meta, cpu_only=True) + for key in self.keys: + data[key] = results[key] + return data + + def __repr__(self): + return self.__class__.__name__ + \ + f'(keys={self.keys}, meta_keys={self.meta_keys})' diff --git a/segmentation/mmseg/datasets/pipelines/loading.py b/segmentation/mmseg/datasets/pipelines/loading.py new file mode 100644 index 0000000..fdfc496 --- /dev/null +++ b/segmentation/mmseg/datasets/pipelines/loading.py @@ -0,0 +1,153 @@ +import os.path as osp + +import mmcv +import numpy as np + +from ..builder import PIPELINES + + +@PIPELINES.register_module() +class LoadImageFromFile(object): + """Load an image from file. + + Required keys are "img_prefix" and "img_info" (a dict that must contain the + key "filename"). Added or updated keys are "filename", "img", "img_shape", + "ori_shape" (same as `img_shape`), "pad_shape" (same as `img_shape`), + "scale_factor" (1.0) and "img_norm_cfg" (means=0 and stds=1). + + Args: + to_float32 (bool): Whether to convert the loaded image to a float32 + numpy array. If set to False, the loaded image is an uint8 array. + Defaults to False. + color_type (str): The flag argument for :func:`mmcv.imfrombytes`. + Defaults to 'color'. + file_client_args (dict): Arguments to instantiate a FileClient. + See :class:`mmcv.fileio.FileClient` for details. + Defaults to ``dict(backend='disk')``. + imdecode_backend (str): Backend for :func:`mmcv.imdecode`. Default: + 'cv2' + """ + + def __init__(self, + to_float32=False, + color_type='color', + file_client_args=dict(backend='disk'), + imdecode_backend='cv2'): + self.to_float32 = to_float32 + self.color_type = color_type + self.file_client_args = file_client_args.copy() + self.file_client = None + self.imdecode_backend = imdecode_backend + + def __call__(self, results): + """Call functions to load image and get image meta information. + + Args: + results (dict): Result dict from :obj:`mmseg.CustomDataset`. + + Returns: + dict: The dict contains loaded image and meta information. + """ + + if self.file_client is None: + self.file_client = mmcv.FileClient(**self.file_client_args) + + if results.get('img_prefix') is not None: + filename = osp.join(results['img_prefix'], + results['img_info']['filename']) + else: + filename = results['img_info']['filename'] + img_bytes = self.file_client.get(filename) + img = mmcv.imfrombytes( + img_bytes, flag=self.color_type, backend=self.imdecode_backend) + if self.to_float32: + img = img.astype(np.float32) + + results['filename'] = filename + results['ori_filename'] = results['img_info']['filename'] + results['img'] = img + results['img_shape'] = img.shape + results['ori_shape'] = img.shape + # Set initial values for default meta_keys + results['pad_shape'] = img.shape + results['scale_factor'] = 1.0 + num_channels = 1 if len(img.shape) < 3 else img.shape[2] + results['img_norm_cfg'] = dict( + mean=np.zeros(num_channels, dtype=np.float32), + std=np.ones(num_channels, dtype=np.float32), + to_rgb=False) + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(to_float32={self.to_float32},' + repr_str += f"color_type='{self.color_type}'," + repr_str += f"imdecode_backend='{self.imdecode_backend}')" + return repr_str + + +@PIPELINES.register_module() +class LoadAnnotations(object): + """Load annotations for semantic segmentation. + + Args: + reduce_zero_label (bool): Whether reduce all label value by 1. + Usually used for datasets where 0 is background label. + Default: False. + file_client_args (dict): Arguments to instantiate a FileClient. + See :class:`mmcv.fileio.FileClient` for details. + Defaults to ``dict(backend='disk')``. + imdecode_backend (str): Backend for :func:`mmcv.imdecode`. Default: + 'pillow' + """ + + def __init__(self, + reduce_zero_label=False, + file_client_args=dict(backend='disk'), + imdecode_backend='pillow'): + self.reduce_zero_label = reduce_zero_label + self.file_client_args = file_client_args.copy() + self.file_client = None + self.imdecode_backend = imdecode_backend + + def __call__(self, results): + """Call function to load multiple types annotations. + + Args: + results (dict): Result dict from :obj:`mmseg.CustomDataset`. + + Returns: + dict: The dict contains loaded semantic segmentation annotations. + """ + + if self.file_client is None: + self.file_client = mmcv.FileClient(**self.file_client_args) + + if results.get('seg_prefix', None) is not None: + filename = osp.join(results['seg_prefix'], + results['ann_info']['seg_map']) + else: + filename = results['ann_info']['seg_map'] + img_bytes = self.file_client.get(filename) + gt_semantic_seg = mmcv.imfrombytes( + img_bytes, flag='unchanged', + backend=self.imdecode_backend).squeeze().astype(np.uint8) + # modify if custom classes + if results.get('label_map', None) is not None: + for old_id, new_id in results['label_map'].items(): + gt_semantic_seg[gt_semantic_seg == old_id] = new_id + # reduce zero_label + if self.reduce_zero_label: + # avoid using underflow conversion + gt_semantic_seg[gt_semantic_seg == 0] = 255 + gt_semantic_seg = gt_semantic_seg - 1 + gt_semantic_seg[gt_semantic_seg == 254] = 255 + results['gt_semantic_seg'] = gt_semantic_seg + results['seg_fields'].append('gt_semantic_seg') + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(reduce_zero_label={self.reduce_zero_label},' + repr_str += f"imdecode_backend='{self.imdecode_backend}')" + return repr_str diff --git a/segmentation/mmseg/datasets/pipelines/test_time_aug.py b/segmentation/mmseg/datasets/pipelines/test_time_aug.py new file mode 100644 index 0000000..473a12b --- /dev/null +++ b/segmentation/mmseg/datasets/pipelines/test_time_aug.py @@ -0,0 +1,133 @@ +import warnings + +import mmcv + +from ..builder import PIPELINES +from .compose import Compose + + +@PIPELINES.register_module() +class MultiScaleFlipAug(object): + """Test-time augmentation with multiple scales and flipping. + + An example configuration is as followed: + + .. code-block:: + + img_scale=(2048, 1024), + img_ratios=[0.5, 1.0], + flip=True, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size_divisor=32), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ] + + After MultiScaleFLipAug with above configuration, the results are wrapped + into lists of the same length as followed: + + .. code-block:: + + dict( + img=[...], + img_shape=[...], + scale=[(1024, 512), (1024, 512), (2048, 1024), (2048, 1024)] + flip=[False, True, False, True] + ... + ) + + Args: + transforms (list[dict]): Transforms to apply in each augmentation. + img_scale (None | tuple | list[tuple]): Images scales for resizing. + img_ratios (float | list[float]): Image ratios for resizing + flip (bool): Whether apply flip augmentation. Default: False. + flip_direction (str | list[str]): Flip augmentation directions, + options are "horizontal" and "vertical". If flip_direction is list, + multiple flip augmentations will be applied. + It has no effect when flip == False. Default: "horizontal". + """ + + def __init__(self, + transforms, + img_scale, + img_ratios=None, + flip=False, + flip_direction='horizontal'): + self.transforms = Compose(transforms) + if img_ratios is not None: + img_ratios = img_ratios if isinstance(img_ratios, + list) else [img_ratios] + assert mmcv.is_list_of(img_ratios, float) + if img_scale is None: + # mode 1: given img_scale=None and a range of image ratio + self.img_scale = None + assert mmcv.is_list_of(img_ratios, float) + elif isinstance(img_scale, tuple) and mmcv.is_list_of( + img_ratios, float): + assert len(img_scale) == 2 + # mode 2: given a scale and a range of image ratio + self.img_scale = [(int(img_scale[0] * ratio), + int(img_scale[1] * ratio)) + for ratio in img_ratios] + else: + # mode 3: given multiple scales + self.img_scale = img_scale if isinstance(img_scale, + list) else [img_scale] + assert mmcv.is_list_of(self.img_scale, tuple) or self.img_scale is None + self.flip = flip + self.img_ratios = img_ratios + self.flip_direction = flip_direction if isinstance( + flip_direction, list) else [flip_direction] + assert mmcv.is_list_of(self.flip_direction, str) + if not self.flip and self.flip_direction != ['horizontal']: + warnings.warn( + 'flip_direction has no effect when flip is set to False') + if (self.flip + and not any([t['type'] == 'RandomFlip' for t in transforms])): + warnings.warn( + 'flip has no effect when RandomFlip is not in transforms') + + def __call__(self, results): + """Call function to apply test time augment transforms on results. + + Args: + results (dict): Result dict contains the data to transform. + + Returns: + dict[str: list]: The augmented data, where each value is wrapped + into a list. + """ + + aug_data = [] + if self.img_scale is None and mmcv.is_list_of(self.img_ratios, float): + h, w = results['img'].shape[:2] + img_scale = [(int(w * ratio), int(h * ratio)) + for ratio in self.img_ratios] + else: + img_scale = self.img_scale + flip_aug = [False, True] if self.flip else [False] + for scale in img_scale: + for flip in flip_aug: + for direction in self.flip_direction: + _results = results.copy() + _results['scale'] = scale + _results['flip'] = flip + _results['flip_direction'] = direction + data = self.transforms(_results) + aug_data.append(data) + # list of dict to dict of list + aug_data_dict = {key: [] for key in aug_data[0]} + for data in aug_data: + for key, val in data.items(): + aug_data_dict[key].append(val) + return aug_data_dict + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(transforms={self.transforms}, ' + repr_str += f'img_scale={self.img_scale}, flip={self.flip})' + repr_str += f'flip_direction={self.flip_direction}' + return repr_str diff --git a/segmentation/mmseg/datasets/pipelines/transforms.py b/segmentation/mmseg/datasets/pipelines/transforms.py new file mode 100644 index 0000000..20753bb --- /dev/null +++ b/segmentation/mmseg/datasets/pipelines/transforms.py @@ -0,0 +1,889 @@ +import mmcv +import numpy as np +from mmcv.utils import deprecated_api_warning, is_tuple_of +from numpy import random + +from ..builder import PIPELINES + + +@PIPELINES.register_module() +class Resize(object): + """Resize images & seg. + + This transform resizes the input image to some scale. If the input dict + contains the key "scale", then the scale in the input dict is used, + otherwise the specified scale in the init method is used. + + ``img_scale`` can be None, a tuple (single-scale) or a list of tuple + (multi-scale). There are 4 multiscale modes: + + - ``ratio_range is not None``: + 1. When img_scale is None, img_scale is the shape of image in results + (img_scale = results['img'].shape[:2]) and the image is resized based + on the original size. (mode 1) + 2. When img_scale is a tuple (single-scale), randomly sample a ratio from + the ratio range and multiply it with the image scale. (mode 2) + + - ``ratio_range is None and multiscale_mode == "range"``: randomly sample a + scale from the a range. (mode 3) + + - ``ratio_range is None and multiscale_mode == "value"``: randomly sample a + scale from multiple scales. (mode 4) + + Args: + img_scale (tuple or list[tuple]): Images scales for resizing. + multiscale_mode (str): Either "range" or "value". + ratio_range (tuple[float]): (min_ratio, max_ratio) + keep_ratio (bool): Whether to keep the aspect ratio when resizing the + image. + """ + + def __init__(self, + img_scale=None, + multiscale_mode='range', + ratio_range=None, + keep_ratio=True): + if img_scale is None: + self.img_scale = None + else: + if isinstance(img_scale, list): + self.img_scale = img_scale + else: + self.img_scale = [img_scale] + assert mmcv.is_list_of(self.img_scale, tuple) + + if ratio_range is not None: + # mode 1: given img_scale=None and a range of image ratio + # mode 2: given a scale and a range of image ratio + assert self.img_scale is None or len(self.img_scale) == 1 + else: + # mode 3 and 4: given multiple scales or a range of scales + assert multiscale_mode in ['value', 'range'] + + self.multiscale_mode = multiscale_mode + self.ratio_range = ratio_range + self.keep_ratio = keep_ratio + + @staticmethod + def random_select(img_scales): + """Randomly select an img_scale from given candidates. + + Args: + img_scales (list[tuple]): Images scales for selection. + + Returns: + (tuple, int): Returns a tuple ``(img_scale, scale_dix)``, + where ``img_scale`` is the selected image scale and + ``scale_idx`` is the selected index in the given candidates. + """ + + assert mmcv.is_list_of(img_scales, tuple) + scale_idx = np.random.randint(len(img_scales)) + img_scale = img_scales[scale_idx] + return img_scale, scale_idx + + @staticmethod + def random_sample(img_scales): + """Randomly sample an img_scale when ``multiscale_mode=='range'``. + + Args: + img_scales (list[tuple]): Images scale range for sampling. + There must be two tuples in img_scales, which specify the lower + and upper bound of image scales. + + Returns: + (tuple, None): Returns a tuple ``(img_scale, None)``, where + ``img_scale`` is sampled scale and None is just a placeholder + to be consistent with :func:`random_select`. + """ + + assert mmcv.is_list_of(img_scales, tuple) and len(img_scales) == 2 + img_scale_long = [max(s) for s in img_scales] + img_scale_short = [min(s) for s in img_scales] + long_edge = np.random.randint( + min(img_scale_long), + max(img_scale_long) + 1) + short_edge = np.random.randint( + min(img_scale_short), + max(img_scale_short) + 1) + img_scale = (long_edge, short_edge) + return img_scale, None + + @staticmethod + def random_sample_ratio(img_scale, ratio_range): + """Randomly sample an img_scale when ``ratio_range`` is specified. + + A ratio will be randomly sampled from the range specified by + ``ratio_range``. Then it would be multiplied with ``img_scale`` to + generate sampled scale. + + Args: + img_scale (tuple): Images scale base to multiply with ratio. + ratio_range (tuple[float]): The minimum and maximum ratio to scale + the ``img_scale``. + + Returns: + (tuple, None): Returns a tuple ``(scale, None)``, where + ``scale`` is sampled ratio multiplied with ``img_scale`` and + None is just a placeholder to be consistent with + :func:`random_select`. + """ + + assert isinstance(img_scale, tuple) and len(img_scale) == 2 + min_ratio, max_ratio = ratio_range + assert min_ratio <= max_ratio + ratio = np.random.random_sample() * (max_ratio - min_ratio) + min_ratio + scale = int(img_scale[0] * ratio), int(img_scale[1] * ratio) + return scale, None + + def _random_scale(self, results): + """Randomly sample an img_scale according to ``ratio_range`` and + ``multiscale_mode``. + + If ``ratio_range`` is specified, a ratio will be sampled and be + multiplied with ``img_scale``. + If multiple scales are specified by ``img_scale``, a scale will be + sampled according to ``multiscale_mode``. + Otherwise, single scale will be used. + + Args: + results (dict): Result dict from :obj:`dataset`. + + Returns: + dict: Two new keys 'scale` and 'scale_idx` are added into + ``results``, which would be used by subsequent pipelines. + """ + + if self.ratio_range is not None: + if self.img_scale is None: + h, w = results['img'].shape[:2] + scale, scale_idx = self.random_sample_ratio((w, h), + self.ratio_range) + else: + scale, scale_idx = self.random_sample_ratio( + self.img_scale[0], self.ratio_range) + elif len(self.img_scale) == 1: + scale, scale_idx = self.img_scale[0], 0 + elif self.multiscale_mode == 'range': + scale, scale_idx = self.random_sample(self.img_scale) + elif self.multiscale_mode == 'value': + scale, scale_idx = self.random_select(self.img_scale) + else: + raise NotImplementedError + + results['scale'] = scale + results['scale_idx'] = scale_idx + + def _resize_img(self, results): + """Resize images with ``results['scale']``.""" + if self.keep_ratio: + img, scale_factor = mmcv.imrescale( + results['img'], results['scale'], return_scale=True) + # the w_scale and h_scale has minor difference + # a real fix should be done in the mmcv.imrescale in the future + new_h, new_w = img.shape[:2] + h, w = results['img'].shape[:2] + w_scale = new_w / w + h_scale = new_h / h + else: + img, w_scale, h_scale = mmcv.imresize( + results['img'], results['scale'], return_scale=True) + scale_factor = np.array([w_scale, h_scale, w_scale, h_scale], + dtype=np.float32) + results['img'] = img + results['img_shape'] = img.shape + results['pad_shape'] = img.shape # in case that there is no padding + results['scale_factor'] = scale_factor + results['keep_ratio'] = self.keep_ratio + + def _resize_seg(self, results): + """Resize semantic segmentation map with ``results['scale']``.""" + for key in results.get('seg_fields', []): + if self.keep_ratio: + gt_seg = mmcv.imrescale( + results[key], results['scale'], interpolation='nearest') + else: + gt_seg = mmcv.imresize( + results[key], results['scale'], interpolation='nearest') + results[key] = gt_seg + + def __call__(self, results): + """Call function to resize images, bounding boxes, masks, semantic + segmentation map. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Resized results, 'img_shape', 'pad_shape', 'scale_factor', + 'keep_ratio' keys are added into result dict. + """ + + if 'scale' not in results: + self._random_scale(results) + self._resize_img(results) + self._resize_seg(results) + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += (f'(img_scale={self.img_scale}, ' + f'multiscale_mode={self.multiscale_mode}, ' + f'ratio_range={self.ratio_range}, ' + f'keep_ratio={self.keep_ratio})') + return repr_str + + +@PIPELINES.register_module() +class RandomFlip(object): + """Flip the image & seg. + + If the input dict contains the key "flip", then the flag will be used, + otherwise it will be randomly decided by a ratio specified in the init + method. + + Args: + prob (float, optional): The flipping probability. Default: None. + direction(str, optional): The flipping direction. Options are + 'horizontal' and 'vertical'. Default: 'horizontal'. + """ + + @deprecated_api_warning({'flip_ratio': 'prob'}, cls_name='RandomFlip') + def __init__(self, prob=None, direction='horizontal'): + self.prob = prob + self.direction = direction + if prob is not None: + assert prob >= 0 and prob <= 1 + assert direction in ['horizontal', 'vertical'] + + def __call__(self, results): + """Call function to flip bounding boxes, masks, semantic segmentation + maps. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Flipped results, 'flip', 'flip_direction' keys are added into + result dict. + """ + + if 'flip' not in results: + flip = True if np.random.rand() < self.prob else False + results['flip'] = flip + if 'flip_direction' not in results: + results['flip_direction'] = self.direction + if results['flip']: + # flip image + results['img'] = mmcv.imflip( + results['img'], direction=results['flip_direction']) + + # flip segs + for key in results.get('seg_fields', []): + # use copy() to make numpy stride positive + results[key] = mmcv.imflip( + results[key], direction=results['flip_direction']).copy() + return results + + def __repr__(self): + return self.__class__.__name__ + f'(prob={self.prob})' + + +@PIPELINES.register_module() +class Pad(object): + """Pad the image & mask. + + There are two padding modes: (1) pad to a fixed size and (2) pad to the + minimum size that is divisible by some number. + Added keys are "pad_shape", "pad_fixed_size", "pad_size_divisor", + + Args: + size (tuple, optional): Fixed padding size. + size_divisor (int, optional): The divisor of padded size. + pad_val (float, optional): Padding value. Default: 0. + seg_pad_val (float, optional): Padding value of segmentation map. + Default: 255. + """ + + def __init__(self, + size=None, + size_divisor=None, + pad_val=0, + seg_pad_val=255): + self.size = size + self.size_divisor = size_divisor + self.pad_val = pad_val + self.seg_pad_val = seg_pad_val + # only one of size and size_divisor should be valid + assert size is not None or size_divisor is not None + assert size is None or size_divisor is None + + def _pad_img(self, results): + """Pad images according to ``self.size``.""" + if self.size is not None: + padded_img = mmcv.impad( + results['img'], shape=self.size, pad_val=self.pad_val) + elif self.size_divisor is not None: + padded_img = mmcv.impad_to_multiple( + results['img'], self.size_divisor, pad_val=self.pad_val) + results['img'] = padded_img + results['pad_shape'] = padded_img.shape + results['pad_fixed_size'] = self.size + results['pad_size_divisor'] = self.size_divisor + + def _pad_seg(self, results): + """Pad masks according to ``results['pad_shape']``.""" + for key in results.get('seg_fields', []): + results[key] = mmcv.impad( + results[key], + shape=results['pad_shape'][:2], + pad_val=self.seg_pad_val) + + def __call__(self, results): + """Call function to pad images, masks, semantic segmentation maps. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Updated result dict. + """ + + self._pad_img(results) + self._pad_seg(results) + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(size={self.size}, size_divisor={self.size_divisor}, ' \ + f'pad_val={self.pad_val})' + return repr_str + + +@PIPELINES.register_module() +class Normalize(object): + """Normalize the image. + + Added key is "img_norm_cfg". + + Args: + mean (sequence): Mean values of 3 channels. + std (sequence): Std values of 3 channels. + to_rgb (bool): Whether to convert the image from BGR to RGB, + default is true. + """ + + def __init__(self, mean, std, to_rgb=True): + self.mean = np.array(mean, dtype=np.float32) + self.std = np.array(std, dtype=np.float32) + self.to_rgb = to_rgb + + def __call__(self, results): + """Call function to normalize images. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Normalized results, 'img_norm_cfg' key is added into + result dict. + """ + + results['img'] = mmcv.imnormalize(results['img'], self.mean, self.std, + self.to_rgb) + results['img_norm_cfg'] = dict( + mean=self.mean, std=self.std, to_rgb=self.to_rgb) + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(mean={self.mean}, std={self.std}, to_rgb=' \ + f'{self.to_rgb})' + return repr_str + + +@PIPELINES.register_module() +class Rerange(object): + """Rerange the image pixel value. + + Args: + min_value (float or int): Minimum value of the reranged image. + Default: 0. + max_value (float or int): Maximum value of the reranged image. + Default: 255. + """ + + def __init__(self, min_value=0, max_value=255): + assert isinstance(min_value, float) or isinstance(min_value, int) + assert isinstance(max_value, float) or isinstance(max_value, int) + assert min_value < max_value + self.min_value = min_value + self.max_value = max_value + + def __call__(self, results): + """Call function to rerange images. + + Args: + results (dict): Result dict from loading pipeline. + Returns: + dict: Reranged results. + """ + + img = results['img'] + img_min_value = np.min(img) + img_max_value = np.max(img) + + assert img_min_value < img_max_value + # rerange to [0, 1] + img = (img - img_min_value) / (img_max_value - img_min_value) + # rerange to [min_value, max_value] + img = img * (self.max_value - self.min_value) + self.min_value + results['img'] = img + + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(min_value={self.min_value}, max_value={self.max_value})' + return repr_str + + +@PIPELINES.register_module() +class CLAHE(object): + """Use CLAHE method to process the image. + + See `ZUIDERVELD,K. Contrast Limited Adaptive Histogram Equalization[J]. + Graphics Gems, 1994:474-485.` for more information. + + Args: + clip_limit (float): Threshold for contrast limiting. Default: 40.0. + tile_grid_size (tuple[int]): Size of grid for histogram equalization. + Input image will be divided into equally sized rectangular tiles. + It defines the number of tiles in row and column. Default: (8, 8). + """ + + def __init__(self, clip_limit=40.0, tile_grid_size=(8, 8)): + assert isinstance(clip_limit, (float, int)) + self.clip_limit = clip_limit + assert is_tuple_of(tile_grid_size, int) + assert len(tile_grid_size) == 2 + self.tile_grid_size = tile_grid_size + + def __call__(self, results): + """Call function to Use CLAHE method process images. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Processed results. + """ + + for i in range(results['img'].shape[2]): + results['img'][:, :, i] = mmcv.clahe( + np.array(results['img'][:, :, i], dtype=np.uint8), + self.clip_limit, self.tile_grid_size) + + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(clip_limit={self.clip_limit}, '\ + f'tile_grid_size={self.tile_grid_size})' + return repr_str + + +@PIPELINES.register_module() +class RandomCrop(object): + """Random crop the image & seg. + + Args: + crop_size (tuple): Expected size after cropping, (h, w). + cat_max_ratio (float): The maximum ratio that single category could + occupy. + """ + + def __init__(self, crop_size, cat_max_ratio=1., ignore_index=255): + assert crop_size[0] > 0 and crop_size[1] > 0 + self.crop_size = crop_size + self.cat_max_ratio = cat_max_ratio + self.ignore_index = ignore_index + + def get_crop_bbox(self, img): + """Randomly get a crop bounding box.""" + margin_h = max(img.shape[0] - self.crop_size[0], 0) + margin_w = max(img.shape[1] - self.crop_size[1], 0) + offset_h = np.random.randint(0, margin_h + 1) + offset_w = np.random.randint(0, margin_w + 1) + crop_y1, crop_y2 = offset_h, offset_h + self.crop_size[0] + crop_x1, crop_x2 = offset_w, offset_w + self.crop_size[1] + + return crop_y1, crop_y2, crop_x1, crop_x2 + + def crop(self, img, crop_bbox): + """Crop from ``img``""" + crop_y1, crop_y2, crop_x1, crop_x2 = crop_bbox + img = img[crop_y1:crop_y2, crop_x1:crop_x2, ...] + return img + + def __call__(self, results): + """Call function to randomly crop images, semantic segmentation maps. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Randomly cropped results, 'img_shape' key in result dict is + updated according to crop size. + """ + + img = results['img'] + crop_bbox = self.get_crop_bbox(img) + if self.cat_max_ratio < 1.: + # Repeat 10 times + for _ in range(10): + seg_temp = self.crop(results['gt_semantic_seg'], crop_bbox) + labels, cnt = np.unique(seg_temp, return_counts=True) + cnt = cnt[labels != self.ignore_index] + if len(cnt) > 1 and np.max(cnt) / np.sum( + cnt) < self.cat_max_ratio: + break + crop_bbox = self.get_crop_bbox(img) + + # crop the image + img = self.crop(img, crop_bbox) + img_shape = img.shape + results['img'] = img + results['img_shape'] = img_shape + + # crop semantic seg + for key in results.get('seg_fields', []): + results[key] = self.crop(results[key], crop_bbox) + + return results + + def __repr__(self): + return self.__class__.__name__ + f'(crop_size={self.crop_size})' + + +@PIPELINES.register_module() +class RandomRotate(object): + """Rotate the image & seg. + + Args: + prob (float): The rotation probability. + degree (float, tuple[float]): Range of degrees to select from. If + degree is a number instead of tuple like (min, max), + the range of degree will be (``-degree``, ``+degree``) + pad_val (float, optional): Padding value of image. Default: 0. + seg_pad_val (float, optional): Padding value of segmentation map. + Default: 255. + center (tuple[float], optional): Center point (w, h) of the rotation in + the source image. If not specified, the center of the image will be + used. Default: None. + auto_bound (bool): Whether to adjust the image size to cover the whole + rotated image. Default: False + """ + + def __init__(self, + prob, + degree, + pad_val=0, + seg_pad_val=255, + center=None, + auto_bound=False): + self.prob = prob + assert prob >= 0 and prob <= 1 + if isinstance(degree, (float, int)): + assert degree > 0, f'degree {degree} should be positive' + self.degree = (-degree, degree) + else: + self.degree = degree + assert len(self.degree) == 2, f'degree {self.degree} should be a ' \ + f'tuple of (min, max)' + self.pal_val = pad_val + self.seg_pad_val = seg_pad_val + self.center = center + self.auto_bound = auto_bound + + def __call__(self, results): + """Call function to rotate image, semantic segmentation maps. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Rotated results. + """ + + rotate = True if np.random.rand() < self.prob else False + degree = np.random.uniform(min(*self.degree), max(*self.degree)) + if rotate: + # rotate image + results['img'] = mmcv.imrotate( + results['img'], + angle=degree, + border_value=self.pal_val, + center=self.center, + auto_bound=self.auto_bound) + + # rotate segs + for key in results.get('seg_fields', []): + results[key] = mmcv.imrotate( + results[key], + angle=degree, + border_value=self.seg_pad_val, + center=self.center, + auto_bound=self.auto_bound, + interpolation='nearest') + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(prob={self.prob}, ' \ + f'degree={self.degree}, ' \ + f'pad_val={self.pal_val}, ' \ + f'seg_pad_val={self.seg_pad_val}, ' \ + f'center={self.center}, ' \ + f'auto_bound={self.auto_bound})' + return repr_str + + +@PIPELINES.register_module() +class RGB2Gray(object): + """Convert RGB image to grayscale image. + + This transform calculate the weighted mean of input image channels with + ``weights`` and then expand the channels to ``out_channels``. When + ``out_channels`` is None, the number of output channels is the same as + input channels. + + Args: + out_channels (int): Expected number of output channels after + transforming. Default: None. + weights (tuple[float]): The weights to calculate the weighted mean. + Default: (0.299, 0.587, 0.114). + """ + + def __init__(self, out_channels=None, weights=(0.299, 0.587, 0.114)): + assert out_channels is None or out_channels > 0 + self.out_channels = out_channels + assert isinstance(weights, tuple) + for item in weights: + assert isinstance(item, (float, int)) + self.weights = weights + + def __call__(self, results): + """Call function to convert RGB image to grayscale image. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Result dict with grayscale image. + """ + img = results['img'] + assert len(img.shape) == 3 + assert img.shape[2] == len(self.weights) + weights = np.array(self.weights).reshape((1, 1, -1)) + img = (img * weights).sum(2, keepdims=True) + if self.out_channels is None: + img = img.repeat(weights.shape[2], axis=2) + else: + img = img.repeat(self.out_channels, axis=2) + + results['img'] = img + results['img_shape'] = img.shape + + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(out_channels={self.out_channels}, ' \ + f'weights={self.weights})' + return repr_str + + +@PIPELINES.register_module() +class AdjustGamma(object): + """Using gamma correction to process the image. + + Args: + gamma (float or int): Gamma value used in gamma correction. + Default: 1.0. + """ + + def __init__(self, gamma=1.0): + assert isinstance(gamma, float) or isinstance(gamma, int) + assert gamma > 0 + self.gamma = gamma + inv_gamma = 1.0 / gamma + self.table = np.array([(i / 255.0)**inv_gamma * 255 + for i in np.arange(256)]).astype('uint8') + + def __call__(self, results): + """Call function to process the image with gamma correction. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Processed results. + """ + + results['img'] = mmcv.lut_transform( + np.array(results['img'], dtype=np.uint8), self.table) + + return results + + def __repr__(self): + return self.__class__.__name__ + f'(gamma={self.gamma})' + + +@PIPELINES.register_module() +class SegRescale(object): + """Rescale semantic segmentation maps. + + Args: + scale_factor (float): The scale factor of the final output. + """ + + def __init__(self, scale_factor=1): + self.scale_factor = scale_factor + + def __call__(self, results): + """Call function to scale the semantic segmentation map. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Result dict with semantic segmentation map scaled. + """ + for key in results.get('seg_fields', []): + if self.scale_factor != 1: + results[key] = mmcv.imrescale( + results[key], self.scale_factor, interpolation='nearest') + return results + + def __repr__(self): + return self.__class__.__name__ + f'(scale_factor={self.scale_factor})' + + +@PIPELINES.register_module() +class PhotoMetricDistortion(object): + """Apply photometric distortion to image sequentially, every transformation + is applied with a probability of 0.5. The position of random contrast is in + second or second to last. + + 1. random brightness + 2. random contrast (mode 0) + 3. convert color from BGR to HSV + 4. random saturation + 5. random hue + 6. convert color from HSV to BGR + 7. random contrast (mode 1) + + Args: + brightness_delta (int): delta of brightness. + contrast_range (tuple): range of contrast. + saturation_range (tuple): range of saturation. + hue_delta (int): delta of hue. + """ + + def __init__(self, + brightness_delta=32, + contrast_range=(0.5, 1.5), + saturation_range=(0.5, 1.5), + hue_delta=18): + self.brightness_delta = brightness_delta + self.contrast_lower, self.contrast_upper = contrast_range + self.saturation_lower, self.saturation_upper = saturation_range + self.hue_delta = hue_delta + + def convert(self, img, alpha=1, beta=0): + """Multiple with alpha and add beat with clip.""" + img = img.astype(np.float32) * alpha + beta + img = np.clip(img, 0, 255) + return img.astype(np.uint8) + + def brightness(self, img): + """Brightness distortion.""" + if random.randint(2): + return self.convert( + img, + beta=random.uniform(-self.brightness_delta, + self.brightness_delta)) + return img + + def contrast(self, img): + """Contrast distortion.""" + if random.randint(2): + return self.convert( + img, + alpha=random.uniform(self.contrast_lower, self.contrast_upper)) + return img + + def saturation(self, img): + """Saturation distortion.""" + if random.randint(2): + img = mmcv.bgr2hsv(img) + img[:, :, 1] = self.convert( + img[:, :, 1], + alpha=random.uniform(self.saturation_lower, + self.saturation_upper)) + img = mmcv.hsv2bgr(img) + return img + + def hue(self, img): + """Hue distortion.""" + if random.randint(2): + img = mmcv.bgr2hsv(img) + img[:, :, + 0] = (img[:, :, 0].astype(int) + + random.randint(-self.hue_delta, self.hue_delta)) % 180 + img = mmcv.hsv2bgr(img) + return img + + def __call__(self, results): + """Call function to perform photometric distortion on images. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Result dict with images distorted. + """ + + img = results['img'] + # random brightness + img = self.brightness(img) + + # mode == 0 --> do random contrast first + # mode == 1 --> do random contrast last + mode = random.randint(2) + if mode == 1: + img = self.contrast(img) + + # random saturation + img = self.saturation(img) + + # random hue + img = self.hue(img) + + # random contrast + if mode == 0: + img = self.contrast(img) + + results['img'] = img + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += (f'(brightness_delta={self.brightness_delta}, ' + f'contrast_range=({self.contrast_lower}, ' + f'{self.contrast_upper}), ' + f'saturation_range=({self.saturation_lower}, ' + f'{self.saturation_upper}), ' + f'hue_delta={self.hue_delta})') + return repr_str diff --git a/segmentation/mmseg/datasets/stare.py b/segmentation/mmseg/datasets/stare.py new file mode 100644 index 0000000..cbd14e0 --- /dev/null +++ b/segmentation/mmseg/datasets/stare.py @@ -0,0 +1,27 @@ +import os.path as osp + +from .builder import DATASETS +from .custom import CustomDataset + + +@DATASETS.register_module() +class STAREDataset(CustomDataset): + """STARE dataset. + + In segmentation map annotation for STARE, 0 stands for background, which is + included in 2 categories. ``reduce_zero_label`` is fixed to False. The + ``img_suffix`` is fixed to '.png' and ``seg_map_suffix`` is fixed to + '.ah.png'. + """ + + CLASSES = ('background', 'vessel') + + PALETTE = [[120, 120, 120], [6, 230, 230]] + + def __init__(self, **kwargs): + super(STAREDataset, self).__init__( + img_suffix='.png', + seg_map_suffix='.ah.png', + reduce_zero_label=False, + **kwargs) + assert osp.exists(self.img_dir) diff --git a/segmentation/mmseg/datasets/voc.py b/segmentation/mmseg/datasets/voc.py new file mode 100644 index 0000000..a885520 --- /dev/null +++ b/segmentation/mmseg/datasets/voc.py @@ -0,0 +1,29 @@ +import os.path as osp + +from .builder import DATASETS +from .custom import CustomDataset + + +@DATASETS.register_module() +class PascalVOCDataset(CustomDataset): + """Pascal VOC dataset. + + Args: + split (str): Split txt file for Pascal VOC. + """ + + CLASSES = ('background', 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', + 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', + 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', + 'train', 'tvmonitor') + + PALETTE = [[0, 0, 0], [128, 0, 0], [0, 128, 0], [128, 128, 0], [0, 0, 128], + [128, 0, 128], [0, 128, 128], [128, 128, 128], [64, 0, 0], + [192, 0, 0], [64, 128, 0], [192, 128, 0], [64, 0, 128], + [192, 0, 128], [64, 128, 128], [192, 128, 128], [0, 64, 0], + [128, 64, 0], [0, 192, 0], [128, 192, 0], [0, 64, 128]] + + def __init__(self, split, **kwargs): + super(PascalVOCDataset, self).__init__( + img_suffix='.jpg', seg_map_suffix='.png', split=split, **kwargs) + assert osp.exists(self.img_dir) and self.split is not None diff --git a/segmentation/mmseg/models/__init__.py b/segmentation/mmseg/models/__init__.py new file mode 100644 index 0000000..3cf93f8 --- /dev/null +++ b/segmentation/mmseg/models/__init__.py @@ -0,0 +1,12 @@ +from .backbones import * # noqa: F401,F403 +from .builder import (BACKBONES, HEADS, LOSSES, SEGMENTORS, build_backbone, + build_head, build_loss, build_segmentor) +from .decode_heads import * # noqa: F401,F403 +from .losses import * # noqa: F401,F403 +from .necks import * # noqa: F401,F403 +from .segmentors import * # noqa: F401,F403 + +__all__ = [ + 'BACKBONES', 'HEADS', 'LOSSES', 'SEGMENTORS', 'build_backbone', + 'build_head', 'build_loss', 'build_segmentor' +] diff --git a/segmentation/mmseg/models/backbones/__init__.py b/segmentation/mmseg/models/backbones/__init__.py new file mode 100644 index 0000000..10c4e89 --- /dev/null +++ b/segmentation/mmseg/models/backbones/__init__.py @@ -0,0 +1,18 @@ +from .cgnet import CGNet +from .fast_scnn import FastSCNN +from .hrnet import HRNet +from .mobilenet_v2 import MobileNetV2 +from .mobilenet_v3 import MobileNetV3 +from .resnest import ResNeSt +from .resnet import ResNet, ResNetV1c, ResNetV1d +from .resnext import ResNeXt +from .unet import UNet +from .vit import VisionTransformer +# from .lit_ti import LIT_Ti +from .litv2 import LITv2 + +__all__ = [ + 'ResNet', 'ResNetV1c', 'ResNetV1d', 'ResNeXt', 'HRNet', 'FastSCNN', + 'ResNeSt', 'MobileNetV2', 'UNet', 'CGNet', 'MobileNetV3', + 'VisionTransformer', 'LITv2' +] diff --git a/segmentation/mmseg/models/backbones/cgnet.py b/segmentation/mmseg/models/backbones/cgnet.py new file mode 100644 index 0000000..032a55d --- /dev/null +++ b/segmentation/mmseg/models/backbones/cgnet.py @@ -0,0 +1,367 @@ +import torch +import torch.nn as nn +import torch.utils.checkpoint as cp +from mmcv.cnn import (ConvModule, build_conv_layer, build_norm_layer, + constant_init, kaiming_init) +from mmcv.runner import load_checkpoint +from mmcv.utils.parrots_wrapper import _BatchNorm + +from mmseg.utils import get_root_logger +from ..builder import BACKBONES + + +class GlobalContextExtractor(nn.Module): + """Global Context Extractor for CGNet. + + This class is employed to refine the joint feature of both local feature + and surrounding context. + + Args: + channel (int): Number of input feature channels. + reduction (int): Reductions for global context extractor. Default: 16. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + """ + + def __init__(self, channel, reduction=16, with_cp=False): + super(GlobalContextExtractor, self).__init__() + self.channel = channel + self.reduction = reduction + assert reduction >= 1 and channel >= reduction + self.with_cp = with_cp + self.avg_pool = nn.AdaptiveAvgPool2d(1) + self.fc = nn.Sequential( + nn.Linear(channel, channel // reduction), nn.ReLU(inplace=True), + nn.Linear(channel // reduction, channel), nn.Sigmoid()) + + def forward(self, x): + + def _inner_forward(x): + num_batch, num_channel = x.size()[:2] + y = self.avg_pool(x).view(num_batch, num_channel) + y = self.fc(y).view(num_batch, num_channel, 1, 1) + return x * y + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x) + else: + out = _inner_forward(x) + + return out + + +class ContextGuidedBlock(nn.Module): + """Context Guided Block for CGNet. + + This class consists of four components: local feature extractor, + surrounding feature extractor, joint feature extractor and global + context extractor. + + Args: + in_channels (int): Number of input feature channels. + out_channels (int): Number of output feature channels. + dilation (int): Dilation rate for surrounding context extractor. + Default: 2. + reduction (int): Reduction for global context extractor. Default: 16. + skip_connect (bool): Add input to output or not. Default: True. + downsample (bool): Downsample the input to 1/2 or not. Default: False. + conv_cfg (dict): Config dict for convolution layer. + Default: None, which means using conv2d. + norm_cfg (dict): Config dict for normalization layer. + Default: dict(type='BN', requires_grad=True). + act_cfg (dict): Config dict for activation layer. + Default: dict(type='PReLU'). + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + """ + + def __init__(self, + in_channels, + out_channels, + dilation=2, + reduction=16, + skip_connect=True, + downsample=False, + conv_cfg=None, + norm_cfg=dict(type='BN', requires_grad=True), + act_cfg=dict(type='PReLU'), + with_cp=False): + super(ContextGuidedBlock, self).__init__() + self.with_cp = with_cp + self.downsample = downsample + + channels = out_channels if downsample else out_channels // 2 + if 'type' in act_cfg and act_cfg['type'] == 'PReLU': + act_cfg['num_parameters'] = channels + kernel_size = 3 if downsample else 1 + stride = 2 if downsample else 1 + padding = (kernel_size - 1) // 2 + + self.conv1x1 = ConvModule( + in_channels, + channels, + kernel_size, + stride, + padding, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + + self.f_loc = build_conv_layer( + conv_cfg, + channels, + channels, + kernel_size=3, + padding=1, + groups=channels, + bias=False) + self.f_sur = build_conv_layer( + conv_cfg, + channels, + channels, + kernel_size=3, + padding=dilation, + groups=channels, + dilation=dilation, + bias=False) + + self.bn = build_norm_layer(norm_cfg, 2 * channels)[1] + self.activate = nn.PReLU(2 * channels) + + if downsample: + self.bottleneck = build_conv_layer( + conv_cfg, + 2 * channels, + out_channels, + kernel_size=1, + bias=False) + + self.skip_connect = skip_connect and not downsample + self.f_glo = GlobalContextExtractor(out_channels, reduction, with_cp) + + def forward(self, x): + + def _inner_forward(x): + out = self.conv1x1(x) + loc = self.f_loc(out) + sur = self.f_sur(out) + + joi_feat = torch.cat([loc, sur], 1) # the joint feature + joi_feat = self.bn(joi_feat) + joi_feat = self.activate(joi_feat) + if self.downsample: + joi_feat = self.bottleneck(joi_feat) # channel = out_channels + # f_glo is employed to refine the joint feature + out = self.f_glo(joi_feat) + + if self.skip_connect: + return x + out + else: + return out + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x) + else: + out = _inner_forward(x) + + return out + + +class InputInjection(nn.Module): + """Downsampling module for CGNet.""" + + def __init__(self, num_downsampling): + super(InputInjection, self).__init__() + self.pool = nn.ModuleList() + for i in range(num_downsampling): + self.pool.append(nn.AvgPool2d(3, stride=2, padding=1)) + + def forward(self, x): + for pool in self.pool: + x = pool(x) + return x + + +@BACKBONES.register_module() +class CGNet(nn.Module): + """CGNet backbone. + + A Light-weight Context Guided Network for Semantic Segmentation + arXiv: https://arxiv.org/abs/1811.08201 + + Args: + in_channels (int): Number of input image channels. Normally 3. + num_channels (tuple[int]): Numbers of feature channels at each stages. + Default: (32, 64, 128). + num_blocks (tuple[int]): Numbers of CG blocks at stage 1 and stage 2. + Default: (3, 21). + dilations (tuple[int]): Dilation rate for surrounding context + extractors at stage 1 and stage 2. Default: (2, 4). + reductions (tuple[int]): Reductions for global context extractors at + stage 1 and stage 2. Default: (8, 16). + conv_cfg (dict): Config dict for convolution layer. + Default: None, which means using conv2d. + norm_cfg (dict): Config dict for normalization layer. + Default: dict(type='BN', requires_grad=True). + act_cfg (dict): Config dict for activation layer. + Default: dict(type='PReLU'). + norm_eval (bool): Whether to set norm layers to eval mode, namely, + freeze running stats (mean and var). Note: Effect on Batch Norm + and its variants only. Default: False. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + """ + + def __init__(self, + in_channels=3, + num_channels=(32, 64, 128), + num_blocks=(3, 21), + dilations=(2, 4), + reductions=(8, 16), + conv_cfg=None, + norm_cfg=dict(type='BN', requires_grad=True), + act_cfg=dict(type='PReLU'), + norm_eval=False, + with_cp=False): + + super(CGNet, self).__init__() + self.in_channels = in_channels + self.num_channels = num_channels + assert isinstance(self.num_channels, tuple) and len( + self.num_channels) == 3 + self.num_blocks = num_blocks + assert isinstance(self.num_blocks, tuple) and len(self.num_blocks) == 2 + self.dilations = dilations + assert isinstance(self.dilations, tuple) and len(self.dilations) == 2 + self.reductions = reductions + assert isinstance(self.reductions, tuple) and len(self.reductions) == 2 + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + if 'type' in self.act_cfg and self.act_cfg['type'] == 'PReLU': + self.act_cfg['num_parameters'] = num_channels[0] + self.norm_eval = norm_eval + self.with_cp = with_cp + + cur_channels = in_channels + self.stem = nn.ModuleList() + for i in range(3): + self.stem.append( + ConvModule( + cur_channels, + num_channels[0], + 3, + 2 if i == 0 else 1, + padding=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg)) + cur_channels = num_channels[0] + + self.inject_2x = InputInjection(1) # down-sample for Input, factor=2 + self.inject_4x = InputInjection(2) # down-sample for Input, factor=4 + + cur_channels += in_channels + self.norm_prelu_0 = nn.Sequential( + build_norm_layer(norm_cfg, cur_channels)[1], + nn.PReLU(cur_channels)) + + # stage 1 + self.level1 = nn.ModuleList() + for i in range(num_blocks[0]): + self.level1.append( + ContextGuidedBlock( + cur_channels if i == 0 else num_channels[1], + num_channels[1], + dilations[0], + reductions[0], + downsample=(i == 0), + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + with_cp=with_cp)) # CG block + + cur_channels = 2 * num_channels[1] + in_channels + self.norm_prelu_1 = nn.Sequential( + build_norm_layer(norm_cfg, cur_channels)[1], + nn.PReLU(cur_channels)) + + # stage 2 + self.level2 = nn.ModuleList() + for i in range(num_blocks[1]): + self.level2.append( + ContextGuidedBlock( + cur_channels if i == 0 else num_channels[2], + num_channels[2], + dilations[1], + reductions[1], + downsample=(i == 0), + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + with_cp=with_cp)) # CG block + + cur_channels = 2 * num_channels[2] + self.norm_prelu_2 = nn.Sequential( + build_norm_layer(norm_cfg, cur_channels)[1], + nn.PReLU(cur_channels)) + + def forward(self, x): + output = [] + + # stage 0 + inp_2x = self.inject_2x(x) + inp_4x = self.inject_4x(x) + for layer in self.stem: + x = layer(x) + x = self.norm_prelu_0(torch.cat([x, inp_2x], 1)) + output.append(x) + + # stage 1 + for i, layer in enumerate(self.level1): + x = layer(x) + if i == 0: + down1 = x + x = self.norm_prelu_1(torch.cat([x, down1, inp_4x], 1)) + output.append(x) + + # stage 2 + for i, layer in enumerate(self.level2): + x = layer(x) + if i == 0: + down2 = x + x = self.norm_prelu_2(torch.cat([down2, x], 1)) + output.append(x) + + return output + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + if isinstance(pretrained, str): + logger = get_root_logger() + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + for m in self.modules(): + if isinstance(m, (nn.Conv2d, nn.Linear)): + kaiming_init(m) + elif isinstance(m, (_BatchNorm, nn.GroupNorm)): + constant_init(m, 1) + elif isinstance(m, nn.PReLU): + constant_init(m, 0) + else: + raise TypeError('pretrained must be a str or None') + + def train(self, mode=True): + """Convert the model into training mode will keeping the normalization + layer freezed.""" + super(CGNet, self).train(mode) + if mode and self.norm_eval: + for m in self.modules(): + # trick: eval have effect on BatchNorm only + if isinstance(m, _BatchNorm): + m.eval() diff --git a/segmentation/mmseg/models/backbones/fast_scnn.py b/segmentation/mmseg/models/backbones/fast_scnn.py new file mode 100644 index 0000000..ee115ff --- /dev/null +++ b/segmentation/mmseg/models/backbones/fast_scnn.py @@ -0,0 +1,375 @@ +import torch +import torch.nn as nn +from mmcv.cnn import (ConvModule, DepthwiseSeparableConvModule, constant_init, + kaiming_init) +from torch.nn.modules.batchnorm import _BatchNorm + +from mmseg.models.decode_heads.psp_head import PPM +from mmseg.ops import resize +from ..builder import BACKBONES +from ..utils.inverted_residual import InvertedResidual + + +class LearningToDownsample(nn.Module): + """Learning to downsample module. + + Args: + in_channels (int): Number of input channels. + dw_channels (tuple[int]): Number of output channels of the first and + the second depthwise conv (dwconv) layers. + out_channels (int): Number of output channels of the whole + 'learning to downsample' module. + conv_cfg (dict | None): Config of conv layers. Default: None + norm_cfg (dict | None): Config of norm layers. Default: + dict(type='BN') + act_cfg (dict): Config of activation layers. Default: + dict(type='ReLU') + """ + + def __init__(self, + in_channels, + dw_channels, + out_channels, + conv_cfg=None, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU')): + super(LearningToDownsample, self).__init__() + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + dw_channels1 = dw_channels[0] + dw_channels2 = dw_channels[1] + + self.conv = ConvModule( + in_channels, + dw_channels1, + 3, + stride=2, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.dsconv1 = DepthwiseSeparableConvModule( + dw_channels1, + dw_channels2, + kernel_size=3, + stride=2, + padding=1, + norm_cfg=self.norm_cfg) + self.dsconv2 = DepthwiseSeparableConvModule( + dw_channels2, + out_channels, + kernel_size=3, + stride=2, + padding=1, + norm_cfg=self.norm_cfg) + + def forward(self, x): + x = self.conv(x) + x = self.dsconv1(x) + x = self.dsconv2(x) + return x + + +class GlobalFeatureExtractor(nn.Module): + """Global feature extractor module. + + Args: + in_channels (int): Number of input channels of the GFE module. + Default: 64 + block_channels (tuple[int]): Tuple of ints. Each int specifies the + number of output channels of each Inverted Residual module. + Default: (64, 96, 128) + out_channels(int): Number of output channels of the GFE module. + Default: 128 + expand_ratio (int): Adjusts number of channels of the hidden layer + in InvertedResidual by this amount. + Default: 6 + num_blocks (tuple[int]): Tuple of ints. Each int specifies the + number of times each Inverted Residual module is repeated. + The repeated Inverted Residual modules are called a 'group'. + Default: (3, 3, 3) + strides (tuple[int]): Tuple of ints. Each int specifies + the downsampling factor of each 'group'. + Default: (2, 2, 1) + pool_scales (tuple[int]): Tuple of ints. Each int specifies + the parameter required in 'global average pooling' within PPM. + Default: (1, 2, 3, 6) + conv_cfg (dict | None): Config of conv layers. Default: None + norm_cfg (dict | None): Config of norm layers. Default: + dict(type='BN') + act_cfg (dict): Config of activation layers. Default: + dict(type='ReLU') + align_corners (bool): align_corners argument of F.interpolate. + Default: False + """ + + def __init__(self, + in_channels=64, + block_channels=(64, 96, 128), + out_channels=128, + expand_ratio=6, + num_blocks=(3, 3, 3), + strides=(2, 2, 1), + pool_scales=(1, 2, 3, 6), + conv_cfg=None, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU'), + align_corners=False): + super(GlobalFeatureExtractor, self).__init__() + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + assert len(block_channels) == len(num_blocks) == 3 + self.bottleneck1 = self._make_layer(in_channels, block_channels[0], + num_blocks[0], strides[0], + expand_ratio) + self.bottleneck2 = self._make_layer(block_channels[0], + block_channels[1], num_blocks[1], + strides[1], expand_ratio) + self.bottleneck3 = self._make_layer(block_channels[1], + block_channels[2], num_blocks[2], + strides[2], expand_ratio) + self.ppm = PPM( + pool_scales, + block_channels[2], + block_channels[2] // 4, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg, + align_corners=align_corners) + self.out = ConvModule( + block_channels[2] * 2, + out_channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def _make_layer(self, + in_channels, + out_channels, + blocks, + stride=1, + expand_ratio=6): + layers = [ + InvertedResidual( + in_channels, + out_channels, + stride, + expand_ratio, + norm_cfg=self.norm_cfg) + ] + for i in range(1, blocks): + layers.append( + InvertedResidual( + out_channels, + out_channels, + 1, + expand_ratio, + norm_cfg=self.norm_cfg)) + return nn.Sequential(*layers) + + def forward(self, x): + x = self.bottleneck1(x) + x = self.bottleneck2(x) + x = self.bottleneck3(x) + x = torch.cat([x, *self.ppm(x)], dim=1) + x = self.out(x) + return x + + +class FeatureFusionModule(nn.Module): + """Feature fusion module. + + Args: + higher_in_channels (int): Number of input channels of the + higher-resolution branch. + lower_in_channels (int): Number of input channels of the + lower-resolution branch. + out_channels (int): Number of output channels. + conv_cfg (dict | None): Config of conv layers. Default: None + norm_cfg (dict | None): Config of norm layers. Default: + dict(type='BN') + act_cfg (dict): Config of activation layers. Default: + dict(type='ReLU') + align_corners (bool): align_corners argument of F.interpolate. + Default: False + """ + + def __init__(self, + higher_in_channels, + lower_in_channels, + out_channels, + conv_cfg=None, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU'), + align_corners=False): + super(FeatureFusionModule, self).__init__() + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + self.align_corners = align_corners + self.dwconv = ConvModule( + lower_in_channels, + out_channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.conv_lower_res = ConvModule( + out_channels, + out_channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=None) + self.conv_higher_res = ConvModule( + higher_in_channels, + out_channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=None) + self.relu = nn.ReLU(True) + + def forward(self, higher_res_feature, lower_res_feature): + lower_res_feature = resize( + lower_res_feature, + size=higher_res_feature.size()[2:], + mode='bilinear', + align_corners=self.align_corners) + lower_res_feature = self.dwconv(lower_res_feature) + lower_res_feature = self.conv_lower_res(lower_res_feature) + + higher_res_feature = self.conv_higher_res(higher_res_feature) + out = higher_res_feature + lower_res_feature + return self.relu(out) + + +@BACKBONES.register_module() +class FastSCNN(nn.Module): + """Fast-SCNN Backbone. + + Args: + in_channels (int): Number of input image channels. Default: 3. + downsample_dw_channels (tuple[int]): Number of output channels after + the first conv layer & the second conv layer in + Learning-To-Downsample (LTD) module. + Default: (32, 48). + global_in_channels (int): Number of input channels of + Global Feature Extractor(GFE). + Equal to number of output channels of LTD. + Default: 64. + global_block_channels (tuple[int]): Tuple of integers that describe + the output channels for each of the MobileNet-v2 bottleneck + residual blocks in GFE. + Default: (64, 96, 128). + global_block_strides (tuple[int]): Tuple of integers + that describe the strides (downsampling factors) for each of the + MobileNet-v2 bottleneck residual blocks in GFE. + Default: (2, 2, 1). + global_out_channels (int): Number of output channels of GFE. + Default: 128. + higher_in_channels (int): Number of input channels of the higher + resolution branch in FFM. + Equal to global_in_channels. + Default: 64. + lower_in_channels (int): Number of input channels of the lower + resolution branch in FFM. + Equal to global_out_channels. + Default: 128. + fusion_out_channels (int): Number of output channels of FFM. + Default: 128. + out_indices (tuple): Tuple of indices of list + [higher_res_features, lower_res_features, fusion_output]. + Often set to (0,1,2) to enable aux. heads. + Default: (0, 1, 2). + conv_cfg (dict | None): Config of conv layers. Default: None + norm_cfg (dict | None): Config of norm layers. Default: + dict(type='BN') + act_cfg (dict): Config of activation layers. Default: + dict(type='ReLU') + align_corners (bool): align_corners argument of F.interpolate. + Default: False + """ + + def __init__(self, + in_channels=3, + downsample_dw_channels=(32, 48), + global_in_channels=64, + global_block_channels=(64, 96, 128), + global_block_strides=(2, 2, 1), + global_out_channels=128, + higher_in_channels=64, + lower_in_channels=128, + fusion_out_channels=128, + out_indices=(0, 1, 2), + conv_cfg=None, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU'), + align_corners=False): + + super(FastSCNN, self).__init__() + if global_in_channels != higher_in_channels: + raise AssertionError('Global Input Channels must be the same \ + with Higher Input Channels!') + elif global_out_channels != lower_in_channels: + raise AssertionError('Global Output Channels must be the same \ + with Lower Input Channels!') + + self.in_channels = in_channels + self.downsample_dw_channels1 = downsample_dw_channels[0] + self.downsample_dw_channels2 = downsample_dw_channels[1] + self.global_in_channels = global_in_channels + self.global_block_channels = global_block_channels + self.global_block_strides = global_block_strides + self.global_out_channels = global_out_channels + self.higher_in_channels = higher_in_channels + self.lower_in_channels = lower_in_channels + self.fusion_out_channels = fusion_out_channels + self.out_indices = out_indices + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + self.align_corners = align_corners + self.learning_to_downsample = LearningToDownsample( + in_channels, + downsample_dw_channels, + global_in_channels, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.global_feature_extractor = GlobalFeatureExtractor( + global_in_channels, + global_block_channels, + global_out_channels, + strides=self.global_block_strides, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg, + align_corners=self.align_corners) + self.feature_fusion = FeatureFusionModule( + higher_in_channels, + lower_in_channels, + fusion_out_channels, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg, + align_corners=self.align_corners) + + def init_weights(self, pretrained=None): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + kaiming_init(m) + elif isinstance(m, (_BatchNorm, nn.GroupNorm)): + constant_init(m, 1) + + def forward(self, x): + higher_res_features = self.learning_to_downsample(x) + lower_res_features = self.global_feature_extractor(higher_res_features) + fusion_output = self.feature_fusion(higher_res_features, + lower_res_features) + + outs = [higher_res_features, lower_res_features, fusion_output] + outs = [outs[i] for i in self.out_indices] + return tuple(outs) diff --git a/segmentation/mmseg/models/backbones/hrnet.py b/segmentation/mmseg/models/backbones/hrnet.py new file mode 100644 index 0000000..5010a2e --- /dev/null +++ b/segmentation/mmseg/models/backbones/hrnet.py @@ -0,0 +1,555 @@ +import torch.nn as nn +from mmcv.cnn import (build_conv_layer, build_norm_layer, constant_init, + kaiming_init) +from mmcv.runner import load_checkpoint +from mmcv.utils.parrots_wrapper import _BatchNorm + +from mmseg.ops import Upsample, resize +from mmseg.utils import get_root_logger +from ..builder import BACKBONES +from .resnet import BasicBlock, Bottleneck + + +class HRModule(nn.Module): + """High-Resolution Module for HRNet. + + In this module, every branch has 4 BasicBlocks/Bottlenecks. Fusion/Exchange + is in this module. + """ + + def __init__(self, + num_branches, + blocks, + num_blocks, + in_channels, + num_channels, + multiscale_output=True, + with_cp=False, + conv_cfg=None, + norm_cfg=dict(type='BN', requires_grad=True)): + super(HRModule, self).__init__() + self._check_branches(num_branches, num_blocks, in_channels, + num_channels) + + self.in_channels = in_channels + self.num_branches = num_branches + + self.multiscale_output = multiscale_output + self.norm_cfg = norm_cfg + self.conv_cfg = conv_cfg + self.with_cp = with_cp + self.branches = self._make_branches(num_branches, blocks, num_blocks, + num_channels) + self.fuse_layers = self._make_fuse_layers() + self.relu = nn.ReLU(inplace=False) + + def _check_branches(self, num_branches, num_blocks, in_channels, + num_channels): + """Check branches configuration.""" + if num_branches != len(num_blocks): + error_msg = f'NUM_BRANCHES({num_branches}) <> NUM_BLOCKS(' \ + f'{len(num_blocks)})' + raise ValueError(error_msg) + + if num_branches != len(num_channels): + error_msg = f'NUM_BRANCHES({num_branches}) <> NUM_CHANNELS(' \ + f'{len(num_channels)})' + raise ValueError(error_msg) + + if num_branches != len(in_channels): + error_msg = f'NUM_BRANCHES({num_branches}) <> NUM_INCHANNELS(' \ + f'{len(in_channels)})' + raise ValueError(error_msg) + + def _make_one_branch(self, + branch_index, + block, + num_blocks, + num_channels, + stride=1): + """Build one branch.""" + downsample = None + if stride != 1 or \ + self.in_channels[branch_index] != \ + num_channels[branch_index] * block.expansion: + downsample = nn.Sequential( + build_conv_layer( + self.conv_cfg, + self.in_channels[branch_index], + num_channels[branch_index] * block.expansion, + kernel_size=1, + stride=stride, + bias=False), + build_norm_layer(self.norm_cfg, num_channels[branch_index] * + block.expansion)[1]) + + layers = [] + layers.append( + block( + self.in_channels[branch_index], + num_channels[branch_index], + stride, + downsample=downsample, + with_cp=self.with_cp, + norm_cfg=self.norm_cfg, + conv_cfg=self.conv_cfg)) + self.in_channels[branch_index] = \ + num_channels[branch_index] * block.expansion + for i in range(1, num_blocks[branch_index]): + layers.append( + block( + self.in_channels[branch_index], + num_channels[branch_index], + with_cp=self.with_cp, + norm_cfg=self.norm_cfg, + conv_cfg=self.conv_cfg)) + + return nn.Sequential(*layers) + + def _make_branches(self, num_branches, block, num_blocks, num_channels): + """Build multiple branch.""" + branches = [] + + for i in range(num_branches): + branches.append( + self._make_one_branch(i, block, num_blocks, num_channels)) + + return nn.ModuleList(branches) + + def _make_fuse_layers(self): + """Build fuse layer.""" + if self.num_branches == 1: + return None + + num_branches = self.num_branches + in_channels = self.in_channels + fuse_layers = [] + num_out_branches = num_branches if self.multiscale_output else 1 + for i in range(num_out_branches): + fuse_layer = [] + for j in range(num_branches): + if j > i: + fuse_layer.append( + nn.Sequential( + build_conv_layer( + self.conv_cfg, + in_channels[j], + in_channels[i], + kernel_size=1, + stride=1, + padding=0, + bias=False), + build_norm_layer(self.norm_cfg, in_channels[i])[1], + # we set align_corners=False for HRNet + Upsample( + scale_factor=2**(j - i), + mode='bilinear', + align_corners=False))) + elif j == i: + fuse_layer.append(None) + else: + conv_downsamples = [] + for k in range(i - j): + if k == i - j - 1: + conv_downsamples.append( + nn.Sequential( + build_conv_layer( + self.conv_cfg, + in_channels[j], + in_channels[i], + kernel_size=3, + stride=2, + padding=1, + bias=False), + build_norm_layer(self.norm_cfg, + in_channels[i])[1])) + else: + conv_downsamples.append( + nn.Sequential( + build_conv_layer( + self.conv_cfg, + in_channels[j], + in_channels[j], + kernel_size=3, + stride=2, + padding=1, + bias=False), + build_norm_layer(self.norm_cfg, + in_channels[j])[1], + nn.ReLU(inplace=False))) + fuse_layer.append(nn.Sequential(*conv_downsamples)) + fuse_layers.append(nn.ModuleList(fuse_layer)) + + return nn.ModuleList(fuse_layers) + + def forward(self, x): + """Forward function.""" + if self.num_branches == 1: + return [self.branches[0](x[0])] + + for i in range(self.num_branches): + x[i] = self.branches[i](x[i]) + + x_fuse = [] + for i in range(len(self.fuse_layers)): + y = 0 + for j in range(self.num_branches): + if i == j: + y += x[j] + elif j > i: + y = y + resize( + self.fuse_layers[i][j](x[j]), + size=x[i].shape[2:], + mode='bilinear', + align_corners=False) + else: + y += self.fuse_layers[i][j](x[j]) + x_fuse.append(self.relu(y)) + return x_fuse + + +@BACKBONES.register_module() +class HRNet(nn.Module): + """HRNet backbone. + + High-Resolution Representations for Labeling Pixels and Regions + arXiv: https://arxiv.org/abs/1904.04514 + + Args: + extra (dict): detailed configuration for each stage of HRNet. + in_channels (int): Number of input image channels. Normally 3. + conv_cfg (dict): dictionary to construct and config conv layer. + norm_cfg (dict): dictionary to construct and config norm layer. + norm_eval (bool): Whether to set norm layers to eval mode, namely, + freeze running stats (mean and var). Note: Effect on Batch Norm + and its variants only. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. + zero_init_residual (bool): whether to use zero init for last norm layer + in resblocks to let them behave as identity. + + Example: + >>> from mmseg.models import HRNet + >>> import torch + >>> extra = dict( + >>> stage1=dict( + >>> num_modules=1, + >>> num_branches=1, + >>> block='BOTTLENECK', + >>> num_blocks=(4, ), + >>> num_channels=(64, )), + >>> stage2=dict( + >>> num_modules=1, + >>> num_branches=2, + >>> block='BASIC', + >>> num_blocks=(4, 4), + >>> num_channels=(32, 64)), + >>> stage3=dict( + >>> num_modules=4, + >>> num_branches=3, + >>> block='BASIC', + >>> num_blocks=(4, 4, 4), + >>> num_channels=(32, 64, 128)), + >>> stage4=dict( + >>> num_modules=3, + >>> num_branches=4, + >>> block='BASIC', + >>> num_blocks=(4, 4, 4, 4), + >>> num_channels=(32, 64, 128, 256))) + >>> self = HRNet(extra, in_channels=1) + >>> self.eval() + >>> inputs = torch.rand(1, 1, 32, 32) + >>> level_outputs = self.forward(inputs) + >>> for level_out in level_outputs: + ... print(tuple(level_out.shape)) + (1, 32, 8, 8) + (1, 64, 4, 4) + (1, 128, 2, 2) + (1, 256, 1, 1) + """ + + blocks_dict = {'BASIC': BasicBlock, 'BOTTLENECK': Bottleneck} + + def __init__(self, + extra, + in_channels=3, + conv_cfg=None, + norm_cfg=dict(type='BN', requires_grad=True), + norm_eval=False, + with_cp=False, + zero_init_residual=False): + super(HRNet, self).__init__() + self.extra = extra + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.norm_eval = norm_eval + self.with_cp = with_cp + self.zero_init_residual = zero_init_residual + + # stem net + self.norm1_name, norm1 = build_norm_layer(self.norm_cfg, 64, postfix=1) + self.norm2_name, norm2 = build_norm_layer(self.norm_cfg, 64, postfix=2) + + self.conv1 = build_conv_layer( + self.conv_cfg, + in_channels, + 64, + kernel_size=3, + stride=2, + padding=1, + bias=False) + + self.add_module(self.norm1_name, norm1) + self.conv2 = build_conv_layer( + self.conv_cfg, + 64, + 64, + kernel_size=3, + stride=2, + padding=1, + bias=False) + + self.add_module(self.norm2_name, norm2) + self.relu = nn.ReLU(inplace=True) + + # stage 1 + self.stage1_cfg = self.extra['stage1'] + num_channels = self.stage1_cfg['num_channels'][0] + block_type = self.stage1_cfg['block'] + num_blocks = self.stage1_cfg['num_blocks'][0] + + block = self.blocks_dict[block_type] + stage1_out_channels = num_channels * block.expansion + self.layer1 = self._make_layer(block, 64, num_channels, num_blocks) + + # stage 2 + self.stage2_cfg = self.extra['stage2'] + num_channels = self.stage2_cfg['num_channels'] + block_type = self.stage2_cfg['block'] + + block = self.blocks_dict[block_type] + num_channels = [channel * block.expansion for channel in num_channels] + self.transition1 = self._make_transition_layer([stage1_out_channels], + num_channels) + self.stage2, pre_stage_channels = self._make_stage( + self.stage2_cfg, num_channels) + + # stage 3 + self.stage3_cfg = self.extra['stage3'] + num_channels = self.stage3_cfg['num_channels'] + block_type = self.stage3_cfg['block'] + + block = self.blocks_dict[block_type] + num_channels = [channel * block.expansion for channel in num_channels] + self.transition2 = self._make_transition_layer(pre_stage_channels, + num_channels) + self.stage3, pre_stage_channels = self._make_stage( + self.stage3_cfg, num_channels) + + # stage 4 + self.stage4_cfg = self.extra['stage4'] + num_channels = self.stage4_cfg['num_channels'] + block_type = self.stage4_cfg['block'] + + block = self.blocks_dict[block_type] + num_channels = [channel * block.expansion for channel in num_channels] + self.transition3 = self._make_transition_layer(pre_stage_channels, + num_channels) + self.stage4, pre_stage_channels = self._make_stage( + self.stage4_cfg, num_channels) + + @property + def norm1(self): + """nn.Module: the normalization layer named "norm1" """ + return getattr(self, self.norm1_name) + + @property + def norm2(self): + """nn.Module: the normalization layer named "norm2" """ + return getattr(self, self.norm2_name) + + def _make_transition_layer(self, num_channels_pre_layer, + num_channels_cur_layer): + """Make transition layer.""" + num_branches_cur = len(num_channels_cur_layer) + num_branches_pre = len(num_channels_pre_layer) + + transition_layers = [] + for i in range(num_branches_cur): + if i < num_branches_pre: + if num_channels_cur_layer[i] != num_channels_pre_layer[i]: + transition_layers.append( + nn.Sequential( + build_conv_layer( + self.conv_cfg, + num_channels_pre_layer[i], + num_channels_cur_layer[i], + kernel_size=3, + stride=1, + padding=1, + bias=False), + build_norm_layer(self.norm_cfg, + num_channels_cur_layer[i])[1], + nn.ReLU(inplace=True))) + else: + transition_layers.append(None) + else: + conv_downsamples = [] + for j in range(i + 1 - num_branches_pre): + in_channels = num_channels_pre_layer[-1] + out_channels = num_channels_cur_layer[i] \ + if j == i - num_branches_pre else in_channels + conv_downsamples.append( + nn.Sequential( + build_conv_layer( + self.conv_cfg, + in_channels, + out_channels, + kernel_size=3, + stride=2, + padding=1, + bias=False), + build_norm_layer(self.norm_cfg, out_channels)[1], + nn.ReLU(inplace=True))) + transition_layers.append(nn.Sequential(*conv_downsamples)) + + return nn.ModuleList(transition_layers) + + def _make_layer(self, block, inplanes, planes, blocks, stride=1): + """Make each layer.""" + downsample = None + if stride != 1 or inplanes != planes * block.expansion: + downsample = nn.Sequential( + build_conv_layer( + self.conv_cfg, + inplanes, + planes * block.expansion, + kernel_size=1, + stride=stride, + bias=False), + build_norm_layer(self.norm_cfg, planes * block.expansion)[1]) + + layers = [] + layers.append( + block( + inplanes, + planes, + stride, + downsample=downsample, + with_cp=self.with_cp, + norm_cfg=self.norm_cfg, + conv_cfg=self.conv_cfg)) + inplanes = planes * block.expansion + for i in range(1, blocks): + layers.append( + block( + inplanes, + planes, + with_cp=self.with_cp, + norm_cfg=self.norm_cfg, + conv_cfg=self.conv_cfg)) + + return nn.Sequential(*layers) + + def _make_stage(self, layer_config, in_channels, multiscale_output=True): + """Make each stage.""" + num_modules = layer_config['num_modules'] + num_branches = layer_config['num_branches'] + num_blocks = layer_config['num_blocks'] + num_channels = layer_config['num_channels'] + block = self.blocks_dict[layer_config['block']] + + hr_modules = [] + for i in range(num_modules): + # multi_scale_output is only used for the last module + if not multiscale_output and i == num_modules - 1: + reset_multiscale_output = False + else: + reset_multiscale_output = True + + hr_modules.append( + HRModule( + num_branches, + block, + num_blocks, + in_channels, + num_channels, + reset_multiscale_output, + with_cp=self.with_cp, + norm_cfg=self.norm_cfg, + conv_cfg=self.conv_cfg)) + + return nn.Sequential(*hr_modules), in_channels + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + if isinstance(pretrained, str): + logger = get_root_logger() + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + for m in self.modules(): + if isinstance(m, nn.Conv2d): + kaiming_init(m) + elif isinstance(m, (_BatchNorm, nn.GroupNorm)): + constant_init(m, 1) + + if self.zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + constant_init(m.norm3, 0) + elif isinstance(m, BasicBlock): + constant_init(m.norm2, 0) + else: + raise TypeError('pretrained must be a str or None') + + def forward(self, x): + """Forward function.""" + + x = self.conv1(x) + x = self.norm1(x) + x = self.relu(x) + x = self.conv2(x) + x = self.norm2(x) + x = self.relu(x) + x = self.layer1(x) + + x_list = [] + for i in range(self.stage2_cfg['num_branches']): + if self.transition1[i] is not None: + x_list.append(self.transition1[i](x)) + else: + x_list.append(x) + y_list = self.stage2(x_list) + + x_list = [] + for i in range(self.stage3_cfg['num_branches']): + if self.transition2[i] is not None: + x_list.append(self.transition2[i](y_list[-1])) + else: + x_list.append(y_list[i]) + y_list = self.stage3(x_list) + + x_list = [] + for i in range(self.stage4_cfg['num_branches']): + if self.transition3[i] is not None: + x_list.append(self.transition3[i](y_list[-1])) + else: + x_list.append(y_list[i]) + y_list = self.stage4(x_list) + + return y_list + + def train(self, mode=True): + """Convert the model into training mode will keeping the normalization + layer freezed.""" + super(HRNet, self).train(mode) + if mode and self.norm_eval: + for m in self.modules(): + # trick: eval have effect on BatchNorm only + if isinstance(m, _BatchNorm): + m.eval() diff --git a/segmentation/mmseg/models/backbones/litv2.py b/segmentation/mmseg/models/backbones/litv2.py new file mode 100644 index 0000000..f29c81e --- /dev/null +++ b/segmentation/mmseg/models/backbones/litv2.py @@ -0,0 +1,641 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as checkpoint +import numpy as np +from timm.models.layers import DropPath, to_2tuple, trunc_normal_ + +from mmcv_custom import load_checkpoint +from mmseg.utils import get_root_logger +from ..builder import BACKBONES +from mm_modules.DCN.modules.deform_conv2d import DeformConv2dPack +import math + +class DWConv(nn.Module): + def __init__(self, dim=768): + super(DWConv, self).__init__() + self.dwconv = nn.Conv2d(dim, dim, 3, 1, 1, bias=True, groups=dim) + + def forward(self, x): + x = self.dwconv(x) + x = x.flatten(2).transpose(1, 2) + return x + +class DWMlp(nn.Module): + def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0., linear=False): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.dwconv = DWConv(hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + self.linear = linear + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + elif isinstance(m, nn.Conv2d): + fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + fan_out //= m.groups + m.weight.data.normal_(0, math.sqrt(2.0 / fan_out)) + if m.bias is not None: + m.bias.data.zero_() + + def forward(self, x, H, W): + x = self.fc1(x) + B, N, C = x.shape + x = x.transpose(1, 2).view(B, C, H, W) + x = self.dwconv(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + +class Mlp(nn.Module): + """ Multilayer perceptron.""" + + def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0., sr_ratio=1, alpha=0.5): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + +class PatchEmbed(nn.Module): + """ Image to Patch Embedding + + Args: + patch_size (int): Patch token size. Default: 4. + in_chans (int): Number of input image channels. Default: 3. + embed_dim (int): Number of linear projection output channels. Default: 96. + norm_layer (nn.Module, optional): Normalization layer. Default: None + """ + + def __init__(self, patch_size=4, in_chans=3, embed_dim=96, norm_layer=None): + super().__init__() + patch_size = to_2tuple(patch_size) + self.patch_size = patch_size + + self.in_chans = in_chans + self.embed_dim = embed_dim + + self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size) + if norm_layer is not None: + self.norm = norm_layer(embed_dim) + else: + self.norm = None + + def forward(self, x): + """Forward function.""" + # padding + _, _, H, W = x.size() + if W % self.patch_size[1] != 0: + x = F.pad(x, (0, self.patch_size[1] - W % self.patch_size[1])) + if H % self.patch_size[0] != 0: + x = F.pad(x, (0, 0, 0, self.patch_size[0] - H % self.patch_size[0])) + + x = self.proj(x) # B C Wh Ww + if self.norm is not None: + Wh, Ww = x.size(2), x.size(3) + x = x.flatten(2).transpose(1, 2) + x = self.norm(x) + x = x.transpose(1, 2).view(-1, self.embed_dim, Wh, Ww) + + return x + + +class HiLo(nn.Module): + """ + HiLo Attention + + Link: https://arxiv.org/abs/2205.13213 + """ + def __init__(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0., window_size=2, alpha=0.5): + super().__init__() + assert dim % num_heads == 0, f"dim {dim} should be divided by num_heads {num_heads}." + head_dim = int(dim/num_heads) + + self.dim = dim + # self-attention heads in Lo-Fi + self.l_heads = int(num_heads * alpha) + # token dimension in Lo-Fi + self.l_dim = self.l_heads * head_dim + + # self-attention heads in Hi-Fi + self.h_heads = num_heads - self.l_heads + # token dimension in Hi-Fi + self.h_dim = self.h_heads * head_dim + + # local window size. The `s` in our paper. + self.ws = window_size + + if self.ws == 1: + # ws == 1 is equal to a standard multi-head self-attention + self.h_heads = 0 + self.h_dim = 0 + self.l_heads = num_heads + self.l_dim = dim + + self.scale = qk_scale or head_dim ** -0.5 + + # Low frequence attention (Lo-Fi) + if self.l_heads > 0: + if self.ws != 1: + self.sr = nn.AvgPool2d(kernel_size=window_size, stride=window_size) + self.l_q = nn.Linear(self.dim, self.l_dim, bias=qkv_bias) + self.l_kv = nn.Linear(self.dim, self.l_dim * 2, bias=qkv_bias) + self.l_proj = nn.Linear(self.l_dim, self.l_dim) + + # High frequence attention (Hi-Fi) + if self.h_heads > 0: + self.h_qkv = nn.Linear(self.dim, self.h_dim * 3, bias=qkv_bias) + self.h_proj = nn.Linear(self.h_dim, self.h_dim) + + + def hifi(self, x): + B, H, W, C = x.shape + h_group, w_group = H // self.ws, W // self.ws + total_groups = h_group * w_group + + x = x.reshape(B, h_group, self.ws, w_group, self.ws, C).transpose(2, 3) + + qkv = self.h_qkv(x).reshape(B, total_groups, -1, 3, self.h_heads, self.h_dim // self.h_heads).permute(3, 0, 1, 4, 2, 5) + q, k, v = qkv[0], qkv[1], qkv[2] # B, hw, n_head, ws*ws, head_dim + attn = (q @ k.transpose(-2, -1)) * self.scale # B, hw, n_head, ws*ws, ws*ws + attn = attn.softmax(dim=-1) + attn = (attn @ v).transpose(2, 3).reshape(B, h_group, w_group, self.ws, self.ws, self.h_dim) + x = attn.transpose(2, 3).reshape(B, h_group * self.ws, w_group * self.ws, self.h_dim) + x = self.h_proj(x) + return x + + def lofi(self, x): + B, H, W, C = x.shape + + q = self.l_q(x).reshape(B, H * W, self.l_heads, self.l_dim // self.l_heads).permute(0, 2, 1, 3) + + if self.ws > 1: + x_ = x.permute(0, 3, 1, 2) + x_ = self.sr(x_).reshape(B, C, -1).permute(0, 2, 1) + kv = self.l_kv(x_).reshape(B, -1, 2, self.l_heads, self.l_dim // self.l_heads).permute(2, 0, 3, 1, 4) + else: + kv = self.l_kv(x).reshape(B, -1, 2, self.l_heads, self.l_dim // self.l_heads).permute(2, 0, 3, 1, 4) + k, v = kv[0], kv[1] + + attn = (q @ k.transpose(-2, -1)) * self.scale + attn = attn.softmax(dim=-1) + + x = (attn @ v).transpose(1, 2).reshape(B, H, W, self.l_dim) + x = self.l_proj(x) + return x + + def forward(self, x, H, W): + B, N, C = x.shape + + x = x.reshape(B, H, W, C) + + # pad feature maps to multiples of window size + pad_l = pad_t = 0 + pad_r = (self.ws - W % self.ws) % self.ws + pad_b = (self.ws - H % self.ws) % self.ws + x = F.pad(x, (0, 0, pad_l, pad_r, pad_t, pad_b)) + + if self.h_heads == 0: + x = self.lofi(x) + if pad_r > 0 or pad_b > 0: + x = x[:, :H, :W, :] + return x.reshape(B, N, C) + + if self.l_heads == 0: + x = self.hifi(x) + if pad_r > 0 or pad_b > 0: + x = x[:, :H, :W, :] + return x.reshape(B, N, C) + + hifi_out = self.hifi(x) + lofi_out = self.lofi(x) + if pad_r > 0 or pad_b > 0: + x = torch.cat((hifi_out[:, :H, :W, :], lofi_out[:, :H, :W, :]), dim=-1) + else: + x = torch.cat((hifi_out, lofi_out), dim=-1) + + x = x.reshape(B, N, C) + return x + + def flops(self, N): + H = int(N ** 0.5) + Hp = Wp = self.ws * math.ceil(H / self.ws) + + Np = Hp * Wp + + # For Hi-Fi + # qkv + hifi_flops = Np * self.dim * self.h_dim * 3 + nW = Np / self.ws / self.ws + window_len = self.ws * self.ws + # q @ k and attn @ v + window_flops = window_len * window_len * self.h_dim * 2 + hifi_flops += nW * window_flops + # projection + hifi_flops += Np * self.h_dim * self.h_dim + + # for Lo-Fi + # q + lofi_flops = Np * self.dim * self.l_dim + # H = int(Np ** 0.5) + kv_len = (Hp // self.ws) ** 2 + # k, v + lofi_flops += kv_len * self.dim * self.l_dim * 2 + # q @ k and attn @ v + lofi_flops += Np * self.l_dim * kv_len * 2 + # projection + lofi_flops += Np * self.l_dim * self.l_dim + + return hifi_flops + lofi_flops + + +class Block(nn.Module): + """ Transformer Block. + + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads. + window_size (int): Window size. + shift_size (int): Shift size for SW-MSA. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float, optional): Stochastic depth rate. Default: 0.0 + act_layer (nn.Module, optional): Activation layer. Default: nn.GELU + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__(self, dim, num_heads, input_resolution, window_size=7, shift_size=0, + mlp_ratio=4., qkv_bias=True, qk_scale=None, drop=0., attn_drop=0., drop_path=0., + act_layer=nn.GELU, norm_layer=nn.LayerNorm, local_ws=1, alpha=0.5): + super().__init__() + self.dim = dim + self.num_heads = num_heads + self.input_resolution = input_resolution + self.shift_size = shift_size + self.mlp_ratio = mlp_ratio + + self.norm1 = norm_layer(dim) + self.attn = HiLo(dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, + attn_drop=attn_drop, proj_drop=drop, window_size=local_ws, alpha=alpha) + + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = DWMlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) + + + def forward(self, x, H, W): + x = x + self.drop_path(self.attn(self.norm1(x), H, W)) + x = x + self.drop_path(self.mlp(self.norm2(x), H, W)) + return x + + +class ConvFFNBlock(nn.Module): + """ Convolutional FFN Block. + + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads. + window_size (int): Window size. + shift_size (int): Shift size for SW-MSA. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float, optional): Stochastic depth rate. Default: 0.0 + act_layer (nn.Module, optional): Activation layer. Default: nn.GELU + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__(self, dim, num_heads, input_resolution, window_size=7, shift_size=0, + mlp_ratio=4., qkv_bias=True, qk_scale=None, drop=0., attn_drop=0., drop_path=0., + act_layer=nn.GELU, norm_layer=nn.LayerNorm, local_ws=1, alpha=0.5): + super().__init__() + self.dim = dim + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = DWMlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) + + + def forward(self, x, H, W): + x = x + self.drop_path(self.mlp(self.norm2(x), H, W)) + return x + + +class DTM(nn.Module): + r""" Deformable Token Merging. + + Link: https://arxiv.org/abs/2105.14217 + + Args: + input_resolution (tuple[int]): Resolution of input feature. + dim (int): Number of input channels. + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__(self, input_resolution, dim, norm_layer=nn.LayerNorm): + super().__init__() + self.input_resolution = input_resolution + self.dim = dim + self.kernel_size = 2 + self.stride = 2 + self.padding = 0 + self.c_in = dim + self.c_out = dim*2 + self.dconv = DeformConv2dPack(dim, dim*2, kernel_size=2, stride=2, padding=0) + self.norm_layer = nn.BatchNorm2d(dim*2) + self.act_layer = nn.GELU() + + def forward(self, x, H, W, return_offset=False): + """ + x: B, H*W, C + """ + B, L, C = x.shape + x = x.reshape(B, H, W, C).permute(0, 3, 1, 2).contiguous() + x, offset = self.dconv(x, return_offset=False) + _, _, new_H, new_W = x.shape + x = self.act_layer(self.norm_layer(x)).flatten(2).transpose(1, 2) + if return_offset: + return x, new_H, new_W, offset + else: + return x, new_H, new_W + + def extra_repr(self) -> str: + return f"input_resolution={self.input_resolution}, dim={self.dim}" + + +class LITLayer(nn.Module): + """ A basic Swin Transformer layer for one stage. + + Args: + dim (int): Number of feature channels + depth (int): Depths of this stage. + num_heads (int): Number of attention head. + window_size (int): Local window size. Default: 7. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None + use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. + """ + + def __init__(self, + dim, + depth, + num_heads, + input_resolution, + window_size=7, + mlp_ratio=4., + qkv_bias=True, + qk_scale=None, + drop=0., + attn_drop=0., + drop_path=0., + norm_layer=nn.LayerNorm, + downsample=None, + use_checkpoint=False, + has_msa=True, + local_ws=1, + alpha=0.5 + ): + super().__init__() + self.window_size = window_size + self.shift_size = window_size // 2 + self.depth = depth + self.use_checkpoint = use_checkpoint + self.input_resolution = input_resolution + # build blocks + self.has_msa = has_msa + block = Block if has_msa else ConvFFNBlock + self.blocks = nn.ModuleList([ + block( + dim=dim, + num_heads=num_heads, + input_resolution=input_resolution, + window_size=window_size, + shift_size=0 if (i % 2 == 0) else window_size // 2, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop, + attn_drop=attn_drop, + drop_path=drop_path[i] if isinstance(drop_path, list) else drop_path, + norm_layer=norm_layer, + local_ws=local_ws, + alpha=alpha + ) + for i in range(depth)]) + + # patch merging layer + if downsample is not None: + self.downsample = downsample(input_resolution, dim=dim, norm_layer=norm_layer) + else: + self.downsample = None + + def forward(self, x, H, W): + """ Forward function. + + Args: + x: Input feature, tensor size (B, H*W, C). + H, W: Spatial resolution of the input feature. + """ + for i, blk in enumerate(self.blocks): + if self.use_checkpoint: + x = checkpoint.checkpoint(blk, x, H, W) + else: + x = blk(x, H, W) + if self.downsample is not None: + x_down, Wh, Ww = self.downsample(x, H, W) + return x, H, W, x_down, Wh, Ww + else: + return x, H, W, x, H, W + +@BACKBONES.register_module() +class LITv2(nn.Module): + def __init__(self, + pretrain_img_size=224, + patch_size=4, + in_chans=3, + embed_dim=96, + depths=[2, 2, 6, 2], + num_heads=[3, 6, 12, 24], + window_size=7, + mlp_ratio=4., + qkv_bias=True, + qk_scale=None, + drop_rate=0., + attn_drop_rate=0., + drop_path_rate=0.2, + norm_layer=nn.LayerNorm, + ape=False, + patch_norm=True, + out_indices=(0, 1, 2, 3), + frozen_stages=-1, + use_checkpoint=False, + has_msa=[0, 0, 1, 1], + alpha=0.5, + local_ws=[0, 0, 2, 1] + ): + super().__init__() + + # new from v2 + self.local_ws = local_ws + self.alpha = alpha + self.num_heads = num_heads + + # self.fp16_enabled = True + self.pretrain_img_size = pretrain_img_size + self.num_layers = len(depths) + self.embed_dim = embed_dim + self.ape = ape + self.patch_norm = patch_norm + self.out_indices = out_indices + self.frozen_stages = frozen_stages + self.has_msa = has_msa + # split image into non-overlapping patches + self.patch_embed = PatchEmbed( + patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim, + norm_layer=norm_layer if self.patch_norm else None) + + # absolute position embedding + input_resolution = [512 // patch_size, 512 // patch_size] + + # stochastic depth + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))] # stochastic depth decay rule + + # build layers + self.layers = nn.ModuleList() + for i_layer in range(self.num_layers): + layer = LITLayer( + dim=int(embed_dim * 2 ** i_layer), + depth=depths[i_layer], + num_heads=num_heads[i_layer], + input_resolution=(input_resolution[0] // (2 ** i_layer), + input_resolution[1] // (2 ** i_layer)), + window_size=window_size, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop_rate, + attn_drop=attn_drop_rate, + drop_path=dpr[sum(depths[:i_layer]):sum(depths[:i_layer + 1])], + norm_layer=norm_layer, + downsample=DTM if (i_layer < self.num_layers - 1) else None, + use_checkpoint=use_checkpoint, + has_msa=self.has_msa[i_layer] == 1, + local_ws=self.local_ws[i_layer], + alpha=alpha + ) + self.layers.append(layer) + + num_features = [int(embed_dim * 2 ** i) for i in range(self.num_layers)] + self.num_features = num_features + + # add a norm layer for each output + for i_layer in out_indices: + layer = norm_layer(num_features[i_layer]) + layer_name = f'norm{i_layer}' + self.add_module(layer_name, layer) + + self._freeze_stages() + + + def _freeze_stages(self): + if self.frozen_stages >= 0: + self.patch_embed.eval() + for param in self.patch_embed.parameters(): + param.requires_grad = False + + if self.frozen_stages >= 1 and self.ape: + self.absolute_pos_embed.requires_grad = False + + if self.frozen_stages >= 2: + self.pos_drop.eval() + for i in range(0, self.frozen_stages - 1): + m = self.layers[i] + m.eval() + for param in m.parameters(): + param.requires_grad = False + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + + def _init_weights(m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + if isinstance(pretrained, str): + self.apply(_init_weights) + logger = get_root_logger() + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + self.apply(_init_weights) + else: + raise TypeError('pretrained must be a str or None') + + def forward(self, x): + """Forward function.""" + + x = self.patch_embed(x) + + Wh, Ww = x.size(2), x.size(3) + x = x.flatten(2).transpose(1, 2) + outs = [] + for i in range(self.num_layers): + layer = self.layers[i] + x_out, H, W, x, Wh, Ww = layer(x, Wh, Ww) + + if i in self.out_indices: + norm_layer = getattr(self, f'norm{i}') + x_out = norm_layer(x_out) + out = x_out.view(-1, H, W, self.num_features[i]).permute(0, 3, 1, 2).contiguous() + outs.append(out) + + return tuple(outs) + + def train(self, mode=True): + """Convert the model into training mode while keep layers freezed.""" + super(LITv2, self).train(mode) + self._freeze_stages() \ No newline at end of file diff --git a/segmentation/mmseg/models/backbones/mobilenet_v2.py b/segmentation/mmseg/models/backbones/mobilenet_v2.py new file mode 100644 index 0000000..9ab628e --- /dev/null +++ b/segmentation/mmseg/models/backbones/mobilenet_v2.py @@ -0,0 +1,180 @@ +import logging + +import torch.nn as nn +from mmcv.cnn import ConvModule, constant_init, kaiming_init +from mmcv.runner import load_checkpoint +from torch.nn.modules.batchnorm import _BatchNorm + +from ..builder import BACKBONES +from ..utils import InvertedResidual, make_divisible + + +@BACKBONES.register_module() +class MobileNetV2(nn.Module): + """MobileNetV2 backbone. + + Args: + widen_factor (float): Width multiplier, multiply number of + channels in each layer by this amount. Default: 1.0. + strides (Sequence[int], optional): Strides of the first block of each + layer. If not specified, default config in ``arch_setting`` will + be used. + dilations (Sequence[int]): Dilation of each layer. + out_indices (None or Sequence[int]): Output from which stages. + Default: (7, ). + frozen_stages (int): Stages to be frozen (all param fixed). + Default: -1, which means not freezing any parameters. + conv_cfg (dict): Config dict for convolution layer. + Default: None, which means using conv2d. + norm_cfg (dict): Config dict for normalization layer. + Default: dict(type='BN'). + act_cfg (dict): Config dict for activation layer. + Default: dict(type='ReLU6'). + norm_eval (bool): Whether to set norm layers to eval mode, namely, + freeze running stats (mean and var). Note: Effect on Batch Norm + and its variants only. Default: False. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + """ + + # Parameters to build layers. 3 parameters are needed to construct a + # layer, from left to right: expand_ratio, channel, num_blocks. + arch_settings = [[1, 16, 1], [6, 24, 2], [6, 32, 3], [6, 64, 4], + [6, 96, 3], [6, 160, 3], [6, 320, 1]] + + def __init__(self, + widen_factor=1., + strides=(1, 2, 2, 2, 1, 2, 1), + dilations=(1, 1, 1, 1, 1, 1, 1), + out_indices=(1, 2, 4, 6), + frozen_stages=-1, + conv_cfg=None, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU6'), + norm_eval=False, + with_cp=False): + super(MobileNetV2, self).__init__() + self.widen_factor = widen_factor + self.strides = strides + self.dilations = dilations + assert len(strides) == len(dilations) == len(self.arch_settings) + self.out_indices = out_indices + for index in out_indices: + if index not in range(0, 7): + raise ValueError('the item in out_indices must in ' + f'range(0, 7). But received {index}') + + if frozen_stages not in range(-1, 7): + raise ValueError('frozen_stages must be in range(-1, 7). ' + f'But received {frozen_stages}') + self.out_indices = out_indices + self.frozen_stages = frozen_stages + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + self.norm_eval = norm_eval + self.with_cp = with_cp + + self.in_channels = make_divisible(32 * widen_factor, 8) + + self.conv1 = ConvModule( + in_channels=3, + out_channels=self.in_channels, + kernel_size=3, + stride=2, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + self.layers = [] + + for i, layer_cfg in enumerate(self.arch_settings): + expand_ratio, channel, num_blocks = layer_cfg + stride = self.strides[i] + dilation = self.dilations[i] + out_channels = make_divisible(channel * widen_factor, 8) + inverted_res_layer = self.make_layer( + out_channels=out_channels, + num_blocks=num_blocks, + stride=stride, + dilation=dilation, + expand_ratio=expand_ratio) + layer_name = f'layer{i + 1}' + self.add_module(layer_name, inverted_res_layer) + self.layers.append(layer_name) + + def make_layer(self, out_channels, num_blocks, stride, dilation, + expand_ratio): + """Stack InvertedResidual blocks to build a layer for MobileNetV2. + + Args: + out_channels (int): out_channels of block. + num_blocks (int): Number of blocks. + stride (int): Stride of the first block. + dilation (int): Dilation of the first block. + expand_ratio (int): Expand the number of channels of the + hidden layer in InvertedResidual by this ratio. + """ + layers = [] + for i in range(num_blocks): + layers.append( + InvertedResidual( + self.in_channels, + out_channels, + stride if i == 0 else 1, + expand_ratio=expand_ratio, + dilation=dilation if i == 0 else 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg, + with_cp=self.with_cp)) + self.in_channels = out_channels + + return nn.Sequential(*layers) + + def init_weights(self, pretrained=None): + if isinstance(pretrained, str): + logger = logging.getLogger() + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + for m in self.modules(): + if isinstance(m, nn.Conv2d): + kaiming_init(m) + elif isinstance(m, (_BatchNorm, nn.GroupNorm)): + constant_init(m, 1) + else: + raise TypeError('pretrained must be a str or None') + + def forward(self, x): + x = self.conv1(x) + + outs = [] + for i, layer_name in enumerate(self.layers): + layer = getattr(self, layer_name) + x = layer(x) + if i in self.out_indices: + outs.append(x) + + if len(outs) == 1: + return outs[0] + else: + return tuple(outs) + + def _freeze_stages(self): + if self.frozen_stages >= 0: + for param in self.conv1.parameters(): + param.requires_grad = False + for i in range(1, self.frozen_stages + 1): + layer = getattr(self, f'layer{i}') + layer.eval() + for param in layer.parameters(): + param.requires_grad = False + + def train(self, mode=True): + super(MobileNetV2, self).train(mode) + self._freeze_stages() + if mode and self.norm_eval: + for m in self.modules(): + if isinstance(m, _BatchNorm): + m.eval() diff --git a/segmentation/mmseg/models/backbones/mobilenet_v3.py b/segmentation/mmseg/models/backbones/mobilenet_v3.py new file mode 100644 index 0000000..f2e9a0c --- /dev/null +++ b/segmentation/mmseg/models/backbones/mobilenet_v3.py @@ -0,0 +1,255 @@ +import logging + +import mmcv +import torch.nn as nn +from mmcv.cnn import ConvModule, constant_init, kaiming_init +from mmcv.cnn.bricks import Conv2dAdaptivePadding +from mmcv.runner import load_checkpoint +from torch.nn.modules.batchnorm import _BatchNorm + +from ..builder import BACKBONES +from ..utils import InvertedResidualV3 as InvertedResidual + + +@BACKBONES.register_module() +class MobileNetV3(nn.Module): + """MobileNetV3 backbone. + + This backbone is the improved implementation of `Searching for MobileNetV3 + `_. + + Args: + arch (str): Architecture of mobilnetv3, from {'small', 'large'}. + Default: 'small'. + conv_cfg (dict): Config dict for convolution layer. + Default: None, which means using conv2d. + norm_cfg (dict): Config dict for normalization layer. + Default: dict(type='BN'). + out_indices (tuple[int]): Output from which layer. + Default: (0, 1, 12). + frozen_stages (int): Stages to be frozen (all param fixed). + Default: -1, which means not freezing any parameters. + norm_eval (bool): Whether to set norm layers to eval mode, namely, + freeze running stats (mean and var). Note: Effect on Batch Norm + and its variants only. Default: False. + with_cp (bool): Use checkpoint or not. Using checkpoint will save + some memory while slowing down the training speed. + Default: False. + """ + # Parameters to build each block: + # [kernel size, mid channels, out channels, with_se, act type, stride] + arch_settings = { + 'small': [[3, 16, 16, True, 'ReLU', 2], # block0 layer1 os=4 + [3, 72, 24, False, 'ReLU', 2], # block1 layer2 os=8 + [3, 88, 24, False, 'ReLU', 1], + [5, 96, 40, True, 'HSwish', 2], # block2 layer4 os=16 + [5, 240, 40, True, 'HSwish', 1], + [5, 240, 40, True, 'HSwish', 1], + [5, 120, 48, True, 'HSwish', 1], # block3 layer7 os=16 + [5, 144, 48, True, 'HSwish', 1], + [5, 288, 96, True, 'HSwish', 2], # block4 layer9 os=32 + [5, 576, 96, True, 'HSwish', 1], + [5, 576, 96, True, 'HSwish', 1]], + 'large': [[3, 16, 16, False, 'ReLU', 1], # block0 layer1 os=2 + [3, 64, 24, False, 'ReLU', 2], # block1 layer2 os=4 + [3, 72, 24, False, 'ReLU', 1], + [5, 72, 40, True, 'ReLU', 2], # block2 layer4 os=8 + [5, 120, 40, True, 'ReLU', 1], + [5, 120, 40, True, 'ReLU', 1], + [3, 240, 80, False, 'HSwish', 2], # block3 layer7 os=16 + [3, 200, 80, False, 'HSwish', 1], + [3, 184, 80, False, 'HSwish', 1], + [3, 184, 80, False, 'HSwish', 1], + [3, 480, 112, True, 'HSwish', 1], # block4 layer11 os=16 + [3, 672, 112, True, 'HSwish', 1], + [5, 672, 160, True, 'HSwish', 2], # block5 layer13 os=32 + [5, 960, 160, True, 'HSwish', 1], + [5, 960, 160, True, 'HSwish', 1]] + } # yapf: disable + + def __init__(self, + arch='small', + conv_cfg=None, + norm_cfg=dict(type='BN'), + out_indices=(0, 1, 12), + frozen_stages=-1, + reduction_factor=1, + norm_eval=False, + with_cp=False): + super(MobileNetV3, self).__init__() + assert arch in self.arch_settings + assert isinstance(reduction_factor, int) and reduction_factor > 0 + assert mmcv.is_tuple_of(out_indices, int) + for index in out_indices: + if index not in range(0, len(self.arch_settings[arch]) + 2): + raise ValueError( + 'the item in out_indices must in ' + f'range(0, {len(self.arch_settings[arch])+2}). ' + f'But received {index}') + + if frozen_stages not in range(-1, len(self.arch_settings[arch]) + 2): + raise ValueError('frozen_stages must be in range(-1, ' + f'{len(self.arch_settings[arch])+2}). ' + f'But received {frozen_stages}') + self.arch = arch + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.out_indices = out_indices + self.frozen_stages = frozen_stages + self.reduction_factor = reduction_factor + self.norm_eval = norm_eval + self.with_cp = with_cp + self.layers = self._make_layer() + + def _make_layer(self): + layers = [] + + # build the first layer (layer0) + in_channels = 16 + layer = ConvModule( + in_channels=3, + out_channels=in_channels, + kernel_size=3, + stride=2, + padding=1, + conv_cfg=dict(type='Conv2dAdaptivePadding'), + norm_cfg=self.norm_cfg, + act_cfg=dict(type='HSwish')) + self.add_module('layer0', layer) + layers.append('layer0') + + layer_setting = self.arch_settings[self.arch] + for i, params in enumerate(layer_setting): + (kernel_size, mid_channels, out_channels, with_se, act, + stride) = params + + if self.arch == 'large' and i >= 12 or self.arch == 'small' and \ + i >= 8: + mid_channels = mid_channels // self.reduction_factor + out_channels = out_channels // self.reduction_factor + + if with_se: + se_cfg = dict( + channels=mid_channels, + ratio=4, + act_cfg=(dict(type='ReLU'), + dict(type='HSigmoid', bias=3.0, divisor=6.0))) + else: + se_cfg = None + + layer = InvertedResidual( + in_channels=in_channels, + out_channels=out_channels, + mid_channels=mid_channels, + kernel_size=kernel_size, + stride=stride, + se_cfg=se_cfg, + with_expand_conv=(in_channels != mid_channels), + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=dict(type=act), + with_cp=self.with_cp) + in_channels = out_channels + layer_name = 'layer{}'.format(i + 1) + self.add_module(layer_name, layer) + layers.append(layer_name) + + # build the last layer + # block5 layer12 os=32 for small model + # block6 layer16 os=32 for large model + layer = ConvModule( + in_channels=in_channels, + out_channels=576 if self.arch == 'small' else 960, + kernel_size=1, + stride=1, + dilation=4, + padding=0, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=dict(type='HSwish')) + layer_name = 'layer{}'.format(len(layer_setting) + 1) + self.add_module(layer_name, layer) + layers.append(layer_name) + + # next, convert backbone MobileNetV3 to a semantic segmentation version + if self.arch == 'small': + self.layer4.depthwise_conv.conv.stride = (1, 1) + self.layer9.depthwise_conv.conv.stride = (1, 1) + for i in range(4, len(layers)): + layer = getattr(self, layers[i]) + if isinstance(layer, InvertedResidual): + modified_module = layer.depthwise_conv.conv + else: + modified_module = layer.conv + + if i < 9: + modified_module.dilation = (2, 2) + pad = 2 + else: + modified_module.dilation = (4, 4) + pad = 4 + + if not isinstance(modified_module, Conv2dAdaptivePadding): + # Adjust padding + pad *= (modified_module.kernel_size[0] - 1) // 2 + modified_module.padding = (pad, pad) + else: + self.layer7.depthwise_conv.conv.stride = (1, 1) + self.layer13.depthwise_conv.conv.stride = (1, 1) + for i in range(7, len(layers)): + layer = getattr(self, layers[i]) + if isinstance(layer, InvertedResidual): + modified_module = layer.depthwise_conv.conv + else: + modified_module = layer.conv + + if i < 13: + modified_module.dilation = (2, 2) + pad = 2 + else: + modified_module.dilation = (4, 4) + pad = 4 + + if not isinstance(modified_module, Conv2dAdaptivePadding): + # Adjust padding + pad *= (modified_module.kernel_size[0] - 1) // 2 + modified_module.padding = (pad, pad) + + return layers + + def init_weights(self, pretrained=None): + if isinstance(pretrained, str): + logger = logging.getLogger() + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + for m in self.modules(): + if isinstance(m, nn.Conv2d): + kaiming_init(m) + elif isinstance(m, nn.BatchNorm2d): + constant_init(m, 1) + else: + raise TypeError('pretrained must be a str or None') + + def forward(self, x): + outs = [] + for i, layer_name in enumerate(self.layers): + layer = getattr(self, layer_name) + x = layer(x) + if i in self.out_indices: + outs.append(x) + return outs + + def _freeze_stages(self): + for i in range(self.frozen_stages + 1): + layer = getattr(self, f'layer{i}') + layer.eval() + for param in layer.parameters(): + param.requires_grad = False + + def train(self, mode=True): + super(MobileNetV3, self).train(mode) + self._freeze_stages() + if mode and self.norm_eval: + for m in self.modules(): + if isinstance(m, _BatchNorm): + m.eval() diff --git a/segmentation/mmseg/models/backbones/resnest.py b/segmentation/mmseg/models/backbones/resnest.py new file mode 100644 index 0000000..8931dec --- /dev/null +++ b/segmentation/mmseg/models/backbones/resnest.py @@ -0,0 +1,314 @@ +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as cp +from mmcv.cnn import build_conv_layer, build_norm_layer + +from ..builder import BACKBONES +from ..utils import ResLayer +from .resnet import Bottleneck as _Bottleneck +from .resnet import ResNetV1d + + +class RSoftmax(nn.Module): + """Radix Softmax module in ``SplitAttentionConv2d``. + + Args: + radix (int): Radix of input. + groups (int): Groups of input. + """ + + def __init__(self, radix, groups): + super().__init__() + self.radix = radix + self.groups = groups + + def forward(self, x): + batch = x.size(0) + if self.radix > 1: + x = x.view(batch, self.groups, self.radix, -1).transpose(1, 2) + x = F.softmax(x, dim=1) + x = x.reshape(batch, -1) + else: + x = torch.sigmoid(x) + return x + + +class SplitAttentionConv2d(nn.Module): + """Split-Attention Conv2d in ResNeSt. + + Args: + in_channels (int): Same as nn.Conv2d. + out_channels (int): Same as nn.Conv2d. + kernel_size (int | tuple[int]): Same as nn.Conv2d. + stride (int | tuple[int]): Same as nn.Conv2d. + padding (int | tuple[int]): Same as nn.Conv2d. + dilation (int | tuple[int]): Same as nn.Conv2d. + groups (int): Same as nn.Conv2d. + radix (int): Radix of SpltAtConv2d. Default: 2 + reduction_factor (int): Reduction factor of inter_channels. Default: 4. + conv_cfg (dict): Config dict for convolution layer. Default: None, + which means using conv2d. + norm_cfg (dict): Config dict for normalization layer. Default: None. + dcn (dict): Config dict for DCN. Default: None. + """ + + def __init__(self, + in_channels, + channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + radix=2, + reduction_factor=4, + conv_cfg=None, + norm_cfg=dict(type='BN'), + dcn=None): + super(SplitAttentionConv2d, self).__init__() + inter_channels = max(in_channels * radix // reduction_factor, 32) + self.radix = radix + self.groups = groups + self.channels = channels + self.with_dcn = dcn is not None + self.dcn = dcn + fallback_on_stride = False + if self.with_dcn: + fallback_on_stride = self.dcn.pop('fallback_on_stride', False) + if self.with_dcn and not fallback_on_stride: + assert conv_cfg is None, 'conv_cfg must be None for DCN' + conv_cfg = dcn + self.conv = build_conv_layer( + conv_cfg, + in_channels, + channels * radix, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups * radix, + bias=False) + self.norm0_name, norm0 = build_norm_layer( + norm_cfg, channels * radix, postfix=0) + self.add_module(self.norm0_name, norm0) + self.relu = nn.ReLU(inplace=True) + self.fc1 = build_conv_layer( + None, channels, inter_channels, 1, groups=self.groups) + self.norm1_name, norm1 = build_norm_layer( + norm_cfg, inter_channels, postfix=1) + self.add_module(self.norm1_name, norm1) + self.fc2 = build_conv_layer( + None, inter_channels, channels * radix, 1, groups=self.groups) + self.rsoftmax = RSoftmax(radix, groups) + + @property + def norm0(self): + """nn.Module: the normalization layer named "norm0" """ + return getattr(self, self.norm0_name) + + @property + def norm1(self): + """nn.Module: the normalization layer named "norm1" """ + return getattr(self, self.norm1_name) + + def forward(self, x): + x = self.conv(x) + x = self.norm0(x) + x = self.relu(x) + + batch, rchannel = x.shape[:2] + batch = x.size(0) + if self.radix > 1: + splits = x.view(batch, self.radix, -1, *x.shape[2:]) + gap = splits.sum(dim=1) + else: + gap = x + gap = F.adaptive_avg_pool2d(gap, 1) + gap = self.fc1(gap) + + gap = self.norm1(gap) + gap = self.relu(gap) + + atten = self.fc2(gap) + atten = self.rsoftmax(atten).view(batch, -1, 1, 1) + + if self.radix > 1: + attens = atten.view(batch, self.radix, -1, *atten.shape[2:]) + out = torch.sum(attens * splits, dim=1) + else: + out = atten * x + return out.contiguous() + + +class Bottleneck(_Bottleneck): + """Bottleneck block for ResNeSt. + + Args: + inplane (int): Input planes of this block. + planes (int): Middle planes of this block. + groups (int): Groups of conv2. + width_per_group (int): Width per group of conv2. 64x4d indicates + ``groups=64, width_per_group=4`` and 32x8d indicates + ``groups=32, width_per_group=8``. + radix (int): Radix of SpltAtConv2d. Default: 2 + reduction_factor (int): Reduction factor of inter_channels in + SplitAttentionConv2d. Default: 4. + avg_down_stride (bool): Whether to use average pool for stride in + Bottleneck. Default: True. + kwargs (dict): Key word arguments for base class. + """ + expansion = 4 + + def __init__(self, + inplanes, + planes, + groups=1, + base_width=4, + base_channels=64, + radix=2, + reduction_factor=4, + avg_down_stride=True, + **kwargs): + """Bottleneck block for ResNeSt.""" + super(Bottleneck, self).__init__(inplanes, planes, **kwargs) + + if groups == 1: + width = self.planes + else: + width = math.floor(self.planes * + (base_width / base_channels)) * groups + + self.avg_down_stride = avg_down_stride and self.conv2_stride > 1 + + self.norm1_name, norm1 = build_norm_layer( + self.norm_cfg, width, postfix=1) + self.norm3_name, norm3 = build_norm_layer( + self.norm_cfg, self.planes * self.expansion, postfix=3) + + self.conv1 = build_conv_layer( + self.conv_cfg, + self.inplanes, + width, + kernel_size=1, + stride=self.conv1_stride, + bias=False) + self.add_module(self.norm1_name, norm1) + self.with_modulated_dcn = False + self.conv2 = SplitAttentionConv2d( + width, + width, + kernel_size=3, + stride=1 if self.avg_down_stride else self.conv2_stride, + padding=self.dilation, + dilation=self.dilation, + groups=groups, + radix=radix, + reduction_factor=reduction_factor, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + dcn=self.dcn) + delattr(self, self.norm2_name) + + if self.avg_down_stride: + self.avd_layer = nn.AvgPool2d(3, self.conv2_stride, padding=1) + + self.conv3 = build_conv_layer( + self.conv_cfg, + width, + self.planes * self.expansion, + kernel_size=1, + bias=False) + self.add_module(self.norm3_name, norm3) + + def forward(self, x): + + def _inner_forward(x): + identity = x + + out = self.conv1(x) + out = self.norm1(out) + out = self.relu(out) + + if self.with_plugins: + out = self.forward_plugin(out, self.after_conv1_plugin_names) + + out = self.conv2(out) + + if self.avg_down_stride: + out = self.avd_layer(out) + + if self.with_plugins: + out = self.forward_plugin(out, self.after_conv2_plugin_names) + + out = self.conv3(out) + out = self.norm3(out) + + if self.with_plugins: + out = self.forward_plugin(out, self.after_conv3_plugin_names) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + + return out + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x) + else: + out = _inner_forward(x) + + out = self.relu(out) + + return out + + +@BACKBONES.register_module() +class ResNeSt(ResNetV1d): + """ResNeSt backbone. + + Args: + groups (int): Number of groups of Bottleneck. Default: 1 + base_width (int): Base width of Bottleneck. Default: 4 + radix (int): Radix of SpltAtConv2d. Default: 2 + reduction_factor (int): Reduction factor of inter_channels in + SplitAttentionConv2d. Default: 4. + avg_down_stride (bool): Whether to use average pool for stride in + Bottleneck. Default: True. + kwargs (dict): Keyword arguments for ResNet. + """ + + arch_settings = { + 50: (Bottleneck, (3, 4, 6, 3)), + 101: (Bottleneck, (3, 4, 23, 3)), + 152: (Bottleneck, (3, 8, 36, 3)), + 200: (Bottleneck, (3, 24, 36, 3)) + } + + def __init__(self, + groups=1, + base_width=4, + radix=2, + reduction_factor=4, + avg_down_stride=True, + **kwargs): + self.groups = groups + self.base_width = base_width + self.radix = radix + self.reduction_factor = reduction_factor + self.avg_down_stride = avg_down_stride + super(ResNeSt, self).__init__(**kwargs) + + def make_res_layer(self, **kwargs): + """Pack all blocks in a stage into a ``ResLayer``.""" + return ResLayer( + groups=self.groups, + base_width=self.base_width, + base_channels=self.base_channels, + radix=self.radix, + reduction_factor=self.reduction_factor, + avg_down_stride=self.avg_down_stride, + **kwargs) diff --git a/segmentation/mmseg/models/backbones/resnet.py b/segmentation/mmseg/models/backbones/resnet.py new file mode 100644 index 0000000..f6c4c08 --- /dev/null +++ b/segmentation/mmseg/models/backbones/resnet.py @@ -0,0 +1,688 @@ +import torch.nn as nn +import torch.utils.checkpoint as cp +from mmcv.cnn import (build_conv_layer, build_norm_layer, build_plugin_layer, + constant_init, kaiming_init) +from mmcv.runner import load_checkpoint +from mmcv.utils.parrots_wrapper import _BatchNorm + +from mmseg.utils import get_root_logger +from ..builder import BACKBONES +from ..utils import ResLayer + + +class BasicBlock(nn.Module): + """Basic block for ResNet.""" + + expansion = 1 + + def __init__(self, + inplanes, + planes, + stride=1, + dilation=1, + downsample=None, + style='pytorch', + with_cp=False, + conv_cfg=None, + norm_cfg=dict(type='BN'), + dcn=None, + plugins=None): + super(BasicBlock, self).__init__() + assert dcn is None, 'Not implemented yet.' + assert plugins is None, 'Not implemented yet.' + + self.norm1_name, norm1 = build_norm_layer(norm_cfg, planes, postfix=1) + self.norm2_name, norm2 = build_norm_layer(norm_cfg, planes, postfix=2) + + self.conv1 = build_conv_layer( + conv_cfg, + inplanes, + planes, + 3, + stride=stride, + padding=dilation, + dilation=dilation, + bias=False) + self.add_module(self.norm1_name, norm1) + self.conv2 = build_conv_layer( + conv_cfg, planes, planes, 3, padding=1, bias=False) + self.add_module(self.norm2_name, norm2) + + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + self.dilation = dilation + self.with_cp = with_cp + + @property + def norm1(self): + """nn.Module: normalization layer after the first convolution layer""" + return getattr(self, self.norm1_name) + + @property + def norm2(self): + """nn.Module: normalization layer after the second convolution layer""" + return getattr(self, self.norm2_name) + + def forward(self, x): + """Forward function.""" + + def _inner_forward(x): + identity = x + + out = self.conv1(x) + out = self.norm1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.norm2(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + + return out + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x) + else: + out = _inner_forward(x) + + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + """Bottleneck block for ResNet. + + If style is "pytorch", the stride-two layer is the 3x3 conv layer, if it is + "caffe", the stride-two layer is the first 1x1 conv layer. + """ + + expansion = 4 + + def __init__(self, + inplanes, + planes, + stride=1, + dilation=1, + downsample=None, + style='pytorch', + with_cp=False, + conv_cfg=None, + norm_cfg=dict(type='BN'), + dcn=None, + plugins=None): + super(Bottleneck, self).__init__() + assert style in ['pytorch', 'caffe'] + assert dcn is None or isinstance(dcn, dict) + assert plugins is None or isinstance(plugins, list) + if plugins is not None: + allowed_position = ['after_conv1', 'after_conv2', 'after_conv3'] + assert all(p['position'] in allowed_position for p in plugins) + + self.inplanes = inplanes + self.planes = planes + self.stride = stride + self.dilation = dilation + self.style = style + self.with_cp = with_cp + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.dcn = dcn + self.with_dcn = dcn is not None + self.plugins = plugins + self.with_plugins = plugins is not None + + if self.with_plugins: + # collect plugins for conv1/conv2/conv3 + self.after_conv1_plugins = [ + plugin['cfg'] for plugin in plugins + if plugin['position'] == 'after_conv1' + ] + self.after_conv2_plugins = [ + plugin['cfg'] for plugin in plugins + if plugin['position'] == 'after_conv2' + ] + self.after_conv3_plugins = [ + plugin['cfg'] for plugin in plugins + if plugin['position'] == 'after_conv3' + ] + + if self.style == 'pytorch': + self.conv1_stride = 1 + self.conv2_stride = stride + else: + self.conv1_stride = stride + self.conv2_stride = 1 + + self.norm1_name, norm1 = build_norm_layer(norm_cfg, planes, postfix=1) + self.norm2_name, norm2 = build_norm_layer(norm_cfg, planes, postfix=2) + self.norm3_name, norm3 = build_norm_layer( + norm_cfg, planes * self.expansion, postfix=3) + + self.conv1 = build_conv_layer( + conv_cfg, + inplanes, + planes, + kernel_size=1, + stride=self.conv1_stride, + bias=False) + self.add_module(self.norm1_name, norm1) + fallback_on_stride = False + if self.with_dcn: + fallback_on_stride = dcn.pop('fallback_on_stride', False) + if not self.with_dcn or fallback_on_stride: + self.conv2 = build_conv_layer( + conv_cfg, + planes, + planes, + kernel_size=3, + stride=self.conv2_stride, + padding=dilation, + dilation=dilation, + bias=False) + else: + assert self.conv_cfg is None, 'conv_cfg must be None for DCN' + self.conv2 = build_conv_layer( + dcn, + planes, + planes, + kernel_size=3, + stride=self.conv2_stride, + padding=dilation, + dilation=dilation, + bias=False) + + self.add_module(self.norm2_name, norm2) + self.conv3 = build_conv_layer( + conv_cfg, + planes, + planes * self.expansion, + kernel_size=1, + bias=False) + self.add_module(self.norm3_name, norm3) + + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + + if self.with_plugins: + self.after_conv1_plugin_names = self.make_block_plugins( + planes, self.after_conv1_plugins) + self.after_conv2_plugin_names = self.make_block_plugins( + planes, self.after_conv2_plugins) + self.after_conv3_plugin_names = self.make_block_plugins( + planes * self.expansion, self.after_conv3_plugins) + + def make_block_plugins(self, in_channels, plugins): + """make plugins for block. + + Args: + in_channels (int): Input channels of plugin. + plugins (list[dict]): List of plugins cfg to build. + + Returns: + list[str]: List of the names of plugin. + """ + assert isinstance(plugins, list) + plugin_names = [] + for plugin in plugins: + plugin = plugin.copy() + name, layer = build_plugin_layer( + plugin, + in_channels=in_channels, + postfix=plugin.pop('postfix', '')) + assert not hasattr(self, name), f'duplicate plugin {name}' + self.add_module(name, layer) + plugin_names.append(name) + return plugin_names + + def forward_plugin(self, x, plugin_names): + """Forward function for plugins.""" + out = x + for name in plugin_names: + out = getattr(self, name)(x) + return out + + @property + def norm1(self): + """nn.Module: normalization layer after the first convolution layer""" + return getattr(self, self.norm1_name) + + @property + def norm2(self): + """nn.Module: normalization layer after the second convolution layer""" + return getattr(self, self.norm2_name) + + @property + def norm3(self): + """nn.Module: normalization layer after the third convolution layer""" + return getattr(self, self.norm3_name) + + def forward(self, x): + """Forward function.""" + + def _inner_forward(x): + identity = x + + out = self.conv1(x) + out = self.norm1(out) + out = self.relu(out) + + if self.with_plugins: + out = self.forward_plugin(out, self.after_conv1_plugin_names) + + out = self.conv2(out) + out = self.norm2(out) + out = self.relu(out) + + if self.with_plugins: + out = self.forward_plugin(out, self.after_conv2_plugin_names) + + out = self.conv3(out) + out = self.norm3(out) + + if self.with_plugins: + out = self.forward_plugin(out, self.after_conv3_plugin_names) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + + return out + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x) + else: + out = _inner_forward(x) + + out = self.relu(out) + + return out + + +@BACKBONES.register_module() +class ResNet(nn.Module): + """ResNet backbone. + + Args: + depth (int): Depth of resnet, from {18, 34, 50, 101, 152}. + in_channels (int): Number of input image channels. Default" 3. + stem_channels (int): Number of stem channels. Default: 64. + base_channels (int): Number of base channels of res layer. Default: 64. + num_stages (int): Resnet stages, normally 4. + strides (Sequence[int]): Strides of the first block of each stage. + dilations (Sequence[int]): Dilation of each stage. + out_indices (Sequence[int]): Output from which stages. + style (str): `pytorch` or `caffe`. If set to "pytorch", the stride-two + layer is the 3x3 conv layer, otherwise the stride-two layer is + the first 1x1 conv layer. + deep_stem (bool): Replace 7x7 conv in input stem with 3 3x3 conv + avg_down (bool): Use AvgPool instead of stride conv when + downsampling in the bottleneck. + frozen_stages (int): Stages to be frozen (stop grad and set eval mode). + -1 means not freezing any parameters. + norm_cfg (dict): Dictionary to construct and config norm layer. + norm_eval (bool): Whether to set norm layers to eval mode, namely, + freeze running stats (mean and var). Note: Effect on Batch Norm + and its variants only. + plugins (list[dict]): List of plugins for stages, each dict contains: + + - cfg (dict, required): Cfg dict to build plugin. + + - position (str, required): Position inside block to insert plugin, + options: 'after_conv1', 'after_conv2', 'after_conv3'. + + - stages (tuple[bool], optional): Stages to apply plugin, length + should be same as 'num_stages' + multi_grid (Sequence[int]|None): Multi grid dilation rates of last + stage. Default: None + contract_dilation (bool): Whether contract first dilation of each layer + Default: False + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. + zero_init_residual (bool): Whether to use zero init for last norm layer + in resblocks to let them behave as identity. + + Example: + >>> from mmseg.models import ResNet + >>> import torch + >>> self = ResNet(depth=18) + >>> self.eval() + >>> inputs = torch.rand(1, 3, 32, 32) + >>> level_outputs = self.forward(inputs) + >>> for level_out in level_outputs: + ... print(tuple(level_out.shape)) + (1, 64, 8, 8) + (1, 128, 4, 4) + (1, 256, 2, 2) + (1, 512, 1, 1) + """ + + arch_settings = { + 18: (BasicBlock, (2, 2, 2, 2)), + 34: (BasicBlock, (3, 4, 6, 3)), + 50: (Bottleneck, (3, 4, 6, 3)), + 101: (Bottleneck, (3, 4, 23, 3)), + 152: (Bottleneck, (3, 8, 36, 3)) + } + + def __init__(self, + depth, + in_channels=3, + stem_channels=64, + base_channels=64, + num_stages=4, + strides=(1, 2, 2, 2), + dilations=(1, 1, 1, 1), + out_indices=(0, 1, 2, 3), + style='pytorch', + deep_stem=False, + avg_down=False, + frozen_stages=-1, + conv_cfg=None, + norm_cfg=dict(type='BN', requires_grad=True), + norm_eval=False, + dcn=None, + stage_with_dcn=(False, False, False, False), + plugins=None, + multi_grid=None, + contract_dilation=False, + with_cp=False, + zero_init_residual=True): + super(ResNet, self).__init__() + if depth not in self.arch_settings: + raise KeyError(f'invalid depth {depth} for resnet') + self.depth = depth + self.stem_channels = stem_channels + self.base_channels = base_channels + self.num_stages = num_stages + assert num_stages >= 1 and num_stages <= 4 + self.strides = strides + self.dilations = dilations + assert len(strides) == len(dilations) == num_stages + self.out_indices = out_indices + assert max(out_indices) < num_stages + self.style = style + self.deep_stem = deep_stem + self.avg_down = avg_down + self.frozen_stages = frozen_stages + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.with_cp = with_cp + self.norm_eval = norm_eval + self.dcn = dcn + self.stage_with_dcn = stage_with_dcn + if dcn is not None: + assert len(stage_with_dcn) == num_stages + self.plugins = plugins + self.multi_grid = multi_grid + self.contract_dilation = contract_dilation + self.zero_init_residual = zero_init_residual + self.block, stage_blocks = self.arch_settings[depth] + self.stage_blocks = stage_blocks[:num_stages] + self.inplanes = stem_channels + + self._make_stem_layer(in_channels, stem_channels) + + self.res_layers = [] + for i, num_blocks in enumerate(self.stage_blocks): + stride = strides[i] + dilation = dilations[i] + dcn = self.dcn if self.stage_with_dcn[i] else None + if plugins is not None: + stage_plugins = self.make_stage_plugins(plugins, i) + else: + stage_plugins = None + # multi grid is applied to last layer only + stage_multi_grid = multi_grid if i == len( + self.stage_blocks) - 1 else None + planes = base_channels * 2**i + res_layer = self.make_res_layer( + block=self.block, + inplanes=self.inplanes, + planes=planes, + num_blocks=num_blocks, + stride=stride, + dilation=dilation, + style=self.style, + avg_down=self.avg_down, + with_cp=with_cp, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + dcn=dcn, + plugins=stage_plugins, + multi_grid=stage_multi_grid, + contract_dilation=contract_dilation) + self.inplanes = planes * self.block.expansion + layer_name = f'layer{i+1}' + self.add_module(layer_name, res_layer) + self.res_layers.append(layer_name) + + self._freeze_stages() + + self.feat_dim = self.block.expansion * base_channels * 2**( + len(self.stage_blocks) - 1) + + def make_stage_plugins(self, plugins, stage_idx): + """make plugins for ResNet 'stage_idx'th stage . + + Currently we support to insert 'context_block', + 'empirical_attention_block', 'nonlocal_block' into the backbone like + ResNet/ResNeXt. They could be inserted after conv1/conv2/conv3 of + Bottleneck. + + An example of plugins format could be : + >>> plugins=[ + ... dict(cfg=dict(type='xxx', arg1='xxx'), + ... stages=(False, True, True, True), + ... position='after_conv2'), + ... dict(cfg=dict(type='yyy'), + ... stages=(True, True, True, True), + ... position='after_conv3'), + ... dict(cfg=dict(type='zzz', postfix='1'), + ... stages=(True, True, True, True), + ... position='after_conv3'), + ... dict(cfg=dict(type='zzz', postfix='2'), + ... stages=(True, True, True, True), + ... position='after_conv3') + ... ] + >>> self = ResNet(depth=18) + >>> stage_plugins = self.make_stage_plugins(plugins, 0) + >>> assert len(stage_plugins) == 3 + + Suppose 'stage_idx=0', the structure of blocks in the stage would be: + conv1-> conv2->conv3->yyy->zzz1->zzz2 + Suppose 'stage_idx=1', the structure of blocks in the stage would be: + conv1-> conv2->xxx->conv3->yyy->zzz1->zzz2 + + If stages is missing, the plugin would be applied to all stages. + + Args: + plugins (list[dict]): List of plugins cfg to build. The postfix is + required if multiple same type plugins are inserted. + stage_idx (int): Index of stage to build + + Returns: + list[dict]: Plugins for current stage + """ + stage_plugins = [] + for plugin in plugins: + plugin = plugin.copy() + stages = plugin.pop('stages', None) + assert stages is None or len(stages) == self.num_stages + # whether to insert plugin into current stage + if stages is None or stages[stage_idx]: + stage_plugins.append(plugin) + + return stage_plugins + + def make_res_layer(self, **kwargs): + """Pack all blocks in a stage into a ``ResLayer``.""" + return ResLayer(**kwargs) + + @property + def norm1(self): + """nn.Module: the normalization layer named "norm1" """ + return getattr(self, self.norm1_name) + + def _make_stem_layer(self, in_channels, stem_channels): + """Make stem layer for ResNet.""" + if self.deep_stem: + self.stem = nn.Sequential( + build_conv_layer( + self.conv_cfg, + in_channels, + stem_channels // 2, + kernel_size=3, + stride=2, + padding=1, + bias=False), + build_norm_layer(self.norm_cfg, stem_channels // 2)[1], + nn.ReLU(inplace=True), + build_conv_layer( + self.conv_cfg, + stem_channels // 2, + stem_channels // 2, + kernel_size=3, + stride=1, + padding=1, + bias=False), + build_norm_layer(self.norm_cfg, stem_channels // 2)[1], + nn.ReLU(inplace=True), + build_conv_layer( + self.conv_cfg, + stem_channels // 2, + stem_channels, + kernel_size=3, + stride=1, + padding=1, + bias=False), + build_norm_layer(self.norm_cfg, stem_channels)[1], + nn.ReLU(inplace=True)) + else: + self.conv1 = build_conv_layer( + self.conv_cfg, + in_channels, + stem_channels, + kernel_size=7, + stride=2, + padding=3, + bias=False) + self.norm1_name, norm1 = build_norm_layer( + self.norm_cfg, stem_channels, postfix=1) + self.add_module(self.norm1_name, norm1) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + + def _freeze_stages(self): + """Freeze stages param and norm stats.""" + if self.frozen_stages >= 0: + if self.deep_stem: + self.stem.eval() + for param in self.stem.parameters(): + param.requires_grad = False + else: + self.norm1.eval() + for m in [self.conv1, self.norm1]: + for param in m.parameters(): + param.requires_grad = False + + for i in range(1, self.frozen_stages + 1): + m = getattr(self, f'layer{i}') + m.eval() + for param in m.parameters(): + param.requires_grad = False + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + if isinstance(pretrained, str): + logger = get_root_logger() + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + for m in self.modules(): + if isinstance(m, nn.Conv2d): + kaiming_init(m) + elif isinstance(m, (_BatchNorm, nn.GroupNorm)): + constant_init(m, 1) + + if self.dcn is not None: + for m in self.modules(): + if isinstance(m, Bottleneck) and hasattr( + m, 'conv2_offset'): + constant_init(m.conv2_offset, 0) + + if self.zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + constant_init(m.norm3, 0) + elif isinstance(m, BasicBlock): + constant_init(m.norm2, 0) + else: + raise TypeError('pretrained must be a str or None') + + def forward(self, x): + """Forward function.""" + if self.deep_stem: + x = self.stem(x) + else: + x = self.conv1(x) + x = self.norm1(x) + x = self.relu(x) + x = self.maxpool(x) + outs = [] + for i, layer_name in enumerate(self.res_layers): + res_layer = getattr(self, layer_name) + x = res_layer(x) + if i in self.out_indices: + outs.append(x) + return tuple(outs) + + def train(self, mode=True): + """Convert the model into training mode while keep normalization layer + freezed.""" + super(ResNet, self).train(mode) + self._freeze_stages() + if mode and self.norm_eval: + for m in self.modules(): + # trick: eval have effect on BatchNorm only + if isinstance(m, _BatchNorm): + m.eval() + + +@BACKBONES.register_module() +class ResNetV1c(ResNet): + """ResNetV1c variant described in [1]_. + + Compared with default ResNet(ResNetV1b), ResNetV1c replaces the 7x7 conv + in the input stem with three 3x3 convs. + + References: + .. [1] https://arxiv.org/pdf/1812.01187.pdf + """ + + def __init__(self, **kwargs): + super(ResNetV1c, self).__init__( + deep_stem=True, avg_down=False, **kwargs) + + +@BACKBONES.register_module() +class ResNetV1d(ResNet): + """ResNetV1d variant described in [1]_. + + Compared with default ResNet(ResNetV1b), ResNetV1d replaces the 7x7 conv in + the input stem with three 3x3 convs. And in the downsampling block, a 2x2 + avg_pool with stride 2 is added before conv, whose stride is changed to 1. + """ + + def __init__(self, **kwargs): + super(ResNetV1d, self).__init__( + deep_stem=True, avg_down=True, **kwargs) diff --git a/segmentation/mmseg/models/backbones/resnext.py b/segmentation/mmseg/models/backbones/resnext.py new file mode 100644 index 0000000..fa8149c --- /dev/null +++ b/segmentation/mmseg/models/backbones/resnext.py @@ -0,0 +1,145 @@ +import math + +from mmcv.cnn import build_conv_layer, build_norm_layer + +from ..builder import BACKBONES +from ..utils import ResLayer +from .resnet import Bottleneck as _Bottleneck +from .resnet import ResNet + + +class Bottleneck(_Bottleneck): + """Bottleneck block for ResNeXt. + + If style is "pytorch", the stride-two layer is the 3x3 conv layer, if it is + "caffe", the stride-two layer is the first 1x1 conv layer. + """ + + def __init__(self, + inplanes, + planes, + groups=1, + base_width=4, + base_channels=64, + **kwargs): + super(Bottleneck, self).__init__(inplanes, planes, **kwargs) + + if groups == 1: + width = self.planes + else: + width = math.floor(self.planes * + (base_width / base_channels)) * groups + + self.norm1_name, norm1 = build_norm_layer( + self.norm_cfg, width, postfix=1) + self.norm2_name, norm2 = build_norm_layer( + self.norm_cfg, width, postfix=2) + self.norm3_name, norm3 = build_norm_layer( + self.norm_cfg, self.planes * self.expansion, postfix=3) + + self.conv1 = build_conv_layer( + self.conv_cfg, + self.inplanes, + width, + kernel_size=1, + stride=self.conv1_stride, + bias=False) + self.add_module(self.norm1_name, norm1) + fallback_on_stride = False + self.with_modulated_dcn = False + if self.with_dcn: + fallback_on_stride = self.dcn.pop('fallback_on_stride', False) + if not self.with_dcn or fallback_on_stride: + self.conv2 = build_conv_layer( + self.conv_cfg, + width, + width, + kernel_size=3, + stride=self.conv2_stride, + padding=self.dilation, + dilation=self.dilation, + groups=groups, + bias=False) + else: + assert self.conv_cfg is None, 'conv_cfg must be None for DCN' + self.conv2 = build_conv_layer( + self.dcn, + width, + width, + kernel_size=3, + stride=self.conv2_stride, + padding=self.dilation, + dilation=self.dilation, + groups=groups, + bias=False) + + self.add_module(self.norm2_name, norm2) + self.conv3 = build_conv_layer( + self.conv_cfg, + width, + self.planes * self.expansion, + kernel_size=1, + bias=False) + self.add_module(self.norm3_name, norm3) + + +@BACKBONES.register_module() +class ResNeXt(ResNet): + """ResNeXt backbone. + + Args: + depth (int): Depth of resnet, from {18, 34, 50, 101, 152}. + in_channels (int): Number of input image channels. Normally 3. + num_stages (int): Resnet stages, normally 4. + groups (int): Group of resnext. + base_width (int): Base width of resnext. + strides (Sequence[int]): Strides of the first block of each stage. + dilations (Sequence[int]): Dilation of each stage. + out_indices (Sequence[int]): Output from which stages. + style (str): `pytorch` or `caffe`. If set to "pytorch", the stride-two + layer is the 3x3 conv layer, otherwise the stride-two layer is + the first 1x1 conv layer. + frozen_stages (int): Stages to be frozen (all param fixed). -1 means + not freezing any parameters. + norm_cfg (dict): dictionary to construct and config norm layer. + norm_eval (bool): Whether to set norm layers to eval mode, namely, + freeze running stats (mean and var). Note: Effect on Batch Norm + and its variants only. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. + zero_init_residual (bool): whether to use zero init for last norm layer + in resblocks to let them behave as identity. + + Example: + >>> from mmseg.models import ResNeXt + >>> import torch + >>> self = ResNeXt(depth=50) + >>> self.eval() + >>> inputs = torch.rand(1, 3, 32, 32) + >>> level_outputs = self.forward(inputs) + >>> for level_out in level_outputs: + ... print(tuple(level_out.shape)) + (1, 256, 8, 8) + (1, 512, 4, 4) + (1, 1024, 2, 2) + (1, 2048, 1, 1) + """ + + arch_settings = { + 50: (Bottleneck, (3, 4, 6, 3)), + 101: (Bottleneck, (3, 4, 23, 3)), + 152: (Bottleneck, (3, 8, 36, 3)) + } + + def __init__(self, groups=1, base_width=4, **kwargs): + self.groups = groups + self.base_width = base_width + super(ResNeXt, self).__init__(**kwargs) + + def make_res_layer(self, **kwargs): + """Pack all blocks in a stage into a ``ResLayer``""" + return ResLayer( + groups=self.groups, + base_width=self.base_width, + base_channels=self.base_channels, + **kwargs) diff --git a/segmentation/mmseg/models/backbones/unet.py b/segmentation/mmseg/models/backbones/unet.py new file mode 100644 index 0000000..6cbda00 --- /dev/null +++ b/segmentation/mmseg/models/backbones/unet.py @@ -0,0 +1,429 @@ +import torch.nn as nn +import torch.utils.checkpoint as cp +from mmcv.cnn import (UPSAMPLE_LAYERS, ConvModule, build_activation_layer, + build_norm_layer, constant_init, kaiming_init) +from mmcv.runner import load_checkpoint +from mmcv.utils.parrots_wrapper import _BatchNorm + +from mmseg.utils import get_root_logger +from ..builder import BACKBONES +from ..utils import UpConvBlock + + +class BasicConvBlock(nn.Module): + """Basic convolutional block for UNet. + + This module consists of several plain convolutional layers. + + Args: + in_channels (int): Number of input channels. + out_channels (int): Number of output channels. + num_convs (int): Number of convolutional layers. Default: 2. + stride (int): Whether use stride convolution to downsample + the input feature map. If stride=2, it only uses stride convolution + in the first convolutional layer to downsample the input feature + map. Options are 1 or 2. Default: 1. + dilation (int): Whether use dilated convolution to expand the + receptive field. Set dilation rate of each convolutional layer and + the dilation rate of the first convolutional layer is always 1. + Default: 1. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + conv_cfg (dict | None): Config dict for convolution layer. + Default: None. + norm_cfg (dict | None): Config dict for normalization layer. + Default: dict(type='BN'). + act_cfg (dict | None): Config dict for activation layer in ConvModule. + Default: dict(type='ReLU'). + dcn (bool): Use deformable convolution in convolutional layer or not. + Default: None. + plugins (dict): plugins for convolutional layers. Default: None. + """ + + def __init__(self, + in_channels, + out_channels, + num_convs=2, + stride=1, + dilation=1, + with_cp=False, + conv_cfg=None, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU'), + dcn=None, + plugins=None): + super(BasicConvBlock, self).__init__() + assert dcn is None, 'Not implemented yet.' + assert plugins is None, 'Not implemented yet.' + + self.with_cp = with_cp + convs = [] + for i in range(num_convs): + convs.append( + ConvModule( + in_channels=in_channels if i == 0 else out_channels, + out_channels=out_channels, + kernel_size=3, + stride=stride if i == 0 else 1, + dilation=1 if i == 0 else dilation, + padding=1 if i == 0 else dilation, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg)) + + self.convs = nn.Sequential(*convs) + + def forward(self, x): + """Forward function.""" + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(self.convs, x) + else: + out = self.convs(x) + return out + + +@UPSAMPLE_LAYERS.register_module() +class DeconvModule(nn.Module): + """Deconvolution upsample module in decoder for UNet (2X upsample). + + This module uses deconvolution to upsample feature map in the decoder + of UNet. + + Args: + in_channels (int): Number of input channels. + out_channels (int): Number of output channels. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + norm_cfg (dict | None): Config dict for normalization layer. + Default: dict(type='BN'). + act_cfg (dict | None): Config dict for activation layer in ConvModule. + Default: dict(type='ReLU'). + kernel_size (int): Kernel size of the convolutional layer. Default: 4. + """ + + def __init__(self, + in_channels, + out_channels, + with_cp=False, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU'), + *, + kernel_size=4, + scale_factor=2): + super(DeconvModule, self).__init__() + + assert (kernel_size - scale_factor >= 0) and\ + (kernel_size - scale_factor) % 2 == 0,\ + f'kernel_size should be greater than or equal to scale_factor '\ + f'and (kernel_size - scale_factor) should be even numbers, '\ + f'while the kernel size is {kernel_size} and scale_factor is '\ + f'{scale_factor}.' + + stride = scale_factor + padding = (kernel_size - scale_factor) // 2 + self.with_cp = with_cp + deconv = nn.ConvTranspose2d( + in_channels, + out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding) + + norm_name, norm = build_norm_layer(norm_cfg, out_channels) + activate = build_activation_layer(act_cfg) + self.deconv_upsamping = nn.Sequential(deconv, norm, activate) + + def forward(self, x): + """Forward function.""" + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(self.deconv_upsamping, x) + else: + out = self.deconv_upsamping(x) + return out + + +@UPSAMPLE_LAYERS.register_module() +class InterpConv(nn.Module): + """Interpolation upsample module in decoder for UNet. + + This module uses interpolation to upsample feature map in the decoder + of UNet. It consists of one interpolation upsample layer and one + convolutional layer. It can be one interpolation upsample layer followed + by one convolutional layer (conv_first=False) or one convolutional layer + followed by one interpolation upsample layer (conv_first=True). + + Args: + in_channels (int): Number of input channels. + out_channels (int): Number of output channels. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + norm_cfg (dict | None): Config dict for normalization layer. + Default: dict(type='BN'). + act_cfg (dict | None): Config dict for activation layer in ConvModule. + Default: dict(type='ReLU'). + conv_cfg (dict | None): Config dict for convolution layer. + Default: None. + conv_first (bool): Whether convolutional layer or interpolation + upsample layer first. Default: False. It means interpolation + upsample layer followed by one convolutional layer. + kernel_size (int): Kernel size of the convolutional layer. Default: 1. + stride (int): Stride of the convolutional layer. Default: 1. + padding (int): Padding of the convolutional layer. Default: 1. + upsample_cfg (dict): Interpolation config of the upsample layer. + Default: dict( + scale_factor=2, mode='bilinear', align_corners=False). + """ + + def __init__(self, + in_channels, + out_channels, + with_cp=False, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU'), + *, + conv_cfg=None, + conv_first=False, + kernel_size=1, + stride=1, + padding=0, + upsample_cfg=dict( + scale_factor=2, mode='bilinear', align_corners=False)): + super(InterpConv, self).__init__() + + self.with_cp = with_cp + conv = ConvModule( + in_channels, + out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + upsample = nn.Upsample(**upsample_cfg) + if conv_first: + self.interp_upsample = nn.Sequential(conv, upsample) + else: + self.interp_upsample = nn.Sequential(upsample, conv) + + def forward(self, x): + """Forward function.""" + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(self.interp_upsample, x) + else: + out = self.interp_upsample(x) + return out + + +@BACKBONES.register_module() +class UNet(nn.Module): + """UNet backbone. + U-Net: Convolutional Networks for Biomedical Image Segmentation. + https://arxiv.org/pdf/1505.04597.pdf + + Args: + in_channels (int): Number of input image channels. Default" 3. + base_channels (int): Number of base channels of each stage. + The output channels of the first stage. Default: 64. + num_stages (int): Number of stages in encoder, normally 5. Default: 5. + strides (Sequence[int 1 | 2]): Strides of each stage in encoder. + len(strides) is equal to num_stages. Normally the stride of the + first stage in encoder is 1. If strides[i]=2, it uses stride + convolution to downsample in the correspondence encoder stage. + Default: (1, 1, 1, 1, 1). + enc_num_convs (Sequence[int]): Number of convolutional layers in the + convolution block of the correspondence encoder stage. + Default: (2, 2, 2, 2, 2). + dec_num_convs (Sequence[int]): Number of convolutional layers in the + convolution block of the correspondence decoder stage. + Default: (2, 2, 2, 2). + downsamples (Sequence[int]): Whether use MaxPool to downsample the + feature map after the first stage of encoder + (stages: [1, num_stages)). If the correspondence encoder stage use + stride convolution (strides[i]=2), it will never use MaxPool to + downsample, even downsamples[i-1]=True. + Default: (True, True, True, True). + enc_dilations (Sequence[int]): Dilation rate of each stage in encoder. + Default: (1, 1, 1, 1, 1). + dec_dilations (Sequence[int]): Dilation rate of each stage in decoder. + Default: (1, 1, 1, 1). + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + conv_cfg (dict | None): Config dict for convolution layer. + Default: None. + norm_cfg (dict | None): Config dict for normalization layer. + Default: dict(type='BN'). + act_cfg (dict | None): Config dict for activation layer in ConvModule. + Default: dict(type='ReLU'). + upsample_cfg (dict): The upsample config of the upsample module in + decoder. Default: dict(type='InterpConv'). + norm_eval (bool): Whether to set norm layers to eval mode, namely, + freeze running stats (mean and var). Note: Effect on Batch Norm + and its variants only. Default: False. + dcn (bool): Use deformable convolution in convolutional layer or not. + Default: None. + plugins (dict): plugins for convolutional layers. Default: None. + + Notice: + The input image size should be divisible by the whole downsample rate + of the encoder. More detail of the whole downsample rate can be found + in UNet._check_input_divisible. + + """ + + def __init__(self, + in_channels=3, + base_channels=64, + num_stages=5, + strides=(1, 1, 1, 1, 1), + enc_num_convs=(2, 2, 2, 2, 2), + dec_num_convs=(2, 2, 2, 2), + downsamples=(True, True, True, True), + enc_dilations=(1, 1, 1, 1, 1), + dec_dilations=(1, 1, 1, 1), + with_cp=False, + conv_cfg=None, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU'), + upsample_cfg=dict(type='InterpConv'), + norm_eval=False, + dcn=None, + plugins=None): + super(UNet, self).__init__() + assert dcn is None, 'Not implemented yet.' + assert plugins is None, 'Not implemented yet.' + assert len(strides) == num_stages, \ + 'The length of strides should be equal to num_stages, '\ + f'while the strides is {strides}, the length of '\ + f'strides is {len(strides)}, and the num_stages is '\ + f'{num_stages}.' + assert len(enc_num_convs) == num_stages, \ + 'The length of enc_num_convs should be equal to num_stages, '\ + f'while the enc_num_convs is {enc_num_convs}, the length of '\ + f'enc_num_convs is {len(enc_num_convs)}, and the num_stages is '\ + f'{num_stages}.' + assert len(dec_num_convs) == (num_stages-1), \ + 'The length of dec_num_convs should be equal to (num_stages-1), '\ + f'while the dec_num_convs is {dec_num_convs}, the length of '\ + f'dec_num_convs is {len(dec_num_convs)}, and the num_stages is '\ + f'{num_stages}.' + assert len(downsamples) == (num_stages-1), \ + 'The length of downsamples should be equal to (num_stages-1), '\ + f'while the downsamples is {downsamples}, the length of '\ + f'downsamples is {len(downsamples)}, and the num_stages is '\ + f'{num_stages}.' + assert len(enc_dilations) == num_stages, \ + 'The length of enc_dilations should be equal to num_stages, '\ + f'while the enc_dilations is {enc_dilations}, the length of '\ + f'enc_dilations is {len(enc_dilations)}, and the num_stages is '\ + f'{num_stages}.' + assert len(dec_dilations) == (num_stages-1), \ + 'The length of dec_dilations should be equal to (num_stages-1), '\ + f'while the dec_dilations is {dec_dilations}, the length of '\ + f'dec_dilations is {len(dec_dilations)}, and the num_stages is '\ + f'{num_stages}.' + self.num_stages = num_stages + self.strides = strides + self.downsamples = downsamples + self.norm_eval = norm_eval + self.base_channels = base_channels + + self.encoder = nn.ModuleList() + self.decoder = nn.ModuleList() + + for i in range(num_stages): + enc_conv_block = [] + if i != 0: + if strides[i] == 1 and downsamples[i - 1]: + enc_conv_block.append(nn.MaxPool2d(kernel_size=2)) + upsample = (strides[i] != 1 or downsamples[i - 1]) + self.decoder.append( + UpConvBlock( + conv_block=BasicConvBlock, + in_channels=base_channels * 2**i, + skip_channels=base_channels * 2**(i - 1), + out_channels=base_channels * 2**(i - 1), + num_convs=dec_num_convs[i - 1], + stride=1, + dilation=dec_dilations[i - 1], + with_cp=with_cp, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + upsample_cfg=upsample_cfg if upsample else None, + dcn=None, + plugins=None)) + + enc_conv_block.append( + BasicConvBlock( + in_channels=in_channels, + out_channels=base_channels * 2**i, + num_convs=enc_num_convs[i], + stride=strides[i], + dilation=enc_dilations[i], + with_cp=with_cp, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + dcn=None, + plugins=None)) + self.encoder.append((nn.Sequential(*enc_conv_block))) + in_channels = base_channels * 2**i + + def forward(self, x): + self._check_input_divisible(x) + enc_outs = [] + for enc in self.encoder: + x = enc(x) + enc_outs.append(x) + dec_outs = [x] + for i in reversed(range(len(self.decoder))): + x = self.decoder[i](enc_outs[i], x) + dec_outs.append(x) + + return dec_outs + + def train(self, mode=True): + """Convert the model into training mode while keep normalization layer + freezed.""" + super(UNet, self).train(mode) + if mode and self.norm_eval: + for m in self.modules(): + # trick: eval have effect on BatchNorm only + if isinstance(m, _BatchNorm): + m.eval() + + def _check_input_divisible(self, x): + h, w = x.shape[-2:] + whole_downsample_rate = 1 + for i in range(1, self.num_stages): + if self.strides[i] == 2 or self.downsamples[i - 1]: + whole_downsample_rate *= 2 + assert (h % whole_downsample_rate == 0) \ + and (w % whole_downsample_rate == 0),\ + f'The input image size {(h, w)} should be divisible by the whole '\ + f'downsample rate {whole_downsample_rate}, when num_stages is '\ + f'{self.num_stages}, strides is {self.strides}, and downsamples '\ + f'is {self.downsamples}.' + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + if isinstance(pretrained, str): + logger = get_root_logger() + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + for m in self.modules(): + if isinstance(m, nn.Conv2d): + kaiming_init(m) + elif isinstance(m, (_BatchNorm, nn.GroupNorm)): + constant_init(m, 1) + else: + raise TypeError('pretrained must be a str or None') diff --git a/segmentation/mmseg/models/backbones/vit.py b/segmentation/mmseg/models/backbones/vit.py new file mode 100644 index 0000000..781c9c1 --- /dev/null +++ b/segmentation/mmseg/models/backbones/vit.py @@ -0,0 +1,470 @@ +"""Modified from https://github.com/rwightman/pytorch-image- +models/blob/master/timm/models/vision_transformer.py.""" + +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as cp +from mmcv.cnn import (Conv2d, Linear, build_activation_layer, build_norm_layer, + constant_init, kaiming_init, normal_init) +from mmcv.runner import _load_checkpoint +from mmcv.utils.parrots_wrapper import _BatchNorm + +from mmseg.utils import get_root_logger +from ..builder import BACKBONES +from ..utils import DropPath, trunc_normal_ + + +class Mlp(nn.Module): + """MLP layer for Encoder block. + + Args: + in_features(int): Input dimension for the first fully + connected layer. + hidden_features(int): Output dimension for the first fully + connected layer. + out_features(int): Output dementsion for the second fully + connected layer. + act_cfg(dict): Config dict for activation layer. + Default: dict(type='GELU'). + drop(float): Drop rate for the dropout layer. Dropout rate has + to be between 0 and 1. Default: 0. + """ + + def __init__(self, + in_features, + hidden_features=None, + out_features=None, + act_cfg=dict(type='GELU'), + drop=0.): + super(Mlp, self).__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = Linear(in_features, hidden_features) + self.act = build_activation_layer(act_cfg) + self.fc2 = Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +class Attention(nn.Module): + """Attention layer for Encoder block. + + Args: + dim (int): Dimension for the input vector. + num_heads (int): Number of parallel attention heads. + qkv_bias (bool): Enable bias for qkv if True. Default: False. + qk_scale (float): Override default qk scale of head_dim ** -0.5 if set. + attn_drop (float): Drop rate for attention output weights. + Default: 0. + proj_drop (float): Drop rate for output weights. Default: 0. + """ + + def __init__(self, + dim, + num_heads=8, + qkv_bias=False, + qk_scale=None, + attn_drop=0., + proj_drop=0.): + super(Attention, self).__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = qk_scale or head_dim**-0.5 + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x): + b, n, c = x.shape + qkv = self.qkv(x).reshape(b, n, 3, self.num_heads, + c // self.num_heads).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[2] + + attn = (q @ k.transpose(-2, -1)) * self.scale + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(b, n, c) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class Block(nn.Module): + """Implements encoder block with residual connection. + + Args: + dim (int): The feature dimension. + num_heads (int): Number of parallel attention heads. + mlp_ratio (int): Ratio of mlp hidden dim to embedding dim. + qk_scale (float): Override default qk scale of head_dim ** -0.5 if set. + drop (float): Drop rate for mlp output weights. Default: 0. + attn_drop (float): Drop rate for attention output weights. + Default: 0. + proj_drop (float): Drop rate for attn layer output weights. + Default: 0. + drop_path (float): Drop rate for paths of model. + Default: 0. + act_cfg (dict): Config dict for activation layer. + Default: dict(type='GELU'). + norm_cfg (dict): Config dict for normalization layer. + Default: dict(type='LN', requires_grad=True). + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + """ + + def __init__(self, + dim, + num_heads, + mlp_ratio=4, + qkv_bias=False, + qk_scale=None, + drop=0., + attn_drop=0., + proj_drop=0., + drop_path=0., + act_cfg=dict(type='GELU'), + norm_cfg=dict(type='LN', eps=1e-6), + with_cp=False): + super(Block, self).__init__() + self.with_cp = with_cp + _, self.norm1 = build_norm_layer(norm_cfg, dim) + self.attn = Attention(dim, num_heads, qkv_bias, qk_scale, attn_drop, + proj_drop) + self.drop_path = DropPath( + drop_path) if drop_path > 0. else nn.Identity() + _, self.norm2 = build_norm_layer(norm_cfg, dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp( + in_features=dim, + hidden_features=mlp_hidden_dim, + act_cfg=act_cfg, + drop=drop) + + def forward(self, x): + + def _inner_forward(x): + out = x + self.drop_path(self.attn(self.norm1(x))) + out = out + self.drop_path(self.mlp(self.norm2(out))) + return out + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x) + else: + out = _inner_forward(x) + + return out + + +class PatchEmbed(nn.Module): + """Image to Patch Embedding. + + Args: + img_size (int | tuple): Input image size. + default: 224. + patch_size (int): Width and height for a patch. + default: 16. + in_channels (int): Input channels for images. Default: 3. + embed_dim (int): The embedding dimension. Default: 768. + """ + + def __init__(self, + img_size=224, + patch_size=16, + in_channels=3, + embed_dim=768): + super(PatchEmbed, self).__init__() + if isinstance(img_size, int): + self.img_size = (img_size, img_size) + elif isinstance(img_size, tuple): + self.img_size = img_size + else: + raise TypeError('img_size must be type of int or tuple') + h, w = self.img_size + self.patch_size = (patch_size, patch_size) + self.num_patches = (h // patch_size) * (w // patch_size) + self.proj = Conv2d( + in_channels, embed_dim, kernel_size=patch_size, stride=patch_size) + + def forward(self, x): + return self.proj(x).flatten(2).transpose(1, 2) + + +@BACKBONES.register_module() +class VisionTransformer(nn.Module): + """Vision transformer backbone. + + A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for + Image Recognition at Scale` - https://arxiv.org/abs/2010.11929 + + Args: + img_size (tuple): input image size. Default: (224, 224). + patch_size (int, tuple): patch size. Default: 16. + in_channels (int): number of input channels. Default: 3. + embed_dim (int): embedding dimension. Default: 768. + depth (int): depth of transformer. Default: 12. + num_heads (int): number of attention heads. Default: 12. + mlp_ratio (int): ratio of mlp hidden dim to embedding dim. + Default: 4. + out_indices (list | tuple | int): Output from which stages. + Default: -1. + qkv_bias (bool): enable bias for qkv if True. Default: True. + qk_scale (float): override default qk scale of head_dim ** -0.5 if set. + drop_rate (float): dropout rate. Default: 0. + attn_drop_rate (float): attention dropout rate. Default: 0. + drop_path_rate (float): Rate of DropPath. Default: 0. + norm_cfg (dict): Config dict for normalization layer. + Default: dict(type='LN', eps=1e-6, requires_grad=True). + act_cfg (dict): Config dict for activation layer. + Default: dict(type='GELU'). + norm_eval (bool): Whether to set norm layers to eval mode, namely, + freeze running stats (mean and var). Note: Effect on Batch Norm + and its variants only. Default: False. + final_norm (bool): Whether to add a additional layer to normalize + final feature map. Default: False. + out_reshape (str): Select the output format of feature information. + Default: NCHW. + interpolate_mode (str): Select the interpolate mode for position + embeding vector resize. Default: bicubic. + with_cls_token (bool): If concatenating class token into image tokens + as transformer input. Default: True. + with_cp (bool): Use checkpoint or not. Using checkpoint + will save some memory while slowing down the training speed. + Default: False. + """ + + def __init__(self, + img_size=(224, 224), + patch_size=16, + in_channels=3, + embed_dim=768, + depth=12, + num_heads=12, + mlp_ratio=4, + out_indices=11, + qkv_bias=True, + qk_scale=None, + drop_rate=0., + attn_drop_rate=0., + drop_path_rate=0., + norm_cfg=dict(type='LN', eps=1e-6, requires_grad=True), + act_cfg=dict(type='GELU'), + norm_eval=False, + final_norm=False, + out_shape='NCHW', + with_cls_token=True, + interpolate_mode='bicubic', + with_cp=False): + super(VisionTransformer, self).__init__() + self.img_size = img_size + self.patch_size = patch_size + self.features = self.embed_dim = embed_dim + self.patch_embed = PatchEmbed( + img_size=img_size, + patch_size=patch_size, + in_channels=in_channels, + embed_dim=embed_dim) + + self.with_cls_token = with_cls_token + self.cls_token = nn.Parameter(torch.zeros(1, 1, self.embed_dim)) + self.pos_embed = nn.Parameter( + torch.zeros(1, self.patch_embed.num_patches + 1, embed_dim)) + self.pos_drop = nn.Dropout(p=drop_rate) + + if isinstance(out_indices, int): + self.out_indices = [out_indices] + elif isinstance(out_indices, list) or isinstance(out_indices, tuple): + self.out_indices = out_indices + else: + raise TypeError('out_indices must be type of int, list or tuple') + + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth) + ] # stochastic depth decay rule + self.blocks = nn.ModuleList([ + Block( + dim=embed_dim, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=dpr[i], + attn_drop=attn_drop_rate, + act_cfg=act_cfg, + norm_cfg=norm_cfg, + with_cp=with_cp) for i in range(depth) + ]) + + assert out_shape in ['NLC', + 'NCHW'], 'output shape must be "NLC" or "NCHW".' + + self.out_shape = out_shape + + self.interpolate_mode = interpolate_mode + self.final_norm = final_norm + if final_norm: + _, self.norm = build_norm_layer(norm_cfg, embed_dim) + + self.norm_eval = norm_eval + self.with_cp = with_cp + + def init_weights(self, pretrained=None): + if isinstance(pretrained, str): + logger = get_root_logger() + checkpoint = _load_checkpoint(pretrained, logger=logger) + if 'state_dict' in checkpoint: + state_dict = checkpoint['state_dict'] + elif 'model' in checkpoint: + state_dict = checkpoint['model'] + else: + state_dict = checkpoint + + if 'pos_embed' in state_dict.keys(): + if self.pos_embed.shape != state_dict['pos_embed'].shape: + logger.info(msg=f'Resize the pos_embed shape from \ +{state_dict["pos_embed"].shape} to {self.pos_embed.shape}') + h, w = self.img_size + pos_size = int( + math.sqrt(state_dict['pos_embed'].shape[1] - 1)) + state_dict['pos_embed'] = self.resize_pos_embed( + state_dict['pos_embed'], (h, w), (pos_size, pos_size), + self.patch_size, self.interpolate_mode) + + self.load_state_dict(state_dict, False) + + elif pretrained is None: + # We only implement the 'jax_impl' initialization implemented at + # https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/vision_transformer.py#L353 # noqa: E501 + trunc_normal_(self.pos_embed, std=.02) + trunc_normal_(self.cls_token, std=.02) + for n, m in self.named_modules(): + if isinstance(m, Linear): + trunc_normal_(m.weight, std=.02) + if m.bias is not None: + if 'mlp' in n: + normal_init(m.bias, std=1e-6) + else: + constant_init(m.bias, 0) + elif isinstance(m, Conv2d): + kaiming_init(m.weight, mode='fan_in') + if m.bias is not None: + constant_init(m.bias, 0) + elif isinstance(m, (_BatchNorm, nn.GroupNorm, nn.LayerNorm)): + constant_init(m.bias, 0) + constant_init(m.weight, 1.0) + else: + raise TypeError('pretrained must be a str or None') + + def _pos_embeding(self, img, patched_img, pos_embed): + """Positiong embeding method. + + Resize the pos_embed, if the input image size doesn't match + the training size. + Args: + img (torch.Tensor): The inference image tensor, the shape + must be [B, C, H, W]. + patched_img (torch.Tensor): The patched image, it should be + shape of [B, L1, C]. + pos_embed (torch.Tensor): The pos_embed weighs, it should be + shape of [B, L2, c]. + Return: + torch.Tensor: The pos encoded image feature. + """ + assert patched_img.ndim == 3 and pos_embed.ndim == 3, \ + 'the shapes of patched_img and pos_embed must be [B, L, C]' + x_len, pos_len = patched_img.shape[1], pos_embed.shape[1] + if x_len != pos_len: + if pos_len == (self.img_size[0] // self.patch_size) * ( + self.img_size[1] // self.patch_size) + 1: + pos_h = self.img_size[0] // self.patch_size + pos_w = self.img_size[1] // self.patch_size + else: + raise ValueError( + 'Unexpected shape of pos_embed, got {}.'.format( + pos_embed.shape)) + pos_embed = self.resize_pos_embed(pos_embed, img.shape[2:], + (pos_h, pos_w), self.patch_size, + self.interpolate_mode) + return self.pos_drop(patched_img + pos_embed) + + @staticmethod + def resize_pos_embed(pos_embed, input_shpae, pos_shape, patch_size, mode): + """Resize pos_embed weights. + + Resize pos_embed using bicubic interpolate method. + Args: + pos_embed (torch.Tensor): pos_embed weights. + input_shpae (tuple): Tuple for (input_h, intput_w). + pos_shape (tuple): Tuple for (pos_h, pos_w). + patch_size (int): Patch size. + Return: + torch.Tensor: The resized pos_embed of shape [B, L_new, C] + """ + assert pos_embed.ndim == 3, 'shape of pos_embed must be [B, L, C]' + input_h, input_w = input_shpae + pos_h, pos_w = pos_shape + cls_token_weight = pos_embed[:, 0] + pos_embed_weight = pos_embed[:, (-1 * pos_h * pos_w):] + pos_embed_weight = pos_embed_weight.reshape( + 1, pos_h, pos_w, pos_embed.shape[2]).permute(0, 3, 1, 2) + pos_embed_weight = F.interpolate( + pos_embed_weight, + size=[input_h // patch_size, input_w // patch_size], + align_corners=False, + mode=mode) + cls_token_weight = cls_token_weight.unsqueeze(1) + pos_embed_weight = torch.flatten(pos_embed_weight, 2).transpose(1, 2) + pos_embed = torch.cat((cls_token_weight, pos_embed_weight), dim=1) + return pos_embed + + def forward(self, inputs): + B = inputs.shape[0] + + x = self.patch_embed(inputs) + + cls_tokens = self.cls_token.expand(B, -1, -1) + x = torch.cat((cls_tokens, x), dim=1) + x = self._pos_embeding(inputs, x, self.pos_embed) + + if not self.with_cls_token: + # Remove class token for transformer input + x = x[:, 1:] + + outs = [] + for i, blk in enumerate(self.blocks): + x = blk(x) + if i == len(self.blocks) - 1: + if self.final_norm: + x = self.norm(x) + if i in self.out_indices: + if self.with_cls_token: + # Remove class token and reshape token for decoder head + out = x[:, 1:] + else: + out = x + if self.out_shape == 'NCHW': + B, _, C = out.shape + out = out.reshape(B, inputs.shape[2] // self.patch_size, + inputs.shape[3] // self.patch_size, + C).permute(0, 3, 1, 2) + outs.append(out) + + return tuple(outs) + + def train(self, mode=True): + super(VisionTransformer, self).train(mode) + if mode and self.norm_eval: + for m in self.modules(): + if isinstance(m, nn.LayerNorm): + m.eval() diff --git a/segmentation/mmseg/models/builder.py b/segmentation/mmseg/models/builder.py new file mode 100644 index 0000000..9b68ff8 --- /dev/null +++ b/segmentation/mmseg/models/builder.py @@ -0,0 +1,46 @@ +import warnings + +from mmcv.cnn import MODELS as MMCV_MODELS +from mmcv.utils import Registry + +MODELS = Registry('models', parent=MMCV_MODELS) + +BACKBONES = MODELS +NECKS = MODELS +HEADS = MODELS +LOSSES = MODELS +SEGMENTORS = MODELS + + +def build_backbone(cfg): + """Build backbone.""" + return BACKBONES.build(cfg) + + +def build_neck(cfg): + """Build neck.""" + return NECKS.build(cfg) + + +def build_head(cfg): + """Build head.""" + return HEADS.build(cfg) + + +def build_loss(cfg): + """Build loss.""" + return LOSSES.build(cfg) + + +def build_segmentor(cfg, train_cfg=None, test_cfg=None): + """Build segmentor.""" + if train_cfg is not None or test_cfg is not None: + warnings.warn( + 'train_cfg and test_cfg is deprecated, ' + 'please specify them in model', UserWarning) + assert cfg.get('train_cfg') is None or train_cfg is None, \ + 'train_cfg specified in both outer field and model field ' + assert cfg.get('test_cfg') is None or test_cfg is None, \ + 'test_cfg specified in both outer field and model field ' + return SEGMENTORS.build( + cfg, default_args=dict(train_cfg=train_cfg, test_cfg=test_cfg)) diff --git a/segmentation/mmseg/models/decode_heads/__init__.py b/segmentation/mmseg/models/decode_heads/__init__.py new file mode 100644 index 0000000..662aae3 --- /dev/null +++ b/segmentation/mmseg/models/decode_heads/__init__.py @@ -0,0 +1,28 @@ +from .ann_head import ANNHead +from .apc_head import APCHead +from .aspp_head import ASPPHead +from .cc_head import CCHead +from .da_head import DAHead +from .dm_head import DMHead +from .dnl_head import DNLHead +from .ema_head import EMAHead +from .enc_head import EncHead +from .fcn_head import FCNHead +from .fpn_head import FPNHead +from .gc_head import GCHead +from .lraspp_head import LRASPPHead +from .nl_head import NLHead +from .ocr_head import OCRHead +from .point_head import PointHead +from .psa_head import PSAHead +from .psp_head import PSPHead +from .sep_aspp_head import DepthwiseSeparableASPPHead +from .sep_fcn_head import DepthwiseSeparableFCNHead +from .uper_head import UPerHead + +__all__ = [ + 'FCNHead', 'PSPHead', 'ASPPHead', 'PSAHead', 'NLHead', 'GCHead', 'CCHead', + 'UPerHead', 'DepthwiseSeparableASPPHead', 'ANNHead', 'DAHead', 'OCRHead', + 'EncHead', 'DepthwiseSeparableFCNHead', 'FPNHead', 'EMAHead', 'DNLHead', + 'PointHead', 'APCHead', 'DMHead', 'LRASPPHead' +] diff --git a/segmentation/mmseg/models/decode_heads/ann_head.py b/segmentation/mmseg/models/decode_heads/ann_head.py new file mode 100644 index 0000000..396c54e --- /dev/null +++ b/segmentation/mmseg/models/decode_heads/ann_head.py @@ -0,0 +1,245 @@ +import torch +import torch.nn as nn +from mmcv.cnn import ConvModule + +from ..builder import HEADS +from ..utils import SelfAttentionBlock as _SelfAttentionBlock +from .decode_head import BaseDecodeHead + + +class PPMConcat(nn.ModuleList): + """Pyramid Pooling Module that only concat the features of each layer. + + Args: + pool_scales (tuple[int]): Pooling scales used in Pooling Pyramid + Module. + """ + + def __init__(self, pool_scales=(1, 3, 6, 8)): + super(PPMConcat, self).__init__( + [nn.AdaptiveAvgPool2d(pool_scale) for pool_scale in pool_scales]) + + def forward(self, feats): + """Forward function.""" + ppm_outs = [] + for ppm in self: + ppm_out = ppm(feats) + ppm_outs.append(ppm_out.view(*feats.shape[:2], -1)) + concat_outs = torch.cat(ppm_outs, dim=2) + return concat_outs + + +class SelfAttentionBlock(_SelfAttentionBlock): + """Make a ANN used SelfAttentionBlock. + + Args: + low_in_channels (int): Input channels of lower level feature, + which is the key feature for self-attention. + high_in_channels (int): Input channels of higher level feature, + which is the query feature for self-attention. + channels (int): Output channels of key/query transform. + out_channels (int): Output channels. + share_key_query (bool): Whether share projection weight between key + and query projection. + query_scale (int): The scale of query feature map. + key_pool_scales (tuple[int]): Pooling scales used in Pooling Pyramid + Module of key feature. + conv_cfg (dict|None): Config of conv layers. + norm_cfg (dict|None): Config of norm layers. + act_cfg (dict|None): Config of activation layers. + """ + + def __init__(self, low_in_channels, high_in_channels, channels, + out_channels, share_key_query, query_scale, key_pool_scales, + conv_cfg, norm_cfg, act_cfg): + key_psp = PPMConcat(key_pool_scales) + if query_scale > 1: + query_downsample = nn.MaxPool2d(kernel_size=query_scale) + else: + query_downsample = None + super(SelfAttentionBlock, self).__init__( + key_in_channels=low_in_channels, + query_in_channels=high_in_channels, + channels=channels, + out_channels=out_channels, + share_key_query=share_key_query, + query_downsample=query_downsample, + key_downsample=key_psp, + key_query_num_convs=1, + key_query_norm=True, + value_out_num_convs=1, + value_out_norm=False, + matmul_norm=True, + with_out=True, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + + +class AFNB(nn.Module): + """Asymmetric Fusion Non-local Block(AFNB) + + Args: + low_in_channels (int): Input channels of lower level feature, + which is the key feature for self-attention. + high_in_channels (int): Input channels of higher level feature, + which is the query feature for self-attention. + channels (int): Output channels of key/query transform. + out_channels (int): Output channels. + and query projection. + query_scales (tuple[int]): The scales of query feature map. + Default: (1,) + key_pool_scales (tuple[int]): Pooling scales used in Pooling Pyramid + Module of key feature. + conv_cfg (dict|None): Config of conv layers. + norm_cfg (dict|None): Config of norm layers. + act_cfg (dict|None): Config of activation layers. + """ + + def __init__(self, low_in_channels, high_in_channels, channels, + out_channels, query_scales, key_pool_scales, conv_cfg, + norm_cfg, act_cfg): + super(AFNB, self).__init__() + self.stages = nn.ModuleList() + for query_scale in query_scales: + self.stages.append( + SelfAttentionBlock( + low_in_channels=low_in_channels, + high_in_channels=high_in_channels, + channels=channels, + out_channels=out_channels, + share_key_query=False, + query_scale=query_scale, + key_pool_scales=key_pool_scales, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg)) + self.bottleneck = ConvModule( + out_channels + high_in_channels, + out_channels, + 1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=None) + + def forward(self, low_feats, high_feats): + """Forward function.""" + priors = [stage(high_feats, low_feats) for stage in self.stages] + context = torch.stack(priors, dim=0).sum(dim=0) + output = self.bottleneck(torch.cat([context, high_feats], 1)) + return output + + +class APNB(nn.Module): + """Asymmetric Pyramid Non-local Block (APNB) + + Args: + in_channels (int): Input channels of key/query feature, + which is the key feature for self-attention. + channels (int): Output channels of key/query transform. + out_channels (int): Output channels. + query_scales (tuple[int]): The scales of query feature map. + Default: (1,) + key_pool_scales (tuple[int]): Pooling scales used in Pooling Pyramid + Module of key feature. + conv_cfg (dict|None): Config of conv layers. + norm_cfg (dict|None): Config of norm layers. + act_cfg (dict|None): Config of activation layers. + """ + + def __init__(self, in_channels, channels, out_channels, query_scales, + key_pool_scales, conv_cfg, norm_cfg, act_cfg): + super(APNB, self).__init__() + self.stages = nn.ModuleList() + for query_scale in query_scales: + self.stages.append( + SelfAttentionBlock( + low_in_channels=in_channels, + high_in_channels=in_channels, + channels=channels, + out_channels=out_channels, + share_key_query=True, + query_scale=query_scale, + key_pool_scales=key_pool_scales, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg)) + self.bottleneck = ConvModule( + 2 * in_channels, + out_channels, + 1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + + def forward(self, feats): + """Forward function.""" + priors = [stage(feats, feats) for stage in self.stages] + context = torch.stack(priors, dim=0).sum(dim=0) + output = self.bottleneck(torch.cat([context, feats], 1)) + return output + + +@HEADS.register_module() +class ANNHead(BaseDecodeHead): + """Asymmetric Non-local Neural Networks for Semantic Segmentation. + + This head is the implementation of `ANNNet + `_. + + Args: + project_channels (int): Projection channels for Nonlocal. + query_scales (tuple[int]): The scales of query feature map. + Default: (1,) + key_pool_scales (tuple[int]): The pooling scales of key feature map. + Default: (1, 3, 6, 8). + """ + + def __init__(self, + project_channels, + query_scales=(1, ), + key_pool_scales=(1, 3, 6, 8), + **kwargs): + super(ANNHead, self).__init__( + input_transform='multiple_select', **kwargs) + assert len(self.in_channels) == 2 + low_in_channels, high_in_channels = self.in_channels + self.project_channels = project_channels + self.fusion = AFNB( + low_in_channels=low_in_channels, + high_in_channels=high_in_channels, + out_channels=high_in_channels, + channels=project_channels, + query_scales=query_scales, + key_pool_scales=key_pool_scales, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.bottleneck = ConvModule( + high_in_channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.context = APNB( + in_channels=self.channels, + out_channels=self.channels, + channels=project_channels, + query_scales=query_scales, + key_pool_scales=key_pool_scales, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, inputs): + """Forward function.""" + low_feats, high_feats = self._transform_inputs(inputs) + output = self.fusion(low_feats, high_feats) + output = self.dropout(output) + output = self.bottleneck(output) + output = self.context(output) + output = self.cls_seg(output) + + return output diff --git a/segmentation/mmseg/models/decode_heads/apc_head.py b/segmentation/mmseg/models/decode_heads/apc_head.py new file mode 100644 index 0000000..2118232 --- /dev/null +++ b/segmentation/mmseg/models/decode_heads/apc_head.py @@ -0,0 +1,158 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import ConvModule + +from mmseg.ops import resize +from ..builder import HEADS +from .decode_head import BaseDecodeHead + + +class ACM(nn.Module): + """Adaptive Context Module used in APCNet. + + Args: + pool_scale (int): Pooling scale used in Adaptive Context + Module to extract region features. + fusion (bool): Add one conv to fuse residual feature. + in_channels (int): Input channels. + channels (int): Channels after modules, before conv_seg. + conv_cfg (dict | None): Config of conv layers. + norm_cfg (dict | None): Config of norm layers. + act_cfg (dict): Config of activation layers. + """ + + def __init__(self, pool_scale, fusion, in_channels, channels, conv_cfg, + norm_cfg, act_cfg): + super(ACM, self).__init__() + self.pool_scale = pool_scale + self.fusion = fusion + self.in_channels = in_channels + self.channels = channels + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + self.pooled_redu_conv = ConvModule( + self.in_channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + self.input_redu_conv = ConvModule( + self.in_channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + self.global_info = ConvModule( + self.channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + self.gla = nn.Conv2d(self.channels, self.pool_scale**2, 1, 1, 0) + + self.residual_conv = ConvModule( + self.channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + if self.fusion: + self.fusion_conv = ConvModule( + self.channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, x): + """Forward function.""" + pooled_x = F.adaptive_avg_pool2d(x, self.pool_scale) + # [batch_size, channels, h, w] + x = self.input_redu_conv(x) + # [batch_size, channels, pool_scale, pool_scale] + pooled_x = self.pooled_redu_conv(pooled_x) + batch_size = x.size(0) + # [batch_size, pool_scale * pool_scale, channels] + pooled_x = pooled_x.view(batch_size, self.channels, + -1).permute(0, 2, 1).contiguous() + # [batch_size, h * w, pool_scale * pool_scale] + affinity_matrix = self.gla(x + resize( + self.global_info(F.adaptive_avg_pool2d(x, 1)), size=x.shape[2:]) + ).permute(0, 2, 3, 1).reshape( + batch_size, -1, self.pool_scale**2) + affinity_matrix = F.sigmoid(affinity_matrix) + # [batch_size, h * w, channels] + z_out = torch.matmul(affinity_matrix, pooled_x) + # [batch_size, channels, h * w] + z_out = z_out.permute(0, 2, 1).contiguous() + # [batch_size, channels, h, w] + z_out = z_out.view(batch_size, self.channels, x.size(2), x.size(3)) + z_out = self.residual_conv(z_out) + z_out = F.relu(z_out + x) + if self.fusion: + z_out = self.fusion_conv(z_out) + + return z_out + + +@HEADS.register_module() +class APCHead(BaseDecodeHead): + """Adaptive Pyramid Context Network for Semantic Segmentation. + + This head is the implementation of + `APCNet `_. + + Args: + pool_scales (tuple[int]): Pooling scales used in Adaptive Context + Module. Default: (1, 2, 3, 6). + fusion (bool): Add one conv to fuse residual feature. + """ + + def __init__(self, pool_scales=(1, 2, 3, 6), fusion=True, **kwargs): + super(APCHead, self).__init__(**kwargs) + assert isinstance(pool_scales, (list, tuple)) + self.pool_scales = pool_scales + self.fusion = fusion + acm_modules = [] + for pool_scale in self.pool_scales: + acm_modules.append( + ACM(pool_scale, + self.fusion, + self.in_channels, + self.channels, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg)) + self.acm_modules = nn.ModuleList(acm_modules) + self.bottleneck = ConvModule( + self.in_channels + len(pool_scales) * self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + acm_outs = [x] + for acm_module in self.acm_modules: + acm_outs.append(acm_module(x)) + acm_outs = torch.cat(acm_outs, dim=1) + output = self.bottleneck(acm_outs) + output = self.cls_seg(output) + return output diff --git a/segmentation/mmseg/models/decode_heads/aspp_head.py b/segmentation/mmseg/models/decode_heads/aspp_head.py new file mode 100644 index 0000000..6332ab1 --- /dev/null +++ b/segmentation/mmseg/models/decode_heads/aspp_head.py @@ -0,0 +1,107 @@ +import torch +import torch.nn as nn +from mmcv.cnn import ConvModule + +from mmseg.ops import resize +from ..builder import HEADS +from .decode_head import BaseDecodeHead + + +class ASPPModule(nn.ModuleList): + """Atrous Spatial Pyramid Pooling (ASPP) Module. + + Args: + dilations (tuple[int]): Dilation rate of each layer. + in_channels (int): Input channels. + channels (int): Channels after modules, before conv_seg. + conv_cfg (dict|None): Config of conv layers. + norm_cfg (dict|None): Config of norm layers. + act_cfg (dict): Config of activation layers. + """ + + def __init__(self, dilations, in_channels, channels, conv_cfg, norm_cfg, + act_cfg): + super(ASPPModule, self).__init__() + self.dilations = dilations + self.in_channels = in_channels + self.channels = channels + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + for dilation in dilations: + self.append( + ConvModule( + self.in_channels, + self.channels, + 1 if dilation == 1 else 3, + dilation=dilation, + padding=0 if dilation == 1 else dilation, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg)) + + def forward(self, x): + """Forward function.""" + aspp_outs = [] + for aspp_module in self: + aspp_outs.append(aspp_module(x)) + + return aspp_outs + + +@HEADS.register_module() +class ASPPHead(BaseDecodeHead): + """Rethinking Atrous Convolution for Semantic Image Segmentation. + + This head is the implementation of `DeepLabV3 + `_. + + Args: + dilations (tuple[int]): Dilation rates for ASPP module. + Default: (1, 6, 12, 18). + """ + + def __init__(self, dilations=(1, 6, 12, 18), **kwargs): + super(ASPPHead, self).__init__(**kwargs) + assert isinstance(dilations, (list, tuple)) + self.dilations = dilations + self.image_pool = nn.Sequential( + nn.AdaptiveAvgPool2d(1), + ConvModule( + self.in_channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg)) + self.aspp_modules = ASPPModule( + dilations, + self.in_channels, + self.channels, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.bottleneck = ConvModule( + (len(dilations) + 1) * self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + aspp_outs = [ + resize( + self.image_pool(x), + size=x.size()[2:], + mode='bilinear', + align_corners=self.align_corners) + ] + aspp_outs.extend(self.aspp_modules(x)) + aspp_outs = torch.cat(aspp_outs, dim=1) + output = self.bottleneck(aspp_outs) + output = self.cls_seg(output) + return output diff --git a/segmentation/mmseg/models/decode_heads/cascade_decode_head.py b/segmentation/mmseg/models/decode_heads/cascade_decode_head.py new file mode 100644 index 0000000..d02122c --- /dev/null +++ b/segmentation/mmseg/models/decode_heads/cascade_decode_head.py @@ -0,0 +1,57 @@ +from abc import ABCMeta, abstractmethod + +from .decode_head import BaseDecodeHead + + +class BaseCascadeDecodeHead(BaseDecodeHead, metaclass=ABCMeta): + """Base class for cascade decode head used in + :class:`CascadeEncoderDecoder.""" + + def __init__(self, *args, **kwargs): + super(BaseCascadeDecodeHead, self).__init__(*args, **kwargs) + + @abstractmethod + def forward(self, inputs, prev_output): + """Placeholder of forward function.""" + pass + + def forward_train(self, inputs, prev_output, img_metas, gt_semantic_seg, + train_cfg): + """Forward function for training. + Args: + inputs (list[Tensor]): List of multi-level img features. + prev_output (Tensor): The output of previous decode head. + img_metas (list[dict]): List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + gt_semantic_seg (Tensor): Semantic segmentation masks + used if the architecture supports semantic segmentation task. + train_cfg (dict): The training config. + + Returns: + dict[str, Tensor]: a dictionary of loss components + """ + seg_logits = self.forward(inputs, prev_output) + losses = self.losses(seg_logits, gt_semantic_seg) + + return losses + + def forward_test(self, inputs, prev_output, img_metas, test_cfg): + """Forward function for testing. + + Args: + inputs (list[Tensor]): List of multi-level img features. + prev_output (Tensor): The output of previous decode head. + img_metas (list[dict]): List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + test_cfg (dict): The testing config. + + Returns: + Tensor: Output segmentation map. + """ + return self.forward(inputs, prev_output) diff --git a/segmentation/mmseg/models/decode_heads/cc_head.py b/segmentation/mmseg/models/decode_heads/cc_head.py new file mode 100644 index 0000000..95c2706 --- /dev/null +++ b/segmentation/mmseg/models/decode_heads/cc_head.py @@ -0,0 +1,42 @@ +import torch + +from ..builder import HEADS +from .fcn_head import FCNHead + +try: + from mmcv.ops import CrissCrossAttention +except ModuleNotFoundError: + CrissCrossAttention = None + + +@HEADS.register_module() +class CCHead(FCNHead): + """CCNet: Criss-Cross Attention for Semantic Segmentation. + + This head is the implementation of `CCNet + `_. + + Args: + recurrence (int): Number of recurrence of Criss Cross Attention + module. Default: 2. + """ + + def __init__(self, recurrence=2, **kwargs): + if CrissCrossAttention is None: + raise RuntimeError('Please install mmcv-full for ' + 'CrissCrossAttention ops') + super(CCHead, self).__init__(num_convs=2, **kwargs) + self.recurrence = recurrence + self.cca = CrissCrossAttention(self.channels) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + output = self.convs[0](x) + for _ in range(self.recurrence): + output = self.cca(output) + output = self.convs[1](output) + if self.concat_input: + output = self.conv_cat(torch.cat([x, output], dim=1)) + output = self.cls_seg(output) + return output diff --git a/segmentation/mmseg/models/decode_heads/da_head.py b/segmentation/mmseg/models/decode_heads/da_head.py new file mode 100644 index 0000000..8ee0e08 --- /dev/null +++ b/segmentation/mmseg/models/decode_heads/da_head.py @@ -0,0 +1,178 @@ +import torch +import torch.nn.functional as F +from mmcv.cnn import ConvModule, Scale +from torch import nn + +from mmseg.core import add_prefix +from ..builder import HEADS +from ..utils import SelfAttentionBlock as _SelfAttentionBlock +from .decode_head import BaseDecodeHead + + +class PAM(_SelfAttentionBlock): + """Position Attention Module (PAM) + + Args: + in_channels (int): Input channels of key/query feature. + channels (int): Output channels of key/query transform. + """ + + def __init__(self, in_channels, channels): + super(PAM, self).__init__( + key_in_channels=in_channels, + query_in_channels=in_channels, + channels=channels, + out_channels=in_channels, + share_key_query=False, + query_downsample=None, + key_downsample=None, + key_query_num_convs=1, + key_query_norm=False, + value_out_num_convs=1, + value_out_norm=False, + matmul_norm=False, + with_out=False, + conv_cfg=None, + norm_cfg=None, + act_cfg=None) + + self.gamma = Scale(0) + + def forward(self, x): + """Forward function.""" + out = super(PAM, self).forward(x, x) + + out = self.gamma(out) + x + return out + + +class CAM(nn.Module): + """Channel Attention Module (CAM)""" + + def __init__(self): + super(CAM, self).__init__() + self.gamma = Scale(0) + + def forward(self, x): + """Forward function.""" + batch_size, channels, height, width = x.size() + proj_query = x.view(batch_size, channels, -1) + proj_key = x.view(batch_size, channels, -1).permute(0, 2, 1) + energy = torch.bmm(proj_query, proj_key) + energy_new = torch.max( + energy, -1, keepdim=True)[0].expand_as(energy) - energy + attention = F.softmax(energy_new, dim=-1) + proj_value = x.view(batch_size, channels, -1) + + out = torch.bmm(attention, proj_value) + out = out.view(batch_size, channels, height, width) + + out = self.gamma(out) + x + return out + + +@HEADS.register_module() +class DAHead(BaseDecodeHead): + """Dual Attention Network for Scene Segmentation. + + This head is the implementation of `DANet + `_. + + Args: + pam_channels (int): The channels of Position Attention Module(PAM). + """ + + def __init__(self, pam_channels, **kwargs): + super(DAHead, self).__init__(**kwargs) + self.pam_channels = pam_channels + self.pam_in_conv = ConvModule( + self.in_channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.pam = PAM(self.channels, pam_channels) + self.pam_out_conv = ConvModule( + self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.pam_conv_seg = nn.Conv2d( + self.channels, self.num_classes, kernel_size=1) + + self.cam_in_conv = ConvModule( + self.in_channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.cam = CAM() + self.cam_out_conv = ConvModule( + self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.cam_conv_seg = nn.Conv2d( + self.channels, self.num_classes, kernel_size=1) + + def pam_cls_seg(self, feat): + """PAM feature classification.""" + if self.dropout is not None: + feat = self.dropout(feat) + output = self.pam_conv_seg(feat) + return output + + def cam_cls_seg(self, feat): + """CAM feature classification.""" + if self.dropout is not None: + feat = self.dropout(feat) + output = self.cam_conv_seg(feat) + return output + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + pam_feat = self.pam_in_conv(x) + pam_feat = self.pam(pam_feat) + pam_feat = self.pam_out_conv(pam_feat) + pam_out = self.pam_cls_seg(pam_feat) + + cam_feat = self.cam_in_conv(x) + cam_feat = self.cam(cam_feat) + cam_feat = self.cam_out_conv(cam_feat) + cam_out = self.cam_cls_seg(cam_feat) + + feat_sum = pam_feat + cam_feat + pam_cam_out = self.cls_seg(feat_sum) + + return pam_cam_out, pam_out, cam_out + + def forward_test(self, inputs, img_metas, test_cfg): + """Forward function for testing, only ``pam_cam`` is used.""" + return self.forward(inputs)[0] + + def losses(self, seg_logit, seg_label): + """Compute ``pam_cam``, ``pam``, ``cam`` loss.""" + pam_cam_seg_logit, pam_seg_logit, cam_seg_logit = seg_logit + loss = dict() + loss.update( + add_prefix( + super(DAHead, self).losses(pam_cam_seg_logit, seg_label), + 'pam_cam')) + loss.update( + add_prefix( + super(DAHead, self).losses(pam_seg_logit, seg_label), 'pam')) + loss.update( + add_prefix( + super(DAHead, self).losses(cam_seg_logit, seg_label), 'cam')) + return loss diff --git a/segmentation/mmseg/models/decode_heads/decode_head.py b/segmentation/mmseg/models/decode_heads/decode_head.py new file mode 100644 index 0000000..86b9b63 --- /dev/null +++ b/segmentation/mmseg/models/decode_heads/decode_head.py @@ -0,0 +1,234 @@ +from abc import ABCMeta, abstractmethod + +import torch +import torch.nn as nn +from mmcv.cnn import normal_init +from mmcv.runner import auto_fp16, force_fp32 + +from mmseg.core import build_pixel_sampler +from mmseg.ops import resize +from ..builder import build_loss +from ..losses import accuracy + + +class BaseDecodeHead(nn.Module, metaclass=ABCMeta): + """Base class for BaseDecodeHead. + + Args: + in_channels (int|Sequence[int]): Input channels. + channels (int): Channels after modules, before conv_seg. + num_classes (int): Number of classes. + dropout_ratio (float): Ratio of dropout layer. Default: 0.1. + conv_cfg (dict|None): Config of conv layers. Default: None. + norm_cfg (dict|None): Config of norm layers. Default: None. + act_cfg (dict): Config of activation layers. + Default: dict(type='ReLU') + in_index (int|Sequence[int]): Input feature index. Default: -1 + input_transform (str|None): Transformation type of input features. + Options: 'resize_concat', 'multiple_select', None. + 'resize_concat': Multiple feature maps will be resize to the + same size as first one and than concat together. + Usually used in FCN head of HRNet. + 'multiple_select': Multiple feature maps will be bundle into + a list and passed into decode head. + None: Only one select feature map is allowed. + Default: None. + loss_decode (dict): Config of decode loss. + Default: dict(type='CrossEntropyLoss'). + ignore_index (int | None): The label index to be ignored. When using + masked BCE loss, ignore_index should be set to None. Default: 255 + sampler (dict|None): The config of segmentation map sampler. + Default: None. + align_corners (bool): align_corners argument of F.interpolate. + Default: False. + """ + + def __init__(self, + in_channels, + channels, + *, + num_classes, + dropout_ratio=0.1, + conv_cfg=None, + norm_cfg=None, + act_cfg=dict(type='ReLU'), + in_index=-1, + input_transform=None, + loss_decode=dict( + type='CrossEntropyLoss', + use_sigmoid=False, + loss_weight=1.0), + ignore_index=255, + sampler=None, + align_corners=False): + super(BaseDecodeHead, self).__init__() + self._init_inputs(in_channels, in_index, input_transform) + self.channels = channels + self.num_classes = num_classes + self.dropout_ratio = dropout_ratio + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + self.in_index = in_index + self.loss_decode = build_loss(loss_decode) + self.ignore_index = ignore_index + self.align_corners = align_corners + if sampler is not None: + self.sampler = build_pixel_sampler(sampler, context=self) + else: + self.sampler = None + + self.conv_seg = nn.Conv2d(channels, num_classes, kernel_size=1) + if dropout_ratio > 0: + self.dropout = nn.Dropout2d(dropout_ratio) + else: + self.dropout = None + self.fp16_enabled = False + + def extra_repr(self): + """Extra repr.""" + s = f'input_transform={self.input_transform}, ' \ + f'ignore_index={self.ignore_index}, ' \ + f'align_corners={self.align_corners}' + return s + + def _init_inputs(self, in_channels, in_index, input_transform): + """Check and initialize input transforms. + + The in_channels, in_index and input_transform must match. + Specifically, when input_transform is None, only single feature map + will be selected. So in_channels and in_index must be of type int. + When input_transform + + Args: + in_channels (int|Sequence[int]): Input channels. + in_index (int|Sequence[int]): Input feature index. + input_transform (str|None): Transformation type of input features. + Options: 'resize_concat', 'multiple_select', None. + 'resize_concat': Multiple feature maps will be resize to the + same size as first one and than concat together. + Usually used in FCN head of HRNet. + 'multiple_select': Multiple feature maps will be bundle into + a list and passed into decode head. + None: Only one select feature map is allowed. + """ + + if input_transform is not None: + assert input_transform in ['resize_concat', 'multiple_select'] + self.input_transform = input_transform + self.in_index = in_index + if input_transform is not None: + assert isinstance(in_channels, (list, tuple)) + assert isinstance(in_index, (list, tuple)) + assert len(in_channels) == len(in_index) + if input_transform == 'resize_concat': + self.in_channels = sum(in_channels) + else: + self.in_channels = in_channels + else: + assert isinstance(in_channels, int) + assert isinstance(in_index, int) + self.in_channels = in_channels + + def init_weights(self): + """Initialize weights of classification layer.""" + normal_init(self.conv_seg, mean=0, std=0.01) + + def _transform_inputs(self, inputs): + """Transform inputs for decoder. + + Args: + inputs (list[Tensor]): List of multi-level img features. + + Returns: + Tensor: The transformed inputs + """ + + if self.input_transform == 'resize_concat': + inputs = [inputs[i] for i in self.in_index] + upsampled_inputs = [ + resize( + input=x, + size=inputs[0].shape[2:], + mode='bilinear', + align_corners=self.align_corners) for x in inputs + ] + inputs = torch.cat(upsampled_inputs, dim=1) + elif self.input_transform == 'multiple_select': + inputs = [inputs[i] for i in self.in_index] + else: + inputs = inputs[self.in_index] + + return inputs + + @auto_fp16() + @abstractmethod + def forward(self, inputs): + """Placeholder of forward function.""" + pass + + def forward_train(self, inputs, img_metas, gt_semantic_seg, train_cfg): + """Forward function for training. + Args: + inputs (list[Tensor]): List of multi-level img features. + img_metas (list[dict]): List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + gt_semantic_seg (Tensor): Semantic segmentation masks + used if the architecture supports semantic segmentation task. + train_cfg (dict): The training config. + + Returns: + dict[str, Tensor]: a dictionary of loss components + """ + seg_logits = self.forward(inputs) + losses = self.losses(seg_logits, gt_semantic_seg) + return losses + + def forward_test(self, inputs, img_metas, test_cfg): + """Forward function for testing. + + Args: + inputs (list[Tensor]): List of multi-level img features. + img_metas (list[dict]): List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + test_cfg (dict): The testing config. + + Returns: + Tensor: Output segmentation map. + """ + return self.forward(inputs) + + def cls_seg(self, feat): + """Classify each pixel.""" + if self.dropout is not None: + feat = self.dropout(feat) + output = self.conv_seg(feat) + return output + + @force_fp32(apply_to=('seg_logit', )) + def losses(self, seg_logit, seg_label): + """Compute segmentation loss.""" + loss = dict() + seg_logit = resize( + input=seg_logit, + size=seg_label.shape[2:], + mode='bilinear', + align_corners=self.align_corners) + if self.sampler is not None: + seg_weight = self.sampler.sample(seg_logit, seg_label) + else: + seg_weight = None + seg_label = seg_label.squeeze(1) + loss['loss_seg'] = self.loss_decode( + seg_logit, + seg_label, + weight=seg_weight, + ignore_index=self.ignore_index) + loss['acc_seg'] = accuracy(seg_logit, seg_label) + return loss diff --git a/segmentation/mmseg/models/decode_heads/dm_head.py b/segmentation/mmseg/models/decode_heads/dm_head.py new file mode 100644 index 0000000..3161b06 --- /dev/null +++ b/segmentation/mmseg/models/decode_heads/dm_head.py @@ -0,0 +1,140 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import ConvModule, build_activation_layer, build_norm_layer + +from ..builder import HEADS +from .decode_head import BaseDecodeHead + + +class DCM(nn.Module): + """Dynamic Convolutional Module used in DMNet. + + Args: + filter_size (int): The filter size of generated convolution kernel + used in Dynamic Convolutional Module. + fusion (bool): Add one conv to fuse DCM output feature. + in_channels (int): Input channels. + channels (int): Channels after modules, before conv_seg. + conv_cfg (dict | None): Config of conv layers. + norm_cfg (dict | None): Config of norm layers. + act_cfg (dict): Config of activation layers. + """ + + def __init__(self, filter_size, fusion, in_channels, channels, conv_cfg, + norm_cfg, act_cfg): + super(DCM, self).__init__() + self.filter_size = filter_size + self.fusion = fusion + self.in_channels = in_channels + self.channels = channels + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + self.filter_gen_conv = nn.Conv2d(self.in_channels, self.channels, 1, 1, + 0) + + self.input_redu_conv = ConvModule( + self.in_channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + if self.norm_cfg is not None: + self.norm = build_norm_layer(self.norm_cfg, self.channels)[1] + else: + self.norm = None + self.activate = build_activation_layer(self.act_cfg) + + if self.fusion: + self.fusion_conv = ConvModule( + self.channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, x): + """Forward function.""" + generated_filter = self.filter_gen_conv( + F.adaptive_avg_pool2d(x, self.filter_size)) + x = self.input_redu_conv(x) + b, c, h, w = x.shape + # [1, b * c, h, w], c = self.channels + x = x.view(1, b * c, h, w) + # [b * c, 1, filter_size, filter_size] + generated_filter = generated_filter.view(b * c, 1, self.filter_size, + self.filter_size) + pad = (self.filter_size - 1) // 2 + if (self.filter_size - 1) % 2 == 0: + p2d = (pad, pad, pad, pad) + else: + p2d = (pad + 1, pad, pad + 1, pad) + x = F.pad(input=x, pad=p2d, mode='constant', value=0) + # [1, b * c, h, w] + output = F.conv2d(input=x, weight=generated_filter, groups=b * c) + # [b, c, h, w] + output = output.view(b, c, h, w) + if self.norm is not None: + output = self.norm(output) + output = self.activate(output) + + if self.fusion: + output = self.fusion_conv(output) + + return output + + +@HEADS.register_module() +class DMHead(BaseDecodeHead): + """Dynamic Multi-scale Filters for Semantic Segmentation. + + This head is the implementation of + `DMNet `_. + + Args: + filter_sizes (tuple[int]): The size of generated convolutional filters + used in Dynamic Convolutional Module. Default: (1, 3, 5, 7). + fusion (bool): Add one conv to fuse DCM output feature. + """ + + def __init__(self, filter_sizes=(1, 3, 5, 7), fusion=False, **kwargs): + super(DMHead, self).__init__(**kwargs) + assert isinstance(filter_sizes, (list, tuple)) + self.filter_sizes = filter_sizes + self.fusion = fusion + dcm_modules = [] + for filter_size in self.filter_sizes: + dcm_modules.append( + DCM(filter_size, + self.fusion, + self.in_channels, + self.channels, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg)) + self.dcm_modules = nn.ModuleList(dcm_modules) + self.bottleneck = ConvModule( + self.in_channels + len(filter_sizes) * self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + dcm_outs = [x] + for dcm_module in self.dcm_modules: + dcm_outs.append(dcm_module(x)) + dcm_outs = torch.cat(dcm_outs, dim=1) + output = self.bottleneck(dcm_outs) + output = self.cls_seg(output) + return output diff --git a/segmentation/mmseg/models/decode_heads/dnl_head.py b/segmentation/mmseg/models/decode_heads/dnl_head.py new file mode 100644 index 0000000..52a662c --- /dev/null +++ b/segmentation/mmseg/models/decode_heads/dnl_head.py @@ -0,0 +1,131 @@ +import torch +from mmcv.cnn import NonLocal2d +from torch import nn + +from ..builder import HEADS +from .fcn_head import FCNHead + + +class DisentangledNonLocal2d(NonLocal2d): + """Disentangled Non-Local Blocks. + + Args: + temperature (float): Temperature to adjust attention. Default: 0.05 + """ + + def __init__(self, *arg, temperature, **kwargs): + super().__init__(*arg, **kwargs) + self.temperature = temperature + self.conv_mask = nn.Conv2d(self.in_channels, 1, kernel_size=1) + + def embedded_gaussian(self, theta_x, phi_x): + """Embedded gaussian with temperature.""" + + # NonLocal2d pairwise_weight: [N, HxW, HxW] + pairwise_weight = torch.matmul(theta_x, phi_x) + if self.use_scale: + # theta_x.shape[-1] is `self.inter_channels` + pairwise_weight /= theta_x.shape[-1]**0.5 + pairwise_weight /= self.temperature + pairwise_weight = pairwise_weight.softmax(dim=-1) + return pairwise_weight + + def forward(self, x): + # x: [N, C, H, W] + n = x.size(0) + + # g_x: [N, HxW, C] + g_x = self.g(x).view(n, self.inter_channels, -1) + g_x = g_x.permute(0, 2, 1) + + # theta_x: [N, HxW, C], phi_x: [N, C, HxW] + if self.mode == 'gaussian': + theta_x = x.view(n, self.in_channels, -1) + theta_x = theta_x.permute(0, 2, 1) + if self.sub_sample: + phi_x = self.phi(x).view(n, self.in_channels, -1) + else: + phi_x = x.view(n, self.in_channels, -1) + elif self.mode == 'concatenation': + theta_x = self.theta(x).view(n, self.inter_channels, -1, 1) + phi_x = self.phi(x).view(n, self.inter_channels, 1, -1) + else: + theta_x = self.theta(x).view(n, self.inter_channels, -1) + theta_x = theta_x.permute(0, 2, 1) + phi_x = self.phi(x).view(n, self.inter_channels, -1) + + # subtract mean + theta_x -= theta_x.mean(dim=-2, keepdim=True) + phi_x -= phi_x.mean(dim=-1, keepdim=True) + + pairwise_func = getattr(self, self.mode) + # pairwise_weight: [N, HxW, HxW] + pairwise_weight = pairwise_func(theta_x, phi_x) + + # y: [N, HxW, C] + y = torch.matmul(pairwise_weight, g_x) + # y: [N, C, H, W] + y = y.permute(0, 2, 1).contiguous().reshape(n, self.inter_channels, + *x.size()[2:]) + + # unary_mask: [N, 1, HxW] + unary_mask = self.conv_mask(x) + unary_mask = unary_mask.view(n, 1, -1) + unary_mask = unary_mask.softmax(dim=-1) + # unary_x: [N, 1, C] + unary_x = torch.matmul(unary_mask, g_x) + # unary_x: [N, C, 1, 1] + unary_x = unary_x.permute(0, 2, 1).contiguous().reshape( + n, self.inter_channels, 1, 1) + + output = x + self.conv_out(y + unary_x) + + return output + + +@HEADS.register_module() +class DNLHead(FCNHead): + """Disentangled Non-Local Neural Networks. + + This head is the implementation of `DNLNet + `_. + + Args: + reduction (int): Reduction factor of projection transform. Default: 2. + use_scale (bool): Whether to scale pairwise_weight by + sqrt(1/inter_channels). Default: False. + mode (str): The nonlocal mode. Options are 'embedded_gaussian', + 'dot_product'. Default: 'embedded_gaussian.'. + temperature (float): Temperature to adjust attention. Default: 0.05 + """ + + def __init__(self, + reduction=2, + use_scale=True, + mode='embedded_gaussian', + temperature=0.05, + **kwargs): + super(DNLHead, self).__init__(num_convs=2, **kwargs) + self.reduction = reduction + self.use_scale = use_scale + self.mode = mode + self.temperature = temperature + self.dnl_block = DisentangledNonLocal2d( + in_channels=self.channels, + reduction=self.reduction, + use_scale=self.use_scale, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + mode=self.mode, + temperature=self.temperature) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + output = self.convs[0](x) + output = self.dnl_block(output) + output = self.convs[1](output) + if self.concat_input: + output = self.conv_cat(torch.cat([x, output], dim=1)) + output = self.cls_seg(output) + return output diff --git a/segmentation/mmseg/models/decode_heads/ema_head.py b/segmentation/mmseg/models/decode_heads/ema_head.py new file mode 100644 index 0000000..619d757 --- /dev/null +++ b/segmentation/mmseg/models/decode_heads/ema_head.py @@ -0,0 +1,168 @@ +import math + +import torch +import torch.distributed as dist +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import ConvModule + +from ..builder import HEADS +from .decode_head import BaseDecodeHead + + +def reduce_mean(tensor): + """Reduce mean when distributed training.""" + if not (dist.is_available() and dist.is_initialized()): + return tensor + tensor = tensor.clone() + dist.all_reduce(tensor.div_(dist.get_world_size()), op=dist.ReduceOp.SUM) + return tensor + + +class EMAModule(nn.Module): + """Expectation Maximization Attention Module used in EMANet. + + Args: + channels (int): Channels of the whole module. + num_bases (int): Number of bases. + num_stages (int): Number of the EM iterations. + """ + + def __init__(self, channels, num_bases, num_stages, momentum): + super(EMAModule, self).__init__() + assert num_stages >= 1, 'num_stages must be at least 1!' + self.num_bases = num_bases + self.num_stages = num_stages + self.momentum = momentum + + bases = torch.zeros(1, channels, self.num_bases) + bases.normal_(0, math.sqrt(2. / self.num_bases)) + # [1, channels, num_bases] + bases = F.normalize(bases, dim=1, p=2) + self.register_buffer('bases', bases) + + def forward(self, feats): + """Forward function.""" + batch_size, channels, height, width = feats.size() + # [batch_size, channels, height*width] + feats = feats.view(batch_size, channels, height * width) + # [batch_size, channels, num_bases] + bases = self.bases.repeat(batch_size, 1, 1) + + with torch.no_grad(): + for i in range(self.num_stages): + # [batch_size, height*width, num_bases] + attention = torch.einsum('bcn,bck->bnk', feats, bases) + attention = F.softmax(attention, dim=2) + # l1 norm + attention_normed = F.normalize(attention, dim=1, p=1) + # [batch_size, channels, num_bases] + bases = torch.einsum('bcn,bnk->bck', feats, attention_normed) + # l2 norm + bases = F.normalize(bases, dim=1, p=2) + + feats_recon = torch.einsum('bck,bnk->bcn', bases, attention) + feats_recon = feats_recon.view(batch_size, channels, height, width) + + if self.training: + bases = bases.mean(dim=0, keepdim=True) + bases = reduce_mean(bases) + # l2 norm + bases = F.normalize(bases, dim=1, p=2) + self.bases = (1 - + self.momentum) * self.bases + self.momentum * bases + + return feats_recon + + +@HEADS.register_module() +class EMAHead(BaseDecodeHead): + """Expectation Maximization Attention Networks for Semantic Segmentation. + + This head is the implementation of `EMANet + `_. + + Args: + ema_channels (int): EMA module channels + num_bases (int): Number of bases. + num_stages (int): Number of the EM iterations. + concat_input (bool): Whether concat the input and output of convs + before classification layer. Default: True + momentum (float): Momentum to update the base. Default: 0.1. + """ + + def __init__(self, + ema_channels, + num_bases, + num_stages, + concat_input=True, + momentum=0.1, + **kwargs): + super(EMAHead, self).__init__(**kwargs) + self.ema_channels = ema_channels + self.num_bases = num_bases + self.num_stages = num_stages + self.concat_input = concat_input + self.momentum = momentum + self.ema_module = EMAModule(self.ema_channels, self.num_bases, + self.num_stages, self.momentum) + + self.ema_in_conv = ConvModule( + self.in_channels, + self.ema_channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + # project (0, inf) -> (-inf, inf) + self.ema_mid_conv = ConvModule( + self.ema_channels, + self.ema_channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=None, + act_cfg=None) + for param in self.ema_mid_conv.parameters(): + param.requires_grad = False + + self.ema_out_conv = ConvModule( + self.ema_channels, + self.ema_channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=None) + self.bottleneck = ConvModule( + self.ema_channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + if self.concat_input: + self.conv_cat = ConvModule( + self.in_channels + self.channels, + self.channels, + kernel_size=3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + feats = self.ema_in_conv(x) + identity = feats + feats = self.ema_mid_conv(feats) + recon = self.ema_module(feats) + recon = F.relu(recon, inplace=True) + recon = self.ema_out_conv(recon) + output = F.relu(identity + recon, inplace=True) + output = self.bottleneck(output) + if self.concat_input: + output = self.conv_cat(torch.cat([x, output], dim=1)) + output = self.cls_seg(output) + return output diff --git a/segmentation/mmseg/models/decode_heads/enc_head.py b/segmentation/mmseg/models/decode_heads/enc_head.py new file mode 100644 index 0000000..0c11994 --- /dev/null +++ b/segmentation/mmseg/models/decode_heads/enc_head.py @@ -0,0 +1,187 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import ConvModule, build_norm_layer + +from mmseg.ops import Encoding, resize +from ..builder import HEADS, build_loss +from .decode_head import BaseDecodeHead + + +class EncModule(nn.Module): + """Encoding Module used in EncNet. + + Args: + in_channels (int): Input channels. + num_codes (int): Number of code words. + conv_cfg (dict|None): Config of conv layers. + norm_cfg (dict|None): Config of norm layers. + act_cfg (dict): Config of activation layers. + """ + + def __init__(self, in_channels, num_codes, conv_cfg, norm_cfg, act_cfg): + super(EncModule, self).__init__() + self.encoding_project = ConvModule( + in_channels, + in_channels, + 1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + # TODO: resolve this hack + # change to 1d + if norm_cfg is not None: + encoding_norm_cfg = norm_cfg.copy() + if encoding_norm_cfg['type'] in ['BN', 'IN']: + encoding_norm_cfg['type'] += '1d' + else: + encoding_norm_cfg['type'] = encoding_norm_cfg['type'].replace( + '2d', '1d') + else: + # fallback to BN1d + encoding_norm_cfg = dict(type='BN1d') + self.encoding = nn.Sequential( + Encoding(channels=in_channels, num_codes=num_codes), + build_norm_layer(encoding_norm_cfg, num_codes)[1], + nn.ReLU(inplace=True)) + self.fc = nn.Sequential( + nn.Linear(in_channels, in_channels), nn.Sigmoid()) + + def forward(self, x): + """Forward function.""" + encoding_projection = self.encoding_project(x) + encoding_feat = self.encoding(encoding_projection).mean(dim=1) + batch_size, channels, _, _ = x.size() + gamma = self.fc(encoding_feat) + y = gamma.view(batch_size, channels, 1, 1) + output = F.relu_(x + x * y) + return encoding_feat, output + + +@HEADS.register_module() +class EncHead(BaseDecodeHead): + """Context Encoding for Semantic Segmentation. + + This head is the implementation of `EncNet + `_. + + Args: + num_codes (int): Number of code words. Default: 32. + use_se_loss (bool): Whether use Semantic Encoding Loss (SE-loss) to + regularize the training. Default: True. + add_lateral (bool): Whether use lateral connection to fuse features. + Default: False. + loss_se_decode (dict): Config of decode loss. + Default: dict(type='CrossEntropyLoss', use_sigmoid=True). + """ + + def __init__(self, + num_codes=32, + use_se_loss=True, + add_lateral=False, + loss_se_decode=dict( + type='CrossEntropyLoss', + use_sigmoid=True, + loss_weight=0.2), + **kwargs): + super(EncHead, self).__init__( + input_transform='multiple_select', **kwargs) + self.use_se_loss = use_se_loss + self.add_lateral = add_lateral + self.num_codes = num_codes + self.bottleneck = ConvModule( + self.in_channels[-1], + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + if add_lateral: + self.lateral_convs = nn.ModuleList() + for in_channels in self.in_channels[:-1]: # skip the last one + self.lateral_convs.append( + ConvModule( + in_channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg)) + self.fusion = ConvModule( + len(self.in_channels) * self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.enc_module = EncModule( + self.channels, + num_codes=num_codes, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + if self.use_se_loss: + self.loss_se_decode = build_loss(loss_se_decode) + self.se_layer = nn.Linear(self.channels, self.num_classes) + + def forward(self, inputs): + """Forward function.""" + inputs = self._transform_inputs(inputs) + feat = self.bottleneck(inputs[-1]) + if self.add_lateral: + laterals = [ + resize( + lateral_conv(inputs[i]), + size=feat.shape[2:], + mode='bilinear', + align_corners=self.align_corners) + for i, lateral_conv in enumerate(self.lateral_convs) + ] + feat = self.fusion(torch.cat([feat, *laterals], 1)) + encode_feat, output = self.enc_module(feat) + output = self.cls_seg(output) + if self.use_se_loss: + se_output = self.se_layer(encode_feat) + return output, se_output + else: + return output + + def forward_test(self, inputs, img_metas, test_cfg): + """Forward function for testing, ignore se_loss.""" + if self.use_se_loss: + return self.forward(inputs)[0] + else: + return self.forward(inputs) + + @staticmethod + def _convert_to_onehot_labels(seg_label, num_classes): + """Convert segmentation label to onehot. + + Args: + seg_label (Tensor): Segmentation label of shape (N, H, W). + num_classes (int): Number of classes. + + Returns: + Tensor: Onehot labels of shape (N, num_classes). + """ + + batch_size = seg_label.size(0) + onehot_labels = seg_label.new_zeros((batch_size, num_classes)) + for i in range(batch_size): + hist = seg_label[i].float().histc( + bins=num_classes, min=0, max=num_classes - 1) + onehot_labels[i] = hist > 0 + return onehot_labels + + def losses(self, seg_logit, seg_label): + """Compute segmentation and semantic encoding loss.""" + seg_logit, se_seg_logit = seg_logit + loss = dict() + loss.update(super(EncHead, self).losses(seg_logit, seg_label)) + se_loss = self.loss_se_decode( + se_seg_logit, + self._convert_to_onehot_labels(seg_label, self.num_classes)) + loss['loss_se'] = se_loss + return loss diff --git a/segmentation/mmseg/models/decode_heads/fcn_head.py b/segmentation/mmseg/models/decode_heads/fcn_head.py new file mode 100644 index 0000000..4ea3742 --- /dev/null +++ b/segmentation/mmseg/models/decode_heads/fcn_head.py @@ -0,0 +1,81 @@ +import torch +import torch.nn as nn +from mmcv.cnn import ConvModule + +from ..builder import HEADS +from .decode_head import BaseDecodeHead + + +@HEADS.register_module() +class FCNHead(BaseDecodeHead): + """Fully Convolution Networks for Semantic Segmentation. + + This head is implemented of `FCNNet `_. + + Args: + num_convs (int): Number of convs in the head. Default: 2. + kernel_size (int): The kernel size for convs in the head. Default: 3. + concat_input (bool): Whether concat the input and output of convs + before classification layer. + dilation (int): The dilation rate for convs in the head. Default: 1. + """ + + def __init__(self, + num_convs=2, + kernel_size=3, + concat_input=True, + dilation=1, + **kwargs): + assert num_convs >= 0 and dilation > 0 and isinstance(dilation, int) + self.num_convs = num_convs + self.concat_input = concat_input + self.kernel_size = kernel_size + super(FCNHead, self).__init__(**kwargs) + if num_convs == 0: + assert self.in_channels == self.channels + + conv_padding = (kernel_size // 2) * dilation + convs = [] + convs.append( + ConvModule( + self.in_channels, + self.channels, + kernel_size=kernel_size, + padding=conv_padding, + dilation=dilation, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg)) + for i in range(num_convs - 1): + convs.append( + ConvModule( + self.channels, + self.channels, + kernel_size=kernel_size, + padding=conv_padding, + dilation=dilation, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg)) + if num_convs == 0: + self.convs = nn.Identity() + else: + self.convs = nn.Sequential(*convs) + if self.concat_input: + self.conv_cat = ConvModule( + self.in_channels + self.channels, + self.channels, + kernel_size=kernel_size, + padding=kernel_size // 2, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + output = self.convs(x) + if self.concat_input: + output = self.conv_cat(torch.cat([x, output], dim=1)) + output = self.cls_seg(output) + return output diff --git a/segmentation/mmseg/models/decode_heads/fpn_head.py b/segmentation/mmseg/models/decode_heads/fpn_head.py new file mode 100644 index 0000000..9b6ada0 --- /dev/null +++ b/segmentation/mmseg/models/decode_heads/fpn_head.py @@ -0,0 +1,68 @@ +import numpy as np +import torch.nn as nn +from mmcv.cnn import ConvModule + +from mmseg.ops import resize +from ..builder import HEADS +from .decode_head import BaseDecodeHead + + +@HEADS.register_module() +class FPNHead(BaseDecodeHead): + """Panoptic Feature Pyramid Networks. + + This head is the implementation of `Semantic FPN + `_. + + Args: + feature_strides (tuple[int]): The strides for input feature maps. + stack_lateral. All strides suppose to be power of 2. The first + one is of largest resolution. + """ + + def __init__(self, feature_strides, **kwargs): + super(FPNHead, self).__init__( + input_transform='multiple_select', **kwargs) + assert len(feature_strides) == len(self.in_channels) + assert min(feature_strides) == feature_strides[0] + self.feature_strides = feature_strides + + self.scale_heads = nn.ModuleList() + for i in range(len(feature_strides)): + head_length = max( + 1, + int(np.log2(feature_strides[i]) - np.log2(feature_strides[0]))) + scale_head = [] + for k in range(head_length): + scale_head.append( + ConvModule( + self.in_channels[i] if k == 0 else self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg)) + if feature_strides[i] != feature_strides[0]: + scale_head.append( + nn.Upsample( + scale_factor=2, + mode='bilinear', + align_corners=self.align_corners)) + self.scale_heads.append(nn.Sequential(*scale_head)) + + def forward(self, inputs): + + x = self._transform_inputs(inputs) + + output = self.scale_heads[0](x[0]) + for i in range(1, len(self.feature_strides)): + # non inplace + output = output + resize( + self.scale_heads[i](x[i]), + size=output.shape[2:], + mode='bilinear', + align_corners=self.align_corners) + + output = self.cls_seg(output) + return output diff --git a/segmentation/mmseg/models/decode_heads/gc_head.py b/segmentation/mmseg/models/decode_heads/gc_head.py new file mode 100644 index 0000000..93f60ad --- /dev/null +++ b/segmentation/mmseg/models/decode_heads/gc_head.py @@ -0,0 +1,47 @@ +import torch +from mmcv.cnn import ContextBlock + +from ..builder import HEADS +from .fcn_head import FCNHead + + +@HEADS.register_module() +class GCHead(FCNHead): + """GCNet: Non-local Networks Meet Squeeze-Excitation Networks and Beyond. + + This head is the implementation of `GCNet + `_. + + Args: + ratio (float): Multiplier of channels ratio. Default: 1/4. + pooling_type (str): The pooling type of context aggregation. + Options are 'att', 'avg'. Default: 'avg'. + fusion_types (tuple[str]): The fusion type for feature fusion. + Options are 'channel_add', 'channel_mul'. Default: ('channel_add',) + """ + + def __init__(self, + ratio=1 / 4., + pooling_type='att', + fusion_types=('channel_add', ), + **kwargs): + super(GCHead, self).__init__(num_convs=2, **kwargs) + self.ratio = ratio + self.pooling_type = pooling_type + self.fusion_types = fusion_types + self.gc_block = ContextBlock( + in_channels=self.channels, + ratio=self.ratio, + pooling_type=self.pooling_type, + fusion_types=self.fusion_types) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + output = self.convs[0](x) + output = self.gc_block(output) + output = self.convs[1](output) + if self.concat_input: + output = self.conv_cat(torch.cat([x, output], dim=1)) + output = self.cls_seg(output) + return output diff --git a/segmentation/mmseg/models/decode_heads/lraspp_head.py b/segmentation/mmseg/models/decode_heads/lraspp_head.py new file mode 100644 index 0000000..32a093c --- /dev/null +++ b/segmentation/mmseg/models/decode_heads/lraspp_head.py @@ -0,0 +1,90 @@ +import torch +import torch.nn as nn +from mmcv import is_tuple_of +from mmcv.cnn import ConvModule + +from mmseg.ops import resize +from ..builder import HEADS +from .decode_head import BaseDecodeHead + + +@HEADS.register_module() +class LRASPPHead(BaseDecodeHead): + """Lite R-ASPP (LRASPP) head is proposed in Searching for MobileNetV3. + + This head is the improved implementation of `Searching for MobileNetV3 + `_. + + Args: + branch_channels (tuple[int]): The number of output channels in every + each branch. Default: (32, 64). + """ + + def __init__(self, branch_channels=(32, 64), **kwargs): + super(LRASPPHead, self).__init__(**kwargs) + if self.input_transform != 'multiple_select': + raise ValueError('in Lite R-ASPP (LRASPP) head, input_transform ' + f'must be \'multiple_select\'. But received ' + f'\'{self.input_transform}\'') + assert is_tuple_of(branch_channels, int) + assert len(branch_channels) == len(self.in_channels) - 1 + self.branch_channels = branch_channels + + self.convs = nn.Sequential() + self.conv_ups = nn.Sequential() + for i in range(len(branch_channels)): + self.convs.add_module( + f'conv{i}', + nn.Conv2d( + self.in_channels[i], branch_channels[i], 1, bias=False)) + self.conv_ups.add_module( + f'conv_up{i}', + ConvModule( + self.channels + branch_channels[i], + self.channels, + 1, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg, + bias=False)) + + self.conv_up_input = nn.Conv2d(self.channels, self.channels, 1) + + self.aspp_conv = ConvModule( + self.in_channels[-1], + self.channels, + 1, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg, + bias=False) + self.image_pool = nn.Sequential( + nn.AvgPool2d(kernel_size=49, stride=(16, 20)), + ConvModule( + self.in_channels[2], + self.channels, + 1, + act_cfg=dict(type='Sigmoid'), + bias=False)) + + def forward(self, inputs): + """Forward function.""" + inputs = self._transform_inputs(inputs) + + x = inputs[-1] + + x = self.aspp_conv(x) * resize( + self.image_pool(x), + size=x.size()[2:], + mode='bilinear', + align_corners=self.align_corners) + x = self.conv_up_input(x) + + for i in range(len(self.branch_channels) - 1, -1, -1): + x = resize( + x, + size=inputs[i].size()[2:], + mode='bilinear', + align_corners=self.align_corners) + x = torch.cat([x, self.convs[i](inputs[i])], 1) + x = self.conv_ups[i](x) + + return self.cls_seg(x) diff --git a/segmentation/mmseg/models/decode_heads/nl_head.py b/segmentation/mmseg/models/decode_heads/nl_head.py new file mode 100644 index 0000000..3165875 --- /dev/null +++ b/segmentation/mmseg/models/decode_heads/nl_head.py @@ -0,0 +1,49 @@ +import torch +from mmcv.cnn import NonLocal2d + +from ..builder import HEADS +from .fcn_head import FCNHead + + +@HEADS.register_module() +class NLHead(FCNHead): + """Non-local Neural Networks. + + This head is the implementation of `NLNet + `_. + + Args: + reduction (int): Reduction factor of projection transform. Default: 2. + use_scale (bool): Whether to scale pairwise_weight by + sqrt(1/inter_channels). Default: True. + mode (str): The nonlocal mode. Options are 'embedded_gaussian', + 'dot_product'. Default: 'embedded_gaussian.'. + """ + + def __init__(self, + reduction=2, + use_scale=True, + mode='embedded_gaussian', + **kwargs): + super(NLHead, self).__init__(num_convs=2, **kwargs) + self.reduction = reduction + self.use_scale = use_scale + self.mode = mode + self.nl_block = NonLocal2d( + in_channels=self.channels, + reduction=self.reduction, + use_scale=self.use_scale, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + mode=self.mode) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + output = self.convs[0](x) + output = self.nl_block(output) + output = self.convs[1](output) + if self.concat_input: + output = self.conv_cat(torch.cat([x, output], dim=1)) + output = self.cls_seg(output) + return output diff --git a/segmentation/mmseg/models/decode_heads/ocr_head.py b/segmentation/mmseg/models/decode_heads/ocr_head.py new file mode 100644 index 0000000..e180e10 --- /dev/null +++ b/segmentation/mmseg/models/decode_heads/ocr_head.py @@ -0,0 +1,127 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import ConvModule + +from mmseg.ops import resize +from ..builder import HEADS +from ..utils import SelfAttentionBlock as _SelfAttentionBlock +from .cascade_decode_head import BaseCascadeDecodeHead + + +class SpatialGatherModule(nn.Module): + """Aggregate the context features according to the initial predicted + probability distribution. + + Employ the soft-weighted method to aggregate the context. + """ + + def __init__(self, scale): + super(SpatialGatherModule, self).__init__() + self.scale = scale + + def forward(self, feats, probs): + """Forward function.""" + batch_size, num_classes, height, width = probs.size() + channels = feats.size(1) + probs = probs.view(batch_size, num_classes, -1) + feats = feats.view(batch_size, channels, -1) + # [batch_size, height*width, num_classes] + feats = feats.permute(0, 2, 1) + # [batch_size, channels, height*width] + probs = F.softmax(self.scale * probs, dim=2) + # [batch_size, channels, num_classes] + ocr_context = torch.matmul(probs, feats) + ocr_context = ocr_context.permute(0, 2, 1).contiguous().unsqueeze(3) + return ocr_context + + +class ObjectAttentionBlock(_SelfAttentionBlock): + """Make a OCR used SelfAttentionBlock.""" + + def __init__(self, in_channels, channels, scale, conv_cfg, norm_cfg, + act_cfg): + if scale > 1: + query_downsample = nn.MaxPool2d(kernel_size=scale) + else: + query_downsample = None + super(ObjectAttentionBlock, self).__init__( + key_in_channels=in_channels, + query_in_channels=in_channels, + channels=channels, + out_channels=in_channels, + share_key_query=False, + query_downsample=query_downsample, + key_downsample=None, + key_query_num_convs=2, + key_query_norm=True, + value_out_num_convs=1, + value_out_norm=True, + matmul_norm=True, + with_out=True, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + self.bottleneck = ConvModule( + in_channels * 2, + in_channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, query_feats, key_feats): + """Forward function.""" + context = super(ObjectAttentionBlock, + self).forward(query_feats, key_feats) + output = self.bottleneck(torch.cat([context, query_feats], dim=1)) + if self.query_downsample is not None: + output = resize(query_feats) + + return output + + +@HEADS.register_module() +class OCRHead(BaseCascadeDecodeHead): + """Object-Contextual Representations for Semantic Segmentation. + + This head is the implementation of `OCRNet + `_. + + Args: + ocr_channels (int): The intermediate channels of OCR block. + scale (int): The scale of probability map in SpatialGatherModule in + Default: 1. + """ + + def __init__(self, ocr_channels, scale=1, **kwargs): + super(OCRHead, self).__init__(**kwargs) + self.ocr_channels = ocr_channels + self.scale = scale + self.object_context_block = ObjectAttentionBlock( + self.channels, + self.ocr_channels, + self.scale, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.spatial_gather_module = SpatialGatherModule(self.scale) + + self.bottleneck = ConvModule( + self.in_channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, inputs, prev_output): + """Forward function.""" + x = self._transform_inputs(inputs) + feats = self.bottleneck(x) + context = self.spatial_gather_module(feats, prev_output) + object_context = self.object_context_block(feats, context) + output = self.cls_seg(object_context) + + return output diff --git a/segmentation/mmseg/models/decode_heads/point_head.py b/segmentation/mmseg/models/decode_heads/point_head.py new file mode 100644 index 0000000..90a2363 --- /dev/null +++ b/segmentation/mmseg/models/decode_heads/point_head.py @@ -0,0 +1,349 @@ +# Modified from https://github.com/facebookresearch/detectron2/tree/master/projects/PointRend/point_head/point_head.py # noqa + +import torch +import torch.nn as nn +from mmcv.cnn import ConvModule, normal_init +from mmcv.ops import point_sample + +from mmseg.models.builder import HEADS +from mmseg.ops import resize +from ..losses import accuracy +from .cascade_decode_head import BaseCascadeDecodeHead + + +def calculate_uncertainty(seg_logits): + """Estimate uncertainty based on seg logits. + + For each location of the prediction ``seg_logits`` we estimate + uncertainty as the difference between top first and top second + predicted logits. + + Args: + seg_logits (Tensor): Semantic segmentation logits, + shape (batch_size, num_classes, height, width). + + Returns: + scores (Tensor): T uncertainty scores with the most uncertain + locations having the highest uncertainty score, shape ( + batch_size, 1, height, width) + """ + top2_scores = torch.topk(seg_logits, k=2, dim=1)[0] + return (top2_scores[:, 1] - top2_scores[:, 0]).unsqueeze(1) + + +@HEADS.register_module() +class PointHead(BaseCascadeDecodeHead): + """A mask point head use in PointRend. + + ``PointHead`` use shared multi-layer perceptron (equivalent to + nn.Conv1d) to predict the logit of input points. The fine-grained feature + and coarse feature will be concatenate together for predication. + + Args: + num_fcs (int): Number of fc layers in the head. Default: 3. + in_channels (int): Number of input channels. Default: 256. + fc_channels (int): Number of fc channels. Default: 256. + num_classes (int): Number of classes for logits. Default: 80. + class_agnostic (bool): Whether use class agnostic classification. + If so, the output channels of logits will be 1. Default: False. + coarse_pred_each_layer (bool): Whether concatenate coarse feature with + the output of each fc layer. Default: True. + conv_cfg (dict|None): Dictionary to construct and config conv layer. + Default: dict(type='Conv1d')) + norm_cfg (dict|None): Dictionary to construct and config norm layer. + Default: None. + loss_point (dict): Dictionary to construct and config loss layer of + point head. Default: dict(type='CrossEntropyLoss', use_mask=True, + loss_weight=1.0). + """ + + def __init__(self, + num_fcs=3, + coarse_pred_each_layer=True, + conv_cfg=dict(type='Conv1d'), + norm_cfg=None, + act_cfg=dict(type='ReLU', inplace=False), + **kwargs): + super(PointHead, self).__init__( + input_transform='multiple_select', + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + **kwargs) + + self.num_fcs = num_fcs + self.coarse_pred_each_layer = coarse_pred_each_layer + + fc_in_channels = sum(self.in_channels) + self.num_classes + fc_channels = self.channels + self.fcs = nn.ModuleList() + for k in range(num_fcs): + fc = ConvModule( + fc_in_channels, + fc_channels, + kernel_size=1, + stride=1, + padding=0, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + self.fcs.append(fc) + fc_in_channels = fc_channels + fc_in_channels += self.num_classes if self.coarse_pred_each_layer \ + else 0 + self.fc_seg = nn.Conv1d( + fc_in_channels, + self.num_classes, + kernel_size=1, + stride=1, + padding=0) + if self.dropout_ratio > 0: + self.dropout = nn.Dropout(self.dropout_ratio) + delattr(self, 'conv_seg') + + def init_weights(self): + """Initialize weights of classification layer.""" + normal_init(self.fc_seg, std=0.001) + + def cls_seg(self, feat): + """Classify each pixel with fc.""" + if self.dropout is not None: + feat = self.dropout(feat) + output = self.fc_seg(feat) + return output + + def forward(self, fine_grained_point_feats, coarse_point_feats): + x = torch.cat([fine_grained_point_feats, coarse_point_feats], dim=1) + for fc in self.fcs: + x = fc(x) + if self.coarse_pred_each_layer: + x = torch.cat((x, coarse_point_feats), dim=1) + return self.cls_seg(x) + + def _get_fine_grained_point_feats(self, x, points): + """Sample from fine grained features. + + Args: + x (list[Tensor]): Feature pyramid from by neck or backbone. + points (Tensor): Point coordinates, shape (batch_size, + num_points, 2). + + Returns: + fine_grained_feats (Tensor): Sampled fine grained feature, + shape (batch_size, sum(channels of x), num_points). + """ + + fine_grained_feats_list = [ + point_sample(_, points, align_corners=self.align_corners) + for _ in x + ] + if len(fine_grained_feats_list) > 1: + fine_grained_feats = torch.cat(fine_grained_feats_list, dim=1) + else: + fine_grained_feats = fine_grained_feats_list[0] + + return fine_grained_feats + + def _get_coarse_point_feats(self, prev_output, points): + """Sample from fine grained features. + + Args: + prev_output (list[Tensor]): Prediction of previous decode head. + points (Tensor): Point coordinates, shape (batch_size, + num_points, 2). + + Returns: + coarse_feats (Tensor): Sampled coarse feature, shape (batch_size, + num_classes, num_points). + """ + + coarse_feats = point_sample( + prev_output, points, align_corners=self.align_corners) + + return coarse_feats + + def forward_train(self, inputs, prev_output, img_metas, gt_semantic_seg, + train_cfg): + """Forward function for training. + Args: + inputs (list[Tensor]): List of multi-level img features. + prev_output (Tensor): The output of previous decode head. + img_metas (list[dict]): List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + gt_semantic_seg (Tensor): Semantic segmentation masks + used if the architecture supports semantic segmentation task. + train_cfg (dict): The training config. + + Returns: + dict[str, Tensor]: a dictionary of loss components + """ + x = self._transform_inputs(inputs) + with torch.no_grad(): + points = self.get_points_train( + prev_output, calculate_uncertainty, cfg=train_cfg) + fine_grained_point_feats = self._get_fine_grained_point_feats( + x, points) + coarse_point_feats = self._get_coarse_point_feats(prev_output, points) + point_logits = self.forward(fine_grained_point_feats, + coarse_point_feats) + point_label = point_sample( + gt_semantic_seg.float(), + points, + mode='nearest', + align_corners=self.align_corners) + point_label = point_label.squeeze(1).long() + + losses = self.losses(point_logits, point_label) + + return losses + + def forward_test(self, inputs, prev_output, img_metas, test_cfg): + """Forward function for testing. + + Args: + inputs (list[Tensor]): List of multi-level img features. + prev_output (Tensor): The output of previous decode head. + img_metas (list[dict]): List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + test_cfg (dict): The testing config. + + Returns: + Tensor: Output segmentation map. + """ + + x = self._transform_inputs(inputs) + refined_seg_logits = prev_output.clone() + for _ in range(test_cfg.subdivision_steps): + refined_seg_logits = resize( + refined_seg_logits, + scale_factor=test_cfg.scale_factor, + mode='bilinear', + align_corners=self.align_corners) + batch_size, channels, height, width = refined_seg_logits.shape + point_indices, points = self.get_points_test( + refined_seg_logits, calculate_uncertainty, cfg=test_cfg) + fine_grained_point_feats = self._get_fine_grained_point_feats( + x, points) + coarse_point_feats = self._get_coarse_point_feats( + prev_output, points) + point_logits = self.forward(fine_grained_point_feats, + coarse_point_feats) + + point_indices = point_indices.unsqueeze(1).expand(-1, channels, -1) + refined_seg_logits = refined_seg_logits.reshape( + batch_size, channels, height * width) + refined_seg_logits = refined_seg_logits.scatter_( + 2, point_indices, point_logits) + refined_seg_logits = refined_seg_logits.view( + batch_size, channels, height, width) + + return refined_seg_logits + + def losses(self, point_logits, point_label): + """Compute segmentation loss.""" + loss = dict() + loss['loss_point'] = self.loss_decode( + point_logits, point_label, ignore_index=self.ignore_index) + loss['acc_point'] = accuracy(point_logits, point_label) + return loss + + def get_points_train(self, seg_logits, uncertainty_func, cfg): + """Sample points for training. + + Sample points in [0, 1] x [0, 1] coordinate space based on their + uncertainty. The uncertainties are calculated for each point using + 'uncertainty_func' function that takes point's logit prediction as + input. + + Args: + seg_logits (Tensor): Semantic segmentation logits, shape ( + batch_size, num_classes, height, width). + uncertainty_func (func): uncertainty calculation function. + cfg (dict): Training config of point head. + + Returns: + point_coords (Tensor): A tensor of shape (batch_size, num_points, + 2) that contains the coordinates of ``num_points`` sampled + points. + """ + num_points = cfg.num_points + oversample_ratio = cfg.oversample_ratio + importance_sample_ratio = cfg.importance_sample_ratio + assert oversample_ratio >= 1 + assert 0 <= importance_sample_ratio <= 1 + batch_size = seg_logits.shape[0] + num_sampled = int(num_points * oversample_ratio) + point_coords = torch.rand( + batch_size, num_sampled, 2, device=seg_logits.device) + point_logits = point_sample(seg_logits, point_coords) + # It is crucial to calculate uncertainty based on the sampled + # prediction value for the points. Calculating uncertainties of the + # coarse predictions first and sampling them for points leads to + # incorrect results. To illustrate this: assume uncertainty func( + # logits)=-abs(logits), a sampled point between two coarse + # predictions with -1 and 1 logits has 0 logits, and therefore 0 + # uncertainty value. However, if we calculate uncertainties for the + # coarse predictions first, both will have -1 uncertainty, + # and sampled point will get -1 uncertainty. + point_uncertainties = uncertainty_func(point_logits) + num_uncertain_points = int(importance_sample_ratio * num_points) + num_random_points = num_points - num_uncertain_points + idx = torch.topk( + point_uncertainties[:, 0, :], k=num_uncertain_points, dim=1)[1] + shift = num_sampled * torch.arange( + batch_size, dtype=torch.long, device=seg_logits.device) + idx += shift[:, None] + point_coords = point_coords.view(-1, 2)[idx.view(-1), :].view( + batch_size, num_uncertain_points, 2) + if num_random_points > 0: + rand_point_coords = torch.rand( + batch_size, num_random_points, 2, device=seg_logits.device) + point_coords = torch.cat((point_coords, rand_point_coords), dim=1) + return point_coords + + def get_points_test(self, seg_logits, uncertainty_func, cfg): + """Sample points for testing. + + Find ``num_points`` most uncertain points from ``uncertainty_map``. + + Args: + seg_logits (Tensor): A tensor of shape (batch_size, num_classes, + height, width) for class-specific or class-agnostic prediction. + uncertainty_func (func): uncertainty calculation function. + cfg (dict): Testing config of point head. + + Returns: + point_indices (Tensor): A tensor of shape (batch_size, num_points) + that contains indices from [0, height x width) of the most + uncertain points. + point_coords (Tensor): A tensor of shape (batch_size, num_points, + 2) that contains [0, 1] x [0, 1] normalized coordinates of the + most uncertain points from the ``height x width`` grid . + """ + + num_points = cfg.subdivision_num_points + uncertainty_map = uncertainty_func(seg_logits) + batch_size, _, height, width = uncertainty_map.shape + h_step = 1.0 / height + w_step = 1.0 / width + + uncertainty_map = uncertainty_map.view(batch_size, height * width) + num_points = min(height * width, num_points) + point_indices = uncertainty_map.topk(num_points, dim=1)[1] + point_coords = torch.zeros( + batch_size, + num_points, + 2, + dtype=torch.float, + device=seg_logits.device) + point_coords[:, :, 0] = w_step / 2.0 + (point_indices % + width).float() * w_step + point_coords[:, :, 1] = h_step / 2.0 + (point_indices // + width).float() * h_step + return point_indices, point_coords diff --git a/segmentation/mmseg/models/decode_heads/psa_head.py b/segmentation/mmseg/models/decode_heads/psa_head.py new file mode 100644 index 0000000..8d915e5 --- /dev/null +++ b/segmentation/mmseg/models/decode_heads/psa_head.py @@ -0,0 +1,196 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import ConvModule + +from mmseg.ops import resize +from ..builder import HEADS +from .decode_head import BaseDecodeHead + +try: + from mmcv.ops import PSAMask +except ModuleNotFoundError: + PSAMask = None + + +@HEADS.register_module() +class PSAHead(BaseDecodeHead): + """Point-wise Spatial Attention Network for Scene Parsing. + + This head is the implementation of `PSANet + `_. + + Args: + mask_size (tuple[int]): The PSA mask size. It usually equals input + size. + psa_type (str): The type of psa module. Options are 'collect', + 'distribute', 'bi-direction'. Default: 'bi-direction' + compact (bool): Whether use compact map for 'collect' mode. + Default: True. + shrink_factor (int): The downsample factors of psa mask. Default: 2. + normalization_factor (float): The normalize factor of attention. + psa_softmax (bool): Whether use softmax for attention. + """ + + def __init__(self, + mask_size, + psa_type='bi-direction', + compact=False, + shrink_factor=2, + normalization_factor=1.0, + psa_softmax=True, + **kwargs): + if PSAMask is None: + raise RuntimeError('Please install mmcv-full for PSAMask ops') + super(PSAHead, self).__init__(**kwargs) + assert psa_type in ['collect', 'distribute', 'bi-direction'] + self.psa_type = psa_type + self.compact = compact + self.shrink_factor = shrink_factor + self.mask_size = mask_size + mask_h, mask_w = mask_size + self.psa_softmax = psa_softmax + if normalization_factor is None: + normalization_factor = mask_h * mask_w + self.normalization_factor = normalization_factor + + self.reduce = ConvModule( + self.in_channels, + self.channels, + kernel_size=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.attention = nn.Sequential( + ConvModule( + self.channels, + self.channels, + kernel_size=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg), + nn.Conv2d( + self.channels, mask_h * mask_w, kernel_size=1, bias=False)) + if psa_type == 'bi-direction': + self.reduce_p = ConvModule( + self.in_channels, + self.channels, + kernel_size=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.attention_p = nn.Sequential( + ConvModule( + self.channels, + self.channels, + kernel_size=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg), + nn.Conv2d( + self.channels, mask_h * mask_w, kernel_size=1, bias=False)) + self.psamask_collect = PSAMask('collect', mask_size) + self.psamask_distribute = PSAMask('distribute', mask_size) + else: + self.psamask = PSAMask(psa_type, mask_size) + self.proj = ConvModule( + self.channels * (2 if psa_type == 'bi-direction' else 1), + self.in_channels, + kernel_size=1, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.bottleneck = ConvModule( + self.in_channels * 2, + self.channels, + kernel_size=3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + identity = x + align_corners = self.align_corners + if self.psa_type in ['collect', 'distribute']: + out = self.reduce(x) + n, c, h, w = out.size() + if self.shrink_factor != 1: + if h % self.shrink_factor and w % self.shrink_factor: + h = (h - 1) // self.shrink_factor + 1 + w = (w - 1) // self.shrink_factor + 1 + align_corners = True + else: + h = h // self.shrink_factor + w = w // self.shrink_factor + align_corners = False + out = resize( + out, + size=(h, w), + mode='bilinear', + align_corners=align_corners) + y = self.attention(out) + if self.compact: + if self.psa_type == 'collect': + y = y.view(n, h * w, + h * w).transpose(1, 2).view(n, h * w, h, w) + else: + y = self.psamask(y) + if self.psa_softmax: + y = F.softmax(y, dim=1) + out = torch.bmm( + out.view(n, c, h * w), y.view(n, h * w, h * w)).view( + n, c, h, w) * (1.0 / self.normalization_factor) + else: + x_col = self.reduce(x) + x_dis = self.reduce_p(x) + n, c, h, w = x_col.size() + if self.shrink_factor != 1: + if h % self.shrink_factor and w % self.shrink_factor: + h = (h - 1) // self.shrink_factor + 1 + w = (w - 1) // self.shrink_factor + 1 + align_corners = True + else: + h = h // self.shrink_factor + w = w // self.shrink_factor + align_corners = False + x_col = resize( + x_col, + size=(h, w), + mode='bilinear', + align_corners=align_corners) + x_dis = resize( + x_dis, + size=(h, w), + mode='bilinear', + align_corners=align_corners) + y_col = self.attention(x_col) + y_dis = self.attention_p(x_dis) + if self.compact: + y_dis = y_dis.view(n, h * w, + h * w).transpose(1, 2).view(n, h * w, h, w) + else: + y_col = self.psamask_collect(y_col) + y_dis = self.psamask_distribute(y_dis) + if self.psa_softmax: + y_col = F.softmax(y_col, dim=1) + y_dis = F.softmax(y_dis, dim=1) + x_col = torch.bmm( + x_col.view(n, c, h * w), y_col.view(n, h * w, h * w)).view( + n, c, h, w) * (1.0 / self.normalization_factor) + x_dis = torch.bmm( + x_dis.view(n, c, h * w), y_dis.view(n, h * w, h * w)).view( + n, c, h, w) * (1.0 / self.normalization_factor) + out = torch.cat([x_col, x_dis], 1) + out = self.proj(out) + out = resize( + out, + size=identity.shape[2:], + mode='bilinear', + align_corners=align_corners) + out = self.bottleneck(torch.cat((identity, out), dim=1)) + out = self.cls_seg(out) + return out diff --git a/segmentation/mmseg/models/decode_heads/psp_head.py b/segmentation/mmseg/models/decode_heads/psp_head.py new file mode 100644 index 0000000..bdbe2c8 --- /dev/null +++ b/segmentation/mmseg/models/decode_heads/psp_head.py @@ -0,0 +1,101 @@ +import torch +import torch.nn as nn +from mmcv.cnn import ConvModule + +from mmseg.ops import resize +from ..builder import HEADS +from .decode_head import BaseDecodeHead + + +class PPM(nn.ModuleList): + """Pooling Pyramid Module used in PSPNet. + + Args: + pool_scales (tuple[int]): Pooling scales used in Pooling Pyramid + Module. + in_channels (int): Input channels. + channels (int): Channels after modules, before conv_seg. + conv_cfg (dict|None): Config of conv layers. + norm_cfg (dict|None): Config of norm layers. + act_cfg (dict): Config of activation layers. + align_corners (bool): align_corners argument of F.interpolate. + """ + + def __init__(self, pool_scales, in_channels, channels, conv_cfg, norm_cfg, + act_cfg, align_corners): + super(PPM, self).__init__() + self.pool_scales = pool_scales + self.align_corners = align_corners + self.in_channels = in_channels + self.channels = channels + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + for pool_scale in pool_scales: + self.append( + nn.Sequential( + nn.AdaptiveAvgPool2d(pool_scale), + ConvModule( + self.in_channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg))) + + def forward(self, x): + """Forward function.""" + ppm_outs = [] + for ppm in self: + ppm_out = ppm(x) + upsampled_ppm_out = resize( + ppm_out, + size=x.size()[2:], + mode='bilinear', + align_corners=self.align_corners) + ppm_outs.append(upsampled_ppm_out) + return ppm_outs + + +@HEADS.register_module() +class PSPHead(BaseDecodeHead): + """Pyramid Scene Parsing Network. + + This head is the implementation of + `PSPNet `_. + + Args: + pool_scales (tuple[int]): Pooling scales used in Pooling Pyramid + Module. Default: (1, 2, 3, 6). + """ + + def __init__(self, pool_scales=(1, 2, 3, 6), **kwargs): + super(PSPHead, self).__init__(**kwargs) + assert isinstance(pool_scales, (list, tuple)) + self.pool_scales = pool_scales + self.psp_modules = PPM( + self.pool_scales, + self.in_channels, + self.channels, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg, + align_corners=self.align_corners) + self.bottleneck = ConvModule( + self.in_channels + len(pool_scales) * self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + psp_outs = [x] + psp_outs.extend(self.psp_modules(x)) + psp_outs = torch.cat(psp_outs, dim=1) + output = self.bottleneck(psp_outs) + output = self.cls_seg(output) + return output diff --git a/segmentation/mmseg/models/decode_heads/sep_aspp_head.py b/segmentation/mmseg/models/decode_heads/sep_aspp_head.py new file mode 100644 index 0000000..50bd52b --- /dev/null +++ b/segmentation/mmseg/models/decode_heads/sep_aspp_head.py @@ -0,0 +1,101 @@ +import torch +import torch.nn as nn +from mmcv.cnn import ConvModule, DepthwiseSeparableConvModule + +from mmseg.ops import resize +from ..builder import HEADS +from .aspp_head import ASPPHead, ASPPModule + + +class DepthwiseSeparableASPPModule(ASPPModule): + """Atrous Spatial Pyramid Pooling (ASPP) Module with depthwise separable + conv.""" + + def __init__(self, **kwargs): + super(DepthwiseSeparableASPPModule, self).__init__(**kwargs) + for i, dilation in enumerate(self.dilations): + if dilation > 1: + self[i] = DepthwiseSeparableConvModule( + self.in_channels, + self.channels, + 3, + dilation=dilation, + padding=dilation, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + +@HEADS.register_module() +class DepthwiseSeparableASPPHead(ASPPHead): + """Encoder-Decoder with Atrous Separable Convolution for Semantic Image + Segmentation. + + This head is the implementation of `DeepLabV3+ + `_. + + Args: + c1_in_channels (int): The input channels of c1 decoder. If is 0, + the no decoder will be used. + c1_channels (int): The intermediate channels of c1 decoder. + """ + + def __init__(self, c1_in_channels, c1_channels, **kwargs): + super(DepthwiseSeparableASPPHead, self).__init__(**kwargs) + assert c1_in_channels >= 0 + self.aspp_modules = DepthwiseSeparableASPPModule( + dilations=self.dilations, + in_channels=self.in_channels, + channels=self.channels, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + if c1_in_channels > 0: + self.c1_bottleneck = ConvModule( + c1_in_channels, + c1_channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + else: + self.c1_bottleneck = None + self.sep_bottleneck = nn.Sequential( + DepthwiseSeparableConvModule( + self.channels + c1_channels, + self.channels, + 3, + padding=1, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg), + DepthwiseSeparableConvModule( + self.channels, + self.channels, + 3, + padding=1, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg)) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + aspp_outs = [ + resize( + self.image_pool(x), + size=x.size()[2:], + mode='bilinear', + align_corners=self.align_corners) + ] + aspp_outs.extend(self.aspp_modules(x)) + aspp_outs = torch.cat(aspp_outs, dim=1) + output = self.bottleneck(aspp_outs) + if self.c1_bottleneck is not None: + c1_output = self.c1_bottleneck(inputs[0]) + output = resize( + input=output, + size=c1_output.shape[2:], + mode='bilinear', + align_corners=self.align_corners) + output = torch.cat([output, c1_output], dim=1) + output = self.sep_bottleneck(output) + output = self.cls_seg(output) + return output diff --git a/segmentation/mmseg/models/decode_heads/sep_fcn_head.py b/segmentation/mmseg/models/decode_heads/sep_fcn_head.py new file mode 100644 index 0000000..a636f70 --- /dev/null +++ b/segmentation/mmseg/models/decode_heads/sep_fcn_head.py @@ -0,0 +1,51 @@ +from mmcv.cnn import DepthwiseSeparableConvModule + +from ..builder import HEADS +from .fcn_head import FCNHead + + +@HEADS.register_module() +class DepthwiseSeparableFCNHead(FCNHead): + """Depthwise-Separable Fully Convolutional Network for Semantic + Segmentation. + + This head is implemented according to Fast-SCNN paper. + Args: + in_channels(int): Number of output channels of FFM. + channels(int): Number of middle-stage channels in the decode head. + concat_input(bool): Whether to concatenate original decode input into + the result of several consecutive convolution layers. + Default: True. + num_classes(int): Used to determine the dimension of + final prediction tensor. + in_index(int): Correspond with 'out_indices' in FastSCNN backbone. + norm_cfg (dict | None): Config of norm layers. + align_corners (bool): align_corners argument of F.interpolate. + Default: False. + loss_decode(dict): Config of loss type and some + relevant additional options. + """ + + def __init__(self, **kwargs): + super(DepthwiseSeparableFCNHead, self).__init__(**kwargs) + self.convs[0] = DepthwiseSeparableConvModule( + self.in_channels, + self.channels, + kernel_size=self.kernel_size, + padding=self.kernel_size // 2, + norm_cfg=self.norm_cfg) + for i in range(1, self.num_convs): + self.convs[i] = DepthwiseSeparableConvModule( + self.channels, + self.channels, + kernel_size=self.kernel_size, + padding=self.kernel_size // 2, + norm_cfg=self.norm_cfg) + + if self.concat_input: + self.conv_cat = DepthwiseSeparableConvModule( + self.in_channels + self.channels, + self.channels, + kernel_size=self.kernel_size, + padding=self.kernel_size // 2, + norm_cfg=self.norm_cfg) diff --git a/segmentation/mmseg/models/decode_heads/uper_head.py b/segmentation/mmseg/models/decode_heads/uper_head.py new file mode 100644 index 0000000..bb617f6 --- /dev/null +++ b/segmentation/mmseg/models/decode_heads/uper_head.py @@ -0,0 +1,126 @@ +import torch +import torch.nn as nn +from mmcv.cnn import ConvModule + +from mmseg.ops import resize +from ..builder import HEADS +from .decode_head import BaseDecodeHead +from .psp_head import PPM + + +@HEADS.register_module() +class UPerHead(BaseDecodeHead): + """Unified Perceptual Parsing for Scene Understanding. + + This head is the implementation of `UPerNet + `_. + + Args: + pool_scales (tuple[int]): Pooling scales used in Pooling Pyramid + Module applied on the last feature. Default: (1, 2, 3, 6). + """ + + def __init__(self, pool_scales=(1, 2, 3, 6), **kwargs): + super(UPerHead, self).__init__( + input_transform='multiple_select', **kwargs) + # PSP Module + self.psp_modules = PPM( + pool_scales, + self.in_channels[-1], + self.channels, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg, + align_corners=self.align_corners) + self.bottleneck = ConvModule( + self.in_channels[-1] + len(pool_scales) * self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + # FPN Module + self.lateral_convs = nn.ModuleList() + self.fpn_convs = nn.ModuleList() + for in_channels in self.in_channels[:-1]: # skip the top layer + l_conv = ConvModule( + in_channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg, + inplace=False) + fpn_conv = ConvModule( + self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg, + inplace=False) + self.lateral_convs.append(l_conv) + self.fpn_convs.append(fpn_conv) + + self.fpn_bottleneck = ConvModule( + len(self.in_channels) * self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def psp_forward(self, inputs): + """Forward function of PSP module.""" + x = inputs[-1] + psp_outs = [x] + psp_outs.extend(self.psp_modules(x)) + psp_outs = torch.cat(psp_outs, dim=1) + output = self.bottleneck(psp_outs) + + return output + + def forward(self, inputs): + """Forward function.""" + + inputs = self._transform_inputs(inputs) + + # build laterals + laterals = [ + lateral_conv(inputs[i]) + for i, lateral_conv in enumerate(self.lateral_convs) + ] + + laterals.append(self.psp_forward(inputs)) + + # build top-down path + used_backbone_levels = len(laterals) + for i in range(used_backbone_levels - 1, 0, -1): + prev_shape = laterals[i - 1].shape[2:] + laterals[i - 1] += resize( + laterals[i], + size=prev_shape, + mode='bilinear', + align_corners=self.align_corners) + + # build outputs + fpn_outs = [ + self.fpn_convs[i](laterals[i]) + for i in range(used_backbone_levels - 1) + ] + # append psp feature + fpn_outs.append(laterals[-1]) + + for i in range(used_backbone_levels - 1, 0, -1): + fpn_outs[i] = resize( + fpn_outs[i], + size=fpn_outs[0].shape[2:], + mode='bilinear', + align_corners=self.align_corners) + fpn_outs = torch.cat(fpn_outs, dim=1) + output = self.fpn_bottleneck(fpn_outs) + output = self.cls_seg(output) + return output diff --git a/segmentation/mmseg/models/losses/__init__.py b/segmentation/mmseg/models/losses/__init__.py new file mode 100644 index 0000000..beca720 --- /dev/null +++ b/segmentation/mmseg/models/losses/__init__.py @@ -0,0 +1,12 @@ +from .accuracy import Accuracy, accuracy +from .cross_entropy_loss import (CrossEntropyLoss, binary_cross_entropy, + cross_entropy, mask_cross_entropy) +from .dice_loss import DiceLoss +from .lovasz_loss import LovaszLoss +from .utils import reduce_loss, weight_reduce_loss, weighted_loss + +__all__ = [ + 'accuracy', 'Accuracy', 'cross_entropy', 'binary_cross_entropy', + 'mask_cross_entropy', 'CrossEntropyLoss', 'reduce_loss', + 'weight_reduce_loss', 'weighted_loss', 'LovaszLoss', 'DiceLoss' +] diff --git a/segmentation/mmseg/models/losses/accuracy.py b/segmentation/mmseg/models/losses/accuracy.py new file mode 100644 index 0000000..c0fd2e7 --- /dev/null +++ b/segmentation/mmseg/models/losses/accuracy.py @@ -0,0 +1,78 @@ +import torch.nn as nn + + +def accuracy(pred, target, topk=1, thresh=None): + """Calculate accuracy according to the prediction and target. + + Args: + pred (torch.Tensor): The model prediction, shape (N, num_class, ...) + target (torch.Tensor): The target of each prediction, shape (N, , ...) + topk (int | tuple[int], optional): If the predictions in ``topk`` + matches the target, the predictions will be regarded as + correct ones. Defaults to 1. + thresh (float, optional): If not None, predictions with scores under + this threshold are considered incorrect. Default to None. + + Returns: + float | tuple[float]: If the input ``topk`` is a single integer, + the function will return a single float as accuracy. If + ``topk`` is a tuple containing multiple integers, the + function will return a tuple containing accuracies of + each ``topk`` number. + """ + assert isinstance(topk, (int, tuple)) + if isinstance(topk, int): + topk = (topk, ) + return_single = True + else: + return_single = False + + maxk = max(topk) + if pred.size(0) == 0: + accu = [pred.new_tensor(0.) for i in range(len(topk))] + return accu[0] if return_single else accu + assert pred.ndim == target.ndim + 1 + assert pred.size(0) == target.size(0) + assert maxk <= pred.size(1), \ + f'maxk {maxk} exceeds pred dimension {pred.size(1)}' + pred_value, pred_label = pred.topk(maxk, dim=1) + # transpose to shape (maxk, N, ...) + pred_label = pred_label.transpose(0, 1) + correct = pred_label.eq(target.unsqueeze(0).expand_as(pred_label)) + if thresh is not None: + # Only prediction values larger than thresh are counted as correct + correct = correct & (pred_value > thresh).t() + res = [] + for k in topk: + correct_k = correct[:k].reshape(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / target.numel())) + return res[0] if return_single else res + + +class Accuracy(nn.Module): + """Accuracy calculation module.""" + + def __init__(self, topk=(1, ), thresh=None): + """Module to calculate the accuracy. + + Args: + topk (tuple, optional): The criterion used to calculate the + accuracy. Defaults to (1,). + thresh (float, optional): If not None, predictions with scores + under this threshold are considered incorrect. Default to None. + """ + super().__init__() + self.topk = topk + self.thresh = thresh + + def forward(self, pred, target): + """Forward function to calculate accuracy. + + Args: + pred (torch.Tensor): Prediction of models. + target (torch.Tensor): Target for each prediction. + + Returns: + tuple[float]: The accuracies under different topk criterions. + """ + return accuracy(pred, target, self.topk, self.thresh) diff --git a/segmentation/mmseg/models/losses/cross_entropy_loss.py b/segmentation/mmseg/models/losses/cross_entropy_loss.py new file mode 100644 index 0000000..42c0790 --- /dev/null +++ b/segmentation/mmseg/models/losses/cross_entropy_loss.py @@ -0,0 +1,198 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ..builder import LOSSES +from .utils import get_class_weight, weight_reduce_loss + + +def cross_entropy(pred, + label, + weight=None, + class_weight=None, + reduction='mean', + avg_factor=None, + ignore_index=-100): + """The wrapper function for :func:`F.cross_entropy`""" + # class_weight is a manual rescaling weight given to each class. + # If given, has to be a Tensor of size C element-wise losses + loss = F.cross_entropy( + pred, + label, + weight=class_weight, + reduction='none', + ignore_index=ignore_index) + + # apply weights and do the reduction + if weight is not None: + weight = weight.float() + loss = weight_reduce_loss( + loss, weight=weight, reduction=reduction, avg_factor=avg_factor) + + return loss + + +def _expand_onehot_labels(labels, label_weights, target_shape, ignore_index): + """Expand onehot labels to match the size of prediction.""" + bin_labels = labels.new_zeros(target_shape) + valid_mask = (labels >= 0) & (labels != ignore_index) + inds = torch.nonzero(valid_mask, as_tuple=True) + + if inds[0].numel() > 0: + if labels.dim() == 3: + bin_labels[inds[0], labels[valid_mask], inds[1], inds[2]] = 1 + else: + bin_labels[inds[0], labels[valid_mask]] = 1 + + valid_mask = valid_mask.unsqueeze(1).expand(target_shape).float() + if label_weights is None: + bin_label_weights = valid_mask + else: + bin_label_weights = label_weights.unsqueeze(1).expand(target_shape) + bin_label_weights *= valid_mask + + return bin_labels, bin_label_weights + + +def binary_cross_entropy(pred, + label, + weight=None, + reduction='mean', + avg_factor=None, + class_weight=None, + ignore_index=255): + """Calculate the binary CrossEntropy loss. + + Args: + pred (torch.Tensor): The prediction with shape (N, 1). + label (torch.Tensor): The learning label of the prediction. + weight (torch.Tensor, optional): Sample-wise loss weight. + reduction (str, optional): The method used to reduce the loss. + Options are "none", "mean" and "sum". + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + class_weight (list[float], optional): The weight for each class. + ignore_index (int | None): The label index to be ignored. Default: 255 + + Returns: + torch.Tensor: The calculated loss + """ + if pred.dim() != label.dim(): + assert (pred.dim() == 2 and label.dim() == 1) or ( + pred.dim() == 4 and label.dim() == 3), \ + 'Only pred shape [N, C], label shape [N] or pred shape [N, C, ' \ + 'H, W], label shape [N, H, W] are supported' + label, weight = _expand_onehot_labels(label, weight, pred.shape, + ignore_index) + + # weighted element-wise losses + if weight is not None: + weight = weight.float() + loss = F.binary_cross_entropy_with_logits( + pred, label.float(), pos_weight=class_weight, reduction='none') + # do the reduction for the weighted loss + loss = weight_reduce_loss( + loss, weight, reduction=reduction, avg_factor=avg_factor) + + return loss + + +def mask_cross_entropy(pred, + target, + label, + reduction='mean', + avg_factor=None, + class_weight=None, + ignore_index=None): + """Calculate the CrossEntropy loss for masks. + + Args: + pred (torch.Tensor): The prediction with shape (N, C), C is the number + of classes. + target (torch.Tensor): The learning label of the prediction. + label (torch.Tensor): ``label`` indicates the class label of the mask' + corresponding object. This will be used to select the mask in the + of the class which the object belongs to when the mask prediction + if not class-agnostic. + reduction (str, optional): The method used to reduce the loss. + Options are "none", "mean" and "sum". + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + class_weight (list[float], optional): The weight for each class. + ignore_index (None): Placeholder, to be consistent with other loss. + Default: None. + + Returns: + torch.Tensor: The calculated loss + """ + assert ignore_index is None, 'BCE loss does not support ignore_index' + # TODO: handle these two reserved arguments + assert reduction == 'mean' and avg_factor is None + num_rois = pred.size()[0] + inds = torch.arange(0, num_rois, dtype=torch.long, device=pred.device) + pred_slice = pred[inds, label].squeeze(1) + return F.binary_cross_entropy_with_logits( + pred_slice, target, weight=class_weight, reduction='mean')[None] + + +@LOSSES.register_module() +class CrossEntropyLoss(nn.Module): + """CrossEntropyLoss. + + Args: + use_sigmoid (bool, optional): Whether the prediction uses sigmoid + of softmax. Defaults to False. + use_mask (bool, optional): Whether to use mask cross entropy loss. + Defaults to False. + reduction (str, optional): . Defaults to 'mean'. + Options are "none", "mean" and "sum". + class_weight (list[float] | str, optional): Weight of each class. If in + str format, read them from a file. Defaults to None. + loss_weight (float, optional): Weight of the loss. Defaults to 1.0. + """ + + def __init__(self, + use_sigmoid=False, + use_mask=False, + reduction='mean', + class_weight=None, + loss_weight=1.0): + super(CrossEntropyLoss, self).__init__() + assert (use_sigmoid is False) or (use_mask is False) + self.use_sigmoid = use_sigmoid + self.use_mask = use_mask + self.reduction = reduction + self.loss_weight = loss_weight + self.class_weight = get_class_weight(class_weight) + + if self.use_sigmoid: + self.cls_criterion = binary_cross_entropy + elif self.use_mask: + self.cls_criterion = mask_cross_entropy + else: + self.cls_criterion = cross_entropy + + def forward(self, + cls_score, + label, + weight=None, + avg_factor=None, + reduction_override=None, + **kwargs): + """Forward function.""" + assert reduction_override in (None, 'none', 'mean', 'sum') + reduction = ( + reduction_override if reduction_override else self.reduction) + if self.class_weight is not None: + class_weight = cls_score.new_tensor(self.class_weight) + else: + class_weight = None + loss_cls = self.loss_weight * self.cls_criterion( + cls_score, + label, + weight, + class_weight=class_weight, + reduction=reduction, + avg_factor=avg_factor, + **kwargs) + return loss_cls diff --git a/segmentation/mmseg/models/losses/dice_loss.py b/segmentation/mmseg/models/losses/dice_loss.py new file mode 100644 index 0000000..27a77b9 --- /dev/null +++ b/segmentation/mmseg/models/losses/dice_loss.py @@ -0,0 +1,119 @@ +"""Modified from https://github.com/LikeLy-Journey/SegmenTron/blob/master/ +segmentron/solver/loss.py (Apache-2.0 License)""" +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ..builder import LOSSES +from .utils import get_class_weight, weighted_loss + + +@weighted_loss +def dice_loss(pred, + target, + valid_mask, + smooth=1, + exponent=2, + class_weight=None, + ignore_index=255): + assert pred.shape[0] == target.shape[0] + total_loss = 0 + num_classes = pred.shape[1] + for i in range(num_classes): + if i != ignore_index: + dice_loss = binary_dice_loss( + pred[:, i], + target[..., i], + valid_mask=valid_mask, + smooth=smooth, + exponent=exponent) + if class_weight is not None: + dice_loss *= class_weight[i] + total_loss += dice_loss + return total_loss / num_classes + + +@weighted_loss +def binary_dice_loss(pred, target, valid_mask, smooth=1, exponent=2, **kwards): + assert pred.shape[0] == target.shape[0] + pred = pred.reshape(pred.shape[0], -1) + target = target.reshape(target.shape[0], -1) + valid_mask = valid_mask.reshape(valid_mask.shape[0], -1) + + num = torch.sum(torch.mul(pred, target) * valid_mask, dim=1) * 2 + smooth + den = torch.sum(pred.pow(exponent) + target.pow(exponent), dim=1) + smooth + + return 1 - num / den + + +@LOSSES.register_module() +class DiceLoss(nn.Module): + """DiceLoss. + + This loss is proposed in `V-Net: Fully Convolutional Neural Networks for + Volumetric Medical Image Segmentation `_. + + Args: + loss_type (str, optional): Binary or multi-class loss. + Default: 'multi_class'. Options are "binary" and "multi_class". + smooth (float): A float number to smooth loss, and avoid NaN error. + Default: 1 + exponent (float): An float number to calculate denominator + value: \\sum{x^exponent} + \\sum{y^exponent}. Default: 2. + reduction (str, optional): The method used to reduce the loss. Options + are "none", "mean" and "sum". This parameter only works when + per_image is True. Default: 'mean'. + class_weight (list[float] | str, optional): Weight of each class. If in + str format, read them from a file. Defaults to None. + loss_weight (float, optional): Weight of the loss. Default to 1.0. + ignore_index (int | None): The label index to be ignored. Default: 255. + """ + + def __init__(self, + smooth=1, + exponent=2, + reduction='mean', + class_weight=None, + loss_weight=1.0, + ignore_index=255, + **kwards): + super(DiceLoss, self).__init__() + self.smooth = smooth + self.exponent = exponent + self.reduction = reduction + self.class_weight = get_class_weight(class_weight) + self.loss_weight = loss_weight + self.ignore_index = ignore_index + + def forward(self, + pred, + target, + avg_factor=None, + reduction_override=None, + **kwards): + assert reduction_override in (None, 'none', 'mean', 'sum') + reduction = ( + reduction_override if reduction_override else self.reduction) + if self.class_weight is not None: + class_weight = pred.new_tensor(self.class_weight) + else: + class_weight = None + + pred = F.softmax(pred, dim=1) + num_classes = pred.shape[1] + one_hot_target = F.one_hot( + torch.clamp(target.long(), 0, num_classes - 1), + num_classes=num_classes) + valid_mask = (target != self.ignore_index).long() + + loss = self.loss_weight * dice_loss( + pred, + one_hot_target, + valid_mask=valid_mask, + reduction=reduction, + avg_factor=avg_factor, + smooth=self.smooth, + exponent=self.exponent, + class_weight=class_weight, + ignore_index=self.ignore_index) + return loss diff --git a/segmentation/mmseg/models/losses/lovasz_loss.py b/segmentation/mmseg/models/losses/lovasz_loss.py new file mode 100644 index 0000000..e8df6e8 --- /dev/null +++ b/segmentation/mmseg/models/losses/lovasz_loss.py @@ -0,0 +1,303 @@ +"""Modified from https://github.com/bermanmaxim/LovaszSoftmax/blob/master/pytor +ch/lovasz_losses.py Lovasz-Softmax and Jaccard hinge loss in PyTorch Maxim +Berman 2018 ESAT-PSI KU Leuven (MIT License)""" + +import mmcv +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ..builder import LOSSES +from .utils import get_class_weight, weight_reduce_loss + + +def lovasz_grad(gt_sorted): + """Computes gradient of the Lovasz extension w.r.t sorted errors. + + See Alg. 1 in paper. + """ + p = len(gt_sorted) + gts = gt_sorted.sum() + intersection = gts - gt_sorted.float().cumsum(0) + union = gts + (1 - gt_sorted).float().cumsum(0) + jaccard = 1. - intersection / union + if p > 1: # cover 1-pixel case + jaccard[1:p] = jaccard[1:p] - jaccard[0:-1] + return jaccard + + +def flatten_binary_logits(logits, labels, ignore_index=None): + """Flattens predictions in the batch (binary case) Remove labels equal to + 'ignore_index'.""" + logits = logits.view(-1) + labels = labels.view(-1) + if ignore_index is None: + return logits, labels + valid = (labels != ignore_index) + vlogits = logits[valid] + vlabels = labels[valid] + return vlogits, vlabels + + +def flatten_probs(probs, labels, ignore_index=None): + """Flattens predictions in the batch.""" + if probs.dim() == 3: + # assumes output of a sigmoid layer + B, H, W = probs.size() + probs = probs.view(B, 1, H, W) + B, C, H, W = probs.size() + probs = probs.permute(0, 2, 3, 1).contiguous().view(-1, C) # B*H*W, C=P,C + labels = labels.view(-1) + if ignore_index is None: + return probs, labels + valid = (labels != ignore_index) + vprobs = probs[valid.nonzero().squeeze()] + vlabels = labels[valid] + return vprobs, vlabels + + +def lovasz_hinge_flat(logits, labels): + """Binary Lovasz hinge loss. + + Args: + logits (torch.Tensor): [P], logits at each prediction + (between -infty and +infty). + labels (torch.Tensor): [P], binary ground truth labels (0 or 1). + + Returns: + torch.Tensor: The calculated loss. + """ + if len(labels) == 0: + # only void pixels, the gradients should be 0 + return logits.sum() * 0. + signs = 2. * labels.float() - 1. + errors = (1. - logits * signs) + errors_sorted, perm = torch.sort(errors, dim=0, descending=True) + perm = perm.data + gt_sorted = labels[perm] + grad = lovasz_grad(gt_sorted) + loss = torch.dot(F.relu(errors_sorted), grad) + return loss + + +def lovasz_hinge(logits, + labels, + classes='present', + per_image=False, + class_weight=None, + reduction='mean', + avg_factor=None, + ignore_index=255): + """Binary Lovasz hinge loss. + + Args: + logits (torch.Tensor): [B, H, W], logits at each pixel + (between -infty and +infty). + labels (torch.Tensor): [B, H, W], binary ground truth masks (0 or 1). + classes (str | list[int], optional): Placeholder, to be consistent with + other loss. Default: None. + per_image (bool, optional): If per_image is True, compute the loss per + image instead of per batch. Default: False. + class_weight (list[float], optional): Placeholder, to be consistent + with other loss. Default: None. + reduction (str, optional): The method used to reduce the loss. Options + are "none", "mean" and "sum". This parameter only works when + per_image is True. Default: 'mean'. + avg_factor (int, optional): Average factor that is used to average + the loss. This parameter only works when per_image is True. + Default: None. + ignore_index (int | None): The label index to be ignored. Default: 255. + + Returns: + torch.Tensor: The calculated loss. + """ + if per_image: + loss = [ + lovasz_hinge_flat(*flatten_binary_logits( + logit.unsqueeze(0), label.unsqueeze(0), ignore_index)) + for logit, label in zip(logits, labels) + ] + loss = weight_reduce_loss( + torch.stack(loss), None, reduction, avg_factor) + else: + loss = lovasz_hinge_flat( + *flatten_binary_logits(logits, labels, ignore_index)) + return loss + + +def lovasz_softmax_flat(probs, labels, classes='present', class_weight=None): + """Multi-class Lovasz-Softmax loss. + + Args: + probs (torch.Tensor): [P, C], class probabilities at each prediction + (between 0 and 1). + labels (torch.Tensor): [P], ground truth labels (between 0 and C - 1). + classes (str | list[int], optional): Classes chosen to calculate loss. + 'all' for all classes, 'present' for classes present in labels, or + a list of classes to average. Default: 'present'. + class_weight (list[float], optional): The weight for each class. + Default: None. + + Returns: + torch.Tensor: The calculated loss. + """ + if probs.numel() == 0: + # only void pixels, the gradients should be 0 + return probs * 0. + C = probs.size(1) + losses = [] + class_to_sum = list(range(C)) if classes in ['all', 'present'] else classes + for c in class_to_sum: + fg = (labels == c).float() # foreground for class c + if (classes == 'present' and fg.sum() == 0): + continue + if C == 1: + if len(classes) > 1: + raise ValueError('Sigmoid output possible only with 1 class') + class_pred = probs[:, 0] + else: + class_pred = probs[:, c] + errors = (fg - class_pred).abs() + errors_sorted, perm = torch.sort(errors, 0, descending=True) + perm = perm.data + fg_sorted = fg[perm] + loss = torch.dot(errors_sorted, lovasz_grad(fg_sorted)) + if class_weight is not None: + loss *= class_weight[c] + losses.append(loss) + return torch.stack(losses).mean() + + +def lovasz_softmax(probs, + labels, + classes='present', + per_image=False, + class_weight=None, + reduction='mean', + avg_factor=None, + ignore_index=255): + """Multi-class Lovasz-Softmax loss. + + Args: + probs (torch.Tensor): [B, C, H, W], class probabilities at each + prediction (between 0 and 1). + labels (torch.Tensor): [B, H, W], ground truth labels (between 0 and + C - 1). + classes (str | list[int], optional): Classes chosen to calculate loss. + 'all' for all classes, 'present' for classes present in labels, or + a list of classes to average. Default: 'present'. + per_image (bool, optional): If per_image is True, compute the loss per + image instead of per batch. Default: False. + class_weight (list[float], optional): The weight for each class. + Default: None. + reduction (str, optional): The method used to reduce the loss. Options + are "none", "mean" and "sum". This parameter only works when + per_image is True. Default: 'mean'. + avg_factor (int, optional): Average factor that is used to average + the loss. This parameter only works when per_image is True. + Default: None. + ignore_index (int | None): The label index to be ignored. Default: 255. + + Returns: + torch.Tensor: The calculated loss. + """ + + if per_image: + loss = [ + lovasz_softmax_flat( + *flatten_probs( + prob.unsqueeze(0), label.unsqueeze(0), ignore_index), + classes=classes, + class_weight=class_weight) + for prob, label in zip(probs, labels) + ] + loss = weight_reduce_loss( + torch.stack(loss), None, reduction, avg_factor) + else: + loss = lovasz_softmax_flat( + *flatten_probs(probs, labels, ignore_index), + classes=classes, + class_weight=class_weight) + return loss + + +@LOSSES.register_module() +class LovaszLoss(nn.Module): + """LovaszLoss. + + This loss is proposed in `The Lovasz-Softmax loss: A tractable surrogate + for the optimization of the intersection-over-union measure in neural + networks `_. + + Args: + loss_type (str, optional): Binary or multi-class loss. + Default: 'multi_class'. Options are "binary" and "multi_class". + classes (str | list[int], optional): Classes chosen to calculate loss. + 'all' for all classes, 'present' for classes present in labels, or + a list of classes to average. Default: 'present'. + per_image (bool, optional): If per_image is True, compute the loss per + image instead of per batch. Default: False. + reduction (str, optional): The method used to reduce the loss. Options + are "none", "mean" and "sum". This parameter only works when + per_image is True. Default: 'mean'. + class_weight (list[float] | str, optional): Weight of each class. If in + str format, read them from a file. Defaults to None. + loss_weight (float, optional): Weight of the loss. Defaults to 1.0. + """ + + def __init__(self, + loss_type='multi_class', + classes='present', + per_image=False, + reduction='mean', + class_weight=None, + loss_weight=1.0): + super(LovaszLoss, self).__init__() + assert loss_type in ('binary', 'multi_class'), "loss_type should be \ + 'binary' or 'multi_class'." + + if loss_type == 'binary': + self.cls_criterion = lovasz_hinge + else: + self.cls_criterion = lovasz_softmax + assert classes in ('all', 'present') or mmcv.is_list_of(classes, int) + if not per_image: + assert reduction == 'none', "reduction should be 'none' when \ + per_image is False." + + self.classes = classes + self.per_image = per_image + self.reduction = reduction + self.loss_weight = loss_weight + self.class_weight = get_class_weight(class_weight) + + def forward(self, + cls_score, + label, + weight=None, + avg_factor=None, + reduction_override=None, + **kwargs): + """Forward function.""" + assert reduction_override in (None, 'none', 'mean', 'sum') + reduction = ( + reduction_override if reduction_override else self.reduction) + if self.class_weight is not None: + class_weight = cls_score.new_tensor(self.class_weight) + else: + class_weight = None + + # if multi-class loss, transform logits to probs + if self.cls_criterion == lovasz_softmax: + cls_score = F.softmax(cls_score, dim=1) + + loss_cls = self.loss_weight * self.cls_criterion( + cls_score, + label, + self.classes, + self.per_image, + class_weight=class_weight, + reduction=reduction, + avg_factor=avg_factor, + **kwargs) + return loss_cls diff --git a/segmentation/mmseg/models/losses/utils.py b/segmentation/mmseg/models/losses/utils.py new file mode 100644 index 0000000..ab58766 --- /dev/null +++ b/segmentation/mmseg/models/losses/utils.py @@ -0,0 +1,121 @@ +import functools + +import mmcv +import numpy as np +import torch.nn.functional as F + + +def get_class_weight(class_weight): + """Get class weight for loss function. + + Args: + class_weight (list[float] | str | None): If class_weight is a str, + take it as a file name and read from it. + """ + if isinstance(class_weight, str): + # take it as a file path + if class_weight.endswith('.npy'): + class_weight = np.load(class_weight) + else: + # pkl, json or yaml + class_weight = mmcv.load(class_weight) + + return class_weight + + +def reduce_loss(loss, reduction): + """Reduce loss as specified. + + Args: + loss (Tensor): Elementwise loss tensor. + reduction (str): Options are "none", "mean" and "sum". + + Return: + Tensor: Reduced loss tensor. + """ + reduction_enum = F._Reduction.get_enum(reduction) + # none: 0, elementwise_mean:1, sum: 2 + if reduction_enum == 0: + return loss + elif reduction_enum == 1: + return loss.mean() + elif reduction_enum == 2: + return loss.sum() + + +def weight_reduce_loss(loss, weight=None, reduction='mean', avg_factor=None): + """Apply element-wise weight and reduce loss. + + Args: + loss (Tensor): Element-wise loss. + weight (Tensor): Element-wise weights. + reduction (str): Same as built-in losses of PyTorch. + avg_factor (float): Avarage factor when computing the mean of losses. + + Returns: + Tensor: Processed loss values. + """ + # if weight is specified, apply element-wise weight + if weight is not None: + assert weight.dim() == loss.dim() + if weight.dim() > 1: + assert weight.size(1) == 1 or weight.size(1) == loss.size(1) + loss = loss * weight + + # if avg_factor is not specified, just reduce the loss + if avg_factor is None: + loss = reduce_loss(loss, reduction) + else: + # if reduction is mean, then average the loss by avg_factor + if reduction == 'mean': + loss = loss.sum() / avg_factor + # if reduction is 'none', then do nothing, otherwise raise an error + elif reduction != 'none': + raise ValueError('avg_factor can not be used with reduction="sum"') + return loss + + +def weighted_loss(loss_func): + """Create a weighted version of a given loss function. + + To use this decorator, the loss function must have the signature like + `loss_func(pred, target, **kwargs)`. The function only needs to compute + element-wise loss without any reduction. This decorator will add weight + and reduction arguments to the function. The decorated function will have + the signature like `loss_func(pred, target, weight=None, reduction='mean', + avg_factor=None, **kwargs)`. + + :Example: + + >>> import torch + >>> @weighted_loss + >>> def l1_loss(pred, target): + >>> return (pred - target).abs() + + >>> pred = torch.Tensor([0, 2, 3]) + >>> target = torch.Tensor([1, 1, 1]) + >>> weight = torch.Tensor([1, 0, 1]) + + >>> l1_loss(pred, target) + tensor(1.3333) + >>> l1_loss(pred, target, weight) + tensor(1.) + >>> l1_loss(pred, target, reduction='none') + tensor([1., 1., 2.]) + >>> l1_loss(pred, target, weight, avg_factor=2) + tensor(1.5000) + """ + + @functools.wraps(loss_func) + def wrapper(pred, + target, + weight=None, + reduction='mean', + avg_factor=None, + **kwargs): + # get element-wise loss + loss = loss_func(pred, target, **kwargs) + loss = weight_reduce_loss(loss, weight, reduction, avg_factor) + return loss + + return wrapper diff --git a/segmentation/mmseg/models/necks/__init__.py b/segmentation/mmseg/models/necks/__init__.py new file mode 100644 index 0000000..9b9d3d5 --- /dev/null +++ b/segmentation/mmseg/models/necks/__init__.py @@ -0,0 +1,4 @@ +from .fpn import FPN +from .multilevel_neck import MultiLevelNeck + +__all__ = ['FPN', 'MultiLevelNeck'] diff --git a/segmentation/mmseg/models/necks/fpn.py b/segmentation/mmseg/models/necks/fpn.py new file mode 100644 index 0000000..f43d1e6 --- /dev/null +++ b/segmentation/mmseg/models/necks/fpn.py @@ -0,0 +1,212 @@ +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import ConvModule, xavier_init + +from ..builder import NECKS + + +@NECKS.register_module() +class FPN(nn.Module): + """Feature Pyramid Network. + + This is an implementation of - Feature Pyramid Networks for Object + Detection (https://arxiv.org/abs/1612.03144) + + Args: + in_channels (List[int]): Number of input channels per scale. + out_channels (int): Number of output channels (used at each scale) + num_outs (int): Number of output scales. + start_level (int): Index of the start input backbone level used to + build the feature pyramid. Default: 0. + end_level (int): Index of the end input backbone level (exclusive) to + build the feature pyramid. Default: -1, which means the last level. + add_extra_convs (bool | str): If bool, it decides whether to add conv + layers on top of the original feature maps. Default to False. + If True, its actual mode is specified by `extra_convs_on_inputs`. + If str, it specifies the source feature map of the extra convs. + Only the following options are allowed + + - 'on_input': Last feat map of neck inputs (i.e. backbone feature). + - 'on_lateral': Last feature map after lateral convs. + - 'on_output': The last output feature map after fpn convs. + extra_convs_on_inputs (bool, deprecated): Whether to apply extra convs + on the original feature from the backbone. If True, + it is equivalent to `add_extra_convs='on_input'`. If False, it is + equivalent to set `add_extra_convs='on_output'`. Default to True. + relu_before_extra_convs (bool): Whether to apply relu before the extra + conv. Default: False. + no_norm_on_lateral (bool): Whether to apply norm on lateral. + Default: False. + conv_cfg (dict): Config dict for convolution layer. Default: None. + norm_cfg (dict): Config dict for normalization layer. Default: None. + act_cfg (str): Config dict for activation layer in ConvModule. + Default: None. + upsample_cfg (dict): Config dict for interpolate layer. + Default: `dict(mode='nearest')` + + Example: + >>> import torch + >>> in_channels = [2, 3, 5, 7] + >>> scales = [340, 170, 84, 43] + >>> inputs = [torch.rand(1, c, s, s) + ... for c, s in zip(in_channels, scales)] + >>> self = FPN(in_channels, 11, len(in_channels)).eval() + >>> outputs = self.forward(inputs) + >>> for i in range(len(outputs)): + ... print(f'outputs[{i}].shape = {outputs[i].shape}') + outputs[0].shape = torch.Size([1, 11, 340, 340]) + outputs[1].shape = torch.Size([1, 11, 170, 170]) + outputs[2].shape = torch.Size([1, 11, 84, 84]) + outputs[3].shape = torch.Size([1, 11, 43, 43]) + """ + + def __init__(self, + in_channels, + out_channels, + num_outs, + start_level=0, + end_level=-1, + add_extra_convs=False, + extra_convs_on_inputs=False, + relu_before_extra_convs=False, + no_norm_on_lateral=False, + conv_cfg=None, + norm_cfg=None, + act_cfg=None, + upsample_cfg=dict(mode='nearest')): + super(FPN, self).__init__() + assert isinstance(in_channels, list) + self.in_channels = in_channels + self.out_channels = out_channels + self.num_ins = len(in_channels) + self.num_outs = num_outs + self.relu_before_extra_convs = relu_before_extra_convs + self.no_norm_on_lateral = no_norm_on_lateral + self.fp16_enabled = False + self.upsample_cfg = upsample_cfg.copy() + + if end_level == -1: + self.backbone_end_level = self.num_ins + assert num_outs >= self.num_ins - start_level + else: + # if end_level < inputs, no extra level is allowed + self.backbone_end_level = end_level + assert end_level <= len(in_channels) + assert num_outs == end_level - start_level + self.start_level = start_level + self.end_level = end_level + self.add_extra_convs = add_extra_convs + assert isinstance(add_extra_convs, (str, bool)) + if isinstance(add_extra_convs, str): + # Extra_convs_source choices: 'on_input', 'on_lateral', 'on_output' + assert add_extra_convs in ('on_input', 'on_lateral', 'on_output') + elif add_extra_convs: # True + if extra_convs_on_inputs: + # For compatibility with previous release + # TODO: deprecate `extra_convs_on_inputs` + self.add_extra_convs = 'on_input' + else: + self.add_extra_convs = 'on_output' + + self.lateral_convs = nn.ModuleList() + self.fpn_convs = nn.ModuleList() + + for i in range(self.start_level, self.backbone_end_level): + l_conv = ConvModule( + in_channels[i], + out_channels, + 1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg if not self.no_norm_on_lateral else None, + act_cfg=act_cfg, + inplace=False) + fpn_conv = ConvModule( + out_channels, + out_channels, + 3, + padding=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + inplace=False) + + self.lateral_convs.append(l_conv) + self.fpn_convs.append(fpn_conv) + + # add extra conv layers (e.g., RetinaNet) + extra_levels = num_outs - self.backbone_end_level + self.start_level + if self.add_extra_convs and extra_levels >= 1: + for i in range(extra_levels): + if i == 0 and self.add_extra_convs == 'on_input': + in_channels = self.in_channels[self.backbone_end_level - 1] + else: + in_channels = out_channels + extra_fpn_conv = ConvModule( + in_channels, + out_channels, + 3, + stride=2, + padding=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + inplace=False) + self.fpn_convs.append(extra_fpn_conv) + + # default init_weights for conv(msra) and norm in ConvModule + def init_weights(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + xavier_init(m, distribution='uniform') + + def forward(self, inputs): + assert len(inputs) == len(self.in_channels) + + # build laterals + laterals = [ + lateral_conv(inputs[i + self.start_level]) + for i, lateral_conv in enumerate(self.lateral_convs) + ] + + # build top-down path + used_backbone_levels = len(laterals) + for i in range(used_backbone_levels - 1, 0, -1): + # In some cases, fixing `scale factor` (e.g. 2) is preferred, but + # it cannot co-exist with `size` in `F.interpolate`. + if 'scale_factor' in self.upsample_cfg: + laterals[i - 1] += F.interpolate(laterals[i], + **self.upsample_cfg) + else: + prev_shape = laterals[i - 1].shape[2:] + laterals[i - 1] += F.interpolate( + laterals[i], size=prev_shape, **self.upsample_cfg) + + # build outputs + # part 1: from original levels + outs = [ + self.fpn_convs[i](laterals[i]) for i in range(used_backbone_levels) + ] + # part 2: add extra levels + if self.num_outs > len(outs): + # use max pool to get more levels on top of outputs + # (e.g., Faster R-CNN, Mask R-CNN) + if not self.add_extra_convs: + for i in range(self.num_outs - used_backbone_levels): + outs.append(F.max_pool2d(outs[-1], 1, stride=2)) + # add conv layers on top of original feature maps (RetinaNet) + else: + if self.add_extra_convs == 'on_input': + extra_source = inputs[self.backbone_end_level - 1] + elif self.add_extra_convs == 'on_lateral': + extra_source = laterals[-1] + elif self.add_extra_convs == 'on_output': + extra_source = outs[-1] + else: + raise NotImplementedError + outs.append(self.fpn_convs[used_backbone_levels](extra_source)) + for i in range(used_backbone_levels + 1, self.num_outs): + if self.relu_before_extra_convs: + outs.append(self.fpn_convs[i](F.relu(outs[-1]))) + else: + outs.append(self.fpn_convs[i](outs[-1])) + return tuple(outs) diff --git a/segmentation/mmseg/models/necks/multilevel_neck.py b/segmentation/mmseg/models/necks/multilevel_neck.py new file mode 100644 index 0000000..941b829 --- /dev/null +++ b/segmentation/mmseg/models/necks/multilevel_neck.py @@ -0,0 +1,69 @@ +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import ConvModule + +from ..builder import NECKS + + +@NECKS.register_module() +class MultiLevelNeck(nn.Module): + """MultiLevelNeck. + + A neck structure connect vit backbone and decoder_heads. + Args: + in_channels (List[int]): Number of input channels per scale. + out_channels (int): Number of output channels (used at each scale). + scales (List[int]): Scale factors for each input feature map. + norm_cfg (dict): Config dict for normalization layer. Default: None. + act_cfg (dict): Config dict for activation layer in ConvModule. + Default: None. + """ + + def __init__(self, + in_channels, + out_channels, + scales=[0.5, 1, 2, 4], + norm_cfg=None, + act_cfg=None): + super(MultiLevelNeck, self).__init__() + assert isinstance(in_channels, list) + self.in_channels = in_channels + self.out_channels = out_channels + self.scales = scales + self.num_outs = len(scales) + self.lateral_convs = nn.ModuleList() + self.convs = nn.ModuleList() + for in_channel in in_channels: + self.lateral_convs.append( + ConvModule( + in_channel, + out_channels, + kernel_size=1, + norm_cfg=norm_cfg, + act_cfg=act_cfg)) + for _ in range(self.num_outs): + self.convs.append( + ConvModule( + out_channels, + out_channels, + kernel_size=3, + padding=1, + stride=1, + norm_cfg=norm_cfg, + act_cfg=act_cfg)) + + def forward(self, inputs): + assert len(inputs) == len(self.in_channels) + inputs = [ + lateral_conv(inputs[i]) + for i, lateral_conv in enumerate(self.lateral_convs) + ] + # for len(inputs) not equal to self.num_outs + if len(inputs) == 1: + inputs = [inputs[0] for _ in range(self.num_outs)] + outs = [] + for i in range(self.num_outs): + x_resize = F.interpolate( + inputs[i], scale_factor=self.scales[i], mode='bilinear') + outs.append(self.convs[i](x_resize)) + return tuple(outs) diff --git a/segmentation/mmseg/models/segmentors/__init__.py b/segmentation/mmseg/models/segmentors/__init__.py new file mode 100644 index 0000000..dca2f09 --- /dev/null +++ b/segmentation/mmseg/models/segmentors/__init__.py @@ -0,0 +1,5 @@ +from .base import BaseSegmentor +from .cascade_encoder_decoder import CascadeEncoderDecoder +from .encoder_decoder import EncoderDecoder + +__all__ = ['BaseSegmentor', 'EncoderDecoder', 'CascadeEncoderDecoder'] diff --git a/segmentation/mmseg/models/segmentors/base.py b/segmentation/mmseg/models/segmentors/base.py new file mode 100644 index 0000000..7b53757 --- /dev/null +++ b/segmentation/mmseg/models/segmentors/base.py @@ -0,0 +1,273 @@ +import logging +import warnings +from abc import ABCMeta, abstractmethod +from collections import OrderedDict + +import mmcv +import numpy as np +import torch +import torch.distributed as dist +import torch.nn as nn +from mmcv.runner import auto_fp16 + + +class BaseSegmentor(nn.Module): + """Base class for segmentors.""" + + __metaclass__ = ABCMeta + + def __init__(self): + super(BaseSegmentor, self).__init__() + self.fp16_enabled = False + + @property + def with_neck(self): + """bool: whether the segmentor has neck""" + return hasattr(self, 'neck') and self.neck is not None + + @property + def with_auxiliary_head(self): + """bool: whether the segmentor has auxiliary head""" + return hasattr(self, + 'auxiliary_head') and self.auxiliary_head is not None + + @property + def with_decode_head(self): + """bool: whether the segmentor has decode head""" + return hasattr(self, 'decode_head') and self.decode_head is not None + + @abstractmethod + def extract_feat(self, imgs): + """Placeholder for extract features from images.""" + pass + + @abstractmethod + def encode_decode(self, img, img_metas): + """Placeholder for encode images with backbone and decode into a + semantic segmentation map of the same size as input.""" + pass + + @abstractmethod + def forward_train(self, imgs, img_metas, **kwargs): + """Placeholder for Forward function for training.""" + pass + + @abstractmethod + def simple_test(self, img, img_meta, **kwargs): + """Placeholder for single image test.""" + pass + + @abstractmethod + def aug_test(self, imgs, img_metas, **kwargs): + """Placeholder for augmentation test.""" + pass + + def init_weights(self, pretrained=None): + """Initialize the weights in segmentor. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + if pretrained is not None: + logger = logging.getLogger() + logger.info(f'load model from: {pretrained}') + + def forward_test(self, imgs, img_metas, **kwargs): + """ + Args: + imgs (List[Tensor]): the outer list indicates test-time + augmentations and inner Tensor should have a shape NxCxHxW, + which contains all images in the batch. + img_metas (List[List[dict]]): the outer list indicates test-time + augs (multiscale, flip, etc.) and the inner list indicates + images in a batch. + """ + for var, name in [(imgs, 'imgs'), (img_metas, 'img_metas')]: + if not isinstance(var, list): + raise TypeError(f'{name} must be a list, but got ' + f'{type(var)}') + + num_augs = len(imgs) + if num_augs != len(img_metas): + raise ValueError(f'num of augmentations ({len(imgs)}) != ' + f'num of image meta ({len(img_metas)})') + # all images in the same aug batch all of the same ori_shape and pad + # shape + for img_meta in img_metas: + ori_shapes = [_['ori_shape'] for _ in img_meta] + assert all(shape == ori_shapes[0] for shape in ori_shapes) + img_shapes = [_['img_shape'] for _ in img_meta] + assert all(shape == img_shapes[0] for shape in img_shapes) + pad_shapes = [_['pad_shape'] for _ in img_meta] + assert all(shape == pad_shapes[0] for shape in pad_shapes) + + if num_augs == 1: + return self.simple_test(imgs[0], img_metas[0], **kwargs) + else: + return self.aug_test(imgs, img_metas, **kwargs) + + @auto_fp16(apply_to=('img', )) + def forward(self, img, img_metas, return_loss=True, **kwargs): + """Calls either :func:`forward_train` or :func:`forward_test` depending + on whether ``return_loss`` is ``True``. + + Note this setting will change the expected inputs. When + ``return_loss=True``, img and img_meta are single-nested (i.e. Tensor + and List[dict]), and when ``resturn_loss=False``, img and img_meta + should be double nested (i.e. List[Tensor], List[List[dict]]), with + the outer list indicating test time augmentations. + """ + if return_loss: + return self.forward_train(img, img_metas, **kwargs) + else: + return self.forward_test(img, img_metas, **kwargs) + + def train_step(self, data_batch, optimizer, **kwargs): + """The iteration step during training. + + This method defines an iteration step during training, except for the + back propagation and optimizer updating, which are done in an optimizer + hook. Note that in some complicated cases or models, the whole process + including back propagation and optimizer updating is also defined in + this method, such as GAN. + + Args: + data (dict): The output of dataloader. + optimizer (:obj:`torch.optim.Optimizer` | dict): The optimizer of + runner is passed to ``train_step()``. This argument is unused + and reserved. + + Returns: + dict: It should contain at least 3 keys: ``loss``, ``log_vars``, + ``num_samples``. + ``loss`` is a tensor for back propagation, which can be a + weighted sum of multiple losses. + ``log_vars`` contains all the variables to be sent to the + logger. + ``num_samples`` indicates the batch size (when the model is + DDP, it means the batch size on each GPU), which is used for + averaging the logs. + """ + losses = self(**data_batch) + loss, log_vars = self._parse_losses(losses) + + outputs = dict( + loss=loss, + log_vars=log_vars, + num_samples=len(data_batch['img_metas'])) + + return outputs + + def val_step(self, data_batch, **kwargs): + """The iteration step during validation. + + This method shares the same signature as :func:`train_step`, but used + during val epochs. Note that the evaluation after training epochs is + not implemented with this method, but an evaluation hook. + """ + output = self(**data_batch, **kwargs) + return output + + @staticmethod + def _parse_losses(losses): + """Parse the raw outputs (losses) of the network. + + Args: + losses (dict): Raw output of the network, which usually contain + losses and other necessary information. + + Returns: + tuple[Tensor, dict]: (loss, log_vars), loss is the loss tensor + which may be a weighted sum of all losses, log_vars contains + all the variables to be sent to the logger. + """ + log_vars = OrderedDict() + for loss_name, loss_value in losses.items(): + if isinstance(loss_value, torch.Tensor): + log_vars[loss_name] = loss_value.mean() + elif isinstance(loss_value, list): + log_vars[loss_name] = sum(_loss.mean() for _loss in loss_value) + else: + raise TypeError( + f'{loss_name} is not a tensor or list of tensors') + + loss = sum(_value for _key, _value in log_vars.items() + if 'loss' in _key) + + log_vars['loss'] = loss + for loss_name, loss_value in log_vars.items(): + # reduce loss when distributed training + if dist.is_available() and dist.is_initialized(): + loss_value = loss_value.data.clone() + dist.all_reduce(loss_value.div_(dist.get_world_size())) + log_vars[loss_name] = loss_value.item() + + return loss, log_vars + + def show_result(self, + img, + result, + palette=None, + win_name='', + show=False, + wait_time=0, + out_file=None, + opacity=0.5): + """Draw `result` over `img`. + + Args: + img (str or Tensor): The image to be displayed. + result (Tensor): The semantic segmentation results to draw over + `img`. + palette (list[list[int]]] | np.ndarray | None): The palette of + segmentation map. If None is given, random palette will be + generated. Default: None + win_name (str): The window name. + wait_time (int): Value of waitKey param. + Default: 0. + show (bool): Whether to show the image. + Default: False. + out_file (str or None): The filename to write the image. + Default: None. + opacity(float): Opacity of painted segmentation map. + Default 0.5. + Must be in (0, 1] range. + Returns: + img (Tensor): Only if not `show` or `out_file` + """ + img = mmcv.imread(img) + img = img.copy() + seg = result[0] + if palette is None: + if self.PALETTE is None: + palette = np.random.randint( + 0, 255, size=(len(self.CLASSES), 3)) + else: + palette = self.PALETTE + palette = np.array(palette) + assert palette.shape[0] == len(self.CLASSES) + assert palette.shape[1] == 3 + assert len(palette.shape) == 2 + assert 0 < opacity <= 1.0 + color_seg = np.zeros((seg.shape[0], seg.shape[1], 3), dtype=np.uint8) + for label, color in enumerate(palette): + color_seg[seg == label, :] = color + # convert to BGR + color_seg = color_seg[..., ::-1] + + img = img * (1 - opacity) + color_seg * opacity + img = img.astype(np.uint8) + # if out_file specified, do not show image in window + if out_file is not None: + show = False + + if show: + mmcv.imshow(img, win_name, wait_time) + if out_file is not None: + mmcv.imwrite(img, out_file) + + if not (show or out_file): + warnings.warn('show==False and out_file is not specified, only ' + 'result image will be returned') + return img diff --git a/segmentation/mmseg/models/segmentors/cascade_encoder_decoder.py b/segmentation/mmseg/models/segmentors/cascade_encoder_decoder.py new file mode 100644 index 0000000..220ab2b --- /dev/null +++ b/segmentation/mmseg/models/segmentors/cascade_encoder_decoder.py @@ -0,0 +1,98 @@ +from torch import nn + +from mmseg.core import add_prefix +from mmseg.ops import resize +from .. import builder +from ..builder import SEGMENTORS +from .encoder_decoder import EncoderDecoder + + +@SEGMENTORS.register_module() +class CascadeEncoderDecoder(EncoderDecoder): + """Cascade Encoder Decoder segmentors. + + CascadeEncoderDecoder almost the same as EncoderDecoder, while decoders of + CascadeEncoderDecoder are cascaded. The output of previous decoder_head + will be the input of next decoder_head. + """ + + def __init__(self, + num_stages, + backbone, + decode_head, + neck=None, + auxiliary_head=None, + train_cfg=None, + test_cfg=None, + pretrained=None): + self.num_stages = num_stages + super(CascadeEncoderDecoder, self).__init__( + backbone=backbone, + decode_head=decode_head, + neck=neck, + auxiliary_head=auxiliary_head, + train_cfg=train_cfg, + test_cfg=test_cfg, + pretrained=pretrained) + + def _init_decode_head(self, decode_head): + """Initialize ``decode_head``""" + assert isinstance(decode_head, list) + assert len(decode_head) == self.num_stages + self.decode_head = nn.ModuleList() + for i in range(self.num_stages): + self.decode_head.append(builder.build_head(decode_head[i])) + self.align_corners = self.decode_head[-1].align_corners + self.num_classes = self.decode_head[-1].num_classes + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone and heads. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + self.backbone.init_weights(pretrained=pretrained) + for i in range(self.num_stages): + self.decode_head[i].init_weights() + if self.with_auxiliary_head: + if isinstance(self.auxiliary_head, nn.ModuleList): + for aux_head in self.auxiliary_head: + aux_head.init_weights() + else: + self.auxiliary_head.init_weights() + + def encode_decode(self, img, img_metas): + """Encode images with backbone and decode into a semantic segmentation + map of the same size as input.""" + x = self.extract_feat(img) + out = self.decode_head[0].forward_test(x, img_metas, self.test_cfg) + for i in range(1, self.num_stages): + out = self.decode_head[i].forward_test(x, out, img_metas, + self.test_cfg) + out = resize( + input=out, + size=img.shape[2:], + mode='bilinear', + align_corners=self.align_corners) + return out + + def _decode_head_forward_train(self, x, img_metas, gt_semantic_seg): + """Run forward function and calculate loss for decode head in + training.""" + losses = dict() + + loss_decode = self.decode_head[0].forward_train( + x, img_metas, gt_semantic_seg, self.train_cfg) + + losses.update(add_prefix(loss_decode, 'decode_0')) + + for i in range(1, self.num_stages): + # forward test again, maybe unnecessary for most methods. + prev_outputs = self.decode_head[i - 1].forward_test( + x, img_metas, self.test_cfg) + loss_decode = self.decode_head[i].forward_train( + x, prev_outputs, img_metas, gt_semantic_seg, self.train_cfg) + losses.update(add_prefix(loss_decode, f'decode_{i}')) + + return losses diff --git a/segmentation/mmseg/models/segmentors/encoder_decoder.py b/segmentation/mmseg/models/segmentors/encoder_decoder.py new file mode 100644 index 0000000..b2d067d --- /dev/null +++ b/segmentation/mmseg/models/segmentors/encoder_decoder.py @@ -0,0 +1,298 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +from mmseg.core import add_prefix +from mmseg.ops import resize +from .. import builder +from ..builder import SEGMENTORS +from .base import BaseSegmentor + + +@SEGMENTORS.register_module() +class EncoderDecoder(BaseSegmentor): + """Encoder Decoder segmentors. + + EncoderDecoder typically consists of backbone, decode_head, auxiliary_head. + Note that auxiliary_head is only used for deep supervision during training, + which could be dumped during inference. + """ + + def __init__(self, + backbone, + decode_head, + neck=None, + auxiliary_head=None, + train_cfg=None, + test_cfg=None, + pretrained=None): + super(EncoderDecoder, self).__init__() + self.backbone = builder.build_backbone(backbone) + if neck is not None: + self.neck = builder.build_neck(neck) + self._init_decode_head(decode_head) + self._init_auxiliary_head(auxiliary_head) + + self.train_cfg = train_cfg + self.test_cfg = test_cfg + + self.init_weights(pretrained=pretrained) + + assert self.with_decode_head + + def _init_decode_head(self, decode_head): + """Initialize ``decode_head``""" + self.decode_head = builder.build_head(decode_head) + self.align_corners = self.decode_head.align_corners + self.num_classes = self.decode_head.num_classes + + def _init_auxiliary_head(self, auxiliary_head): + """Initialize ``auxiliary_head``""" + if auxiliary_head is not None: + if isinstance(auxiliary_head, list): + self.auxiliary_head = nn.ModuleList() + for head_cfg in auxiliary_head: + self.auxiliary_head.append(builder.build_head(head_cfg)) + else: + self.auxiliary_head = builder.build_head(auxiliary_head) + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone and heads. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + + super(EncoderDecoder, self).init_weights(pretrained) + self.backbone.init_weights(pretrained=pretrained) + self.decode_head.init_weights() + if self.with_auxiliary_head: + if isinstance(self.auxiliary_head, nn.ModuleList): + for aux_head in self.auxiliary_head: + aux_head.init_weights() + else: + self.auxiliary_head.init_weights() + + def extract_feat(self, img): + """Extract features from images.""" + x = self.backbone(img) + if self.with_neck: + x = self.neck(x) + return x + + def encode_decode(self, img, img_metas): + """Encode images with backbone and decode into a semantic segmentation + map of the same size as input.""" + x = self.extract_feat(img) + out = self._decode_head_forward_test(x, img_metas) + out = resize( + input=out, + size=img.shape[2:], + mode='bilinear', + align_corners=self.align_corners) + return out + + def _decode_head_forward_train(self, x, img_metas, gt_semantic_seg): + """Run forward function and calculate loss for decode head in + training.""" + losses = dict() + loss_decode = self.decode_head.forward_train(x, img_metas, + gt_semantic_seg, + self.train_cfg) + + losses.update(add_prefix(loss_decode, 'decode')) + return losses + + def _decode_head_forward_test(self, x, img_metas): + """Run forward function and calculate loss for decode head in + inference.""" + seg_logits = self.decode_head.forward_test(x, img_metas, self.test_cfg) + return seg_logits + + def _auxiliary_head_forward_train(self, x, img_metas, gt_semantic_seg): + """Run forward function and calculate loss for auxiliary head in + training.""" + losses = dict() + if isinstance(self.auxiliary_head, nn.ModuleList): + for idx, aux_head in enumerate(self.auxiliary_head): + loss_aux = aux_head.forward_train(x, img_metas, + gt_semantic_seg, + self.train_cfg) + losses.update(add_prefix(loss_aux, f'aux_{idx}')) + else: + loss_aux = self.auxiliary_head.forward_train( + x, img_metas, gt_semantic_seg, self.train_cfg) + losses.update(add_prefix(loss_aux, 'aux')) + + return losses + + def forward_dummy(self, img): + """Dummy forward function.""" + seg_logit = self.encode_decode(img, None) + + return seg_logit + + def forward_train(self, img, img_metas, gt_semantic_seg): + """Forward function for training. + + Args: + img (Tensor): Input images. + img_metas (list[dict]): List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + gt_semantic_seg (Tensor): Semantic segmentation masks + used if the architecture supports semantic segmentation task. + + Returns: + dict[str, Tensor]: a dictionary of loss components + """ + + x = self.extract_feat(img) + + losses = dict() + + loss_decode = self._decode_head_forward_train(x, img_metas, + gt_semantic_seg) + losses.update(loss_decode) + + if self.with_auxiliary_head: + loss_aux = self._auxiliary_head_forward_train( + x, img_metas, gt_semantic_seg) + losses.update(loss_aux) + + return losses + + # TODO refactor + def slide_inference(self, img, img_meta, rescale): + """Inference by sliding-window with overlap. + + If h_crop > h_img or w_crop > w_img, the small patch will be used to + decode without padding. + """ + + h_stride, w_stride = self.test_cfg.stride + h_crop, w_crop = self.test_cfg.crop_size + batch_size, _, h_img, w_img = img.size() + num_classes = self.num_classes + h_grids = max(h_img - h_crop + h_stride - 1, 0) // h_stride + 1 + w_grids = max(w_img - w_crop + w_stride - 1, 0) // w_stride + 1 + preds = img.new_zeros((batch_size, num_classes, h_img, w_img)) + count_mat = img.new_zeros((batch_size, 1, h_img, w_img)) + for h_idx in range(h_grids): + for w_idx in range(w_grids): + y1 = h_idx * h_stride + x1 = w_idx * w_stride + y2 = min(y1 + h_crop, h_img) + x2 = min(x1 + w_crop, w_img) + y1 = max(y2 - h_crop, 0) + x1 = max(x2 - w_crop, 0) + crop_img = img[:, :, y1:y2, x1:x2] + crop_seg_logit = self.encode_decode(crop_img, img_meta) + preds += F.pad(crop_seg_logit, + (int(x1), int(preds.shape[3] - x2), int(y1), + int(preds.shape[2] - y2))) + + count_mat[:, :, y1:y2, x1:x2] += 1 + assert (count_mat == 0).sum() == 0 + if torch.onnx.is_in_onnx_export(): + # cast count_mat to constant while exporting to ONNX + count_mat = torch.from_numpy( + count_mat.cpu().detach().numpy()).to(device=img.device) + preds = preds / count_mat + if rescale: + preds = resize( + preds, + size=img_meta[0]['ori_shape'][:2], + mode='bilinear', + align_corners=self.align_corners, + warning=False) + return preds + + def whole_inference(self, img, img_meta, rescale): + """Inference with full image.""" + + seg_logit = self.encode_decode(img, img_meta) + if rescale: + # support dynamic shape for onnx + if torch.onnx.is_in_onnx_export(): + size = img.shape[2:] + else: + size = img_meta[0]['ori_shape'][:2] + seg_logit = resize( + seg_logit, + size=size, + mode='bilinear', + align_corners=self.align_corners, + warning=False) + + return seg_logit + + def inference(self, img, img_meta, rescale): + """Inference with slide/whole style. + + Args: + img (Tensor): The input image of shape (N, 3, H, W). + img_meta (dict): Image info dict where each dict has: 'img_shape', + 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + rescale (bool): Whether rescale back to original shape. + + Returns: + Tensor: The output segmentation map. + """ + + assert self.test_cfg.mode in ['slide', 'whole'] + ori_shape = img_meta[0]['ori_shape'] + assert all(_['ori_shape'] == ori_shape for _ in img_meta) + if self.test_cfg.mode == 'slide': + seg_logit = self.slide_inference(img, img_meta, rescale) + else: + seg_logit = self.whole_inference(img, img_meta, rescale) + output = F.softmax(seg_logit, dim=1) + flip = img_meta[0]['flip'] + if flip: + flip_direction = img_meta[0]['flip_direction'] + assert flip_direction in ['horizontal', 'vertical'] + if flip_direction == 'horizontal': + output = output.flip(dims=(3, )) + elif flip_direction == 'vertical': + output = output.flip(dims=(2, )) + + return output + + def simple_test(self, img, img_meta, rescale=True): + """Simple test with single image.""" + seg_logit = self.inference(img, img_meta, rescale) + seg_pred = seg_logit.argmax(dim=1) + if torch.onnx.is_in_onnx_export(): + # our inference backend only support 4D output + seg_pred = seg_pred.unsqueeze(0) + return seg_pred + seg_pred = seg_pred.cpu().numpy() + # unravel batch dim + seg_pred = list(seg_pred) + return seg_pred + + def aug_test(self, imgs, img_metas, rescale=True): + """Test with augmentations. + + Only rescale=True is supported. + """ + # aug_test rescale all imgs back to ori_shape for now + assert rescale + # to save memory, we get augmented seg logit inplace + seg_logit = self.inference(imgs[0], img_metas[0], rescale) + for i in range(1, len(imgs)): + cur_seg_logit = self.inference(imgs[i], img_metas[i], rescale) + seg_logit += cur_seg_logit + seg_logit /= len(imgs) + seg_pred = seg_logit.argmax(dim=1) + seg_pred = seg_pred.cpu().numpy() + # unravel batch dim + seg_pred = list(seg_pred) + return seg_pred diff --git a/segmentation/mmseg/models/utils/__init__.py b/segmentation/mmseg/models/utils/__init__.py new file mode 100644 index 0000000..3d3bdd3 --- /dev/null +++ b/segmentation/mmseg/models/utils/__init__.py @@ -0,0 +1,13 @@ +from .drop import DropPath +from .inverted_residual import InvertedResidual, InvertedResidualV3 +from .make_divisible import make_divisible +from .res_layer import ResLayer +from .se_layer import SELayer +from .self_attention_block import SelfAttentionBlock +from .up_conv_block import UpConvBlock +from .weight_init import trunc_normal_ + +__all__ = [ + 'ResLayer', 'SelfAttentionBlock', 'make_divisible', 'InvertedResidual', + 'UpConvBlock', 'InvertedResidualV3', 'SELayer', 'DropPath', 'trunc_normal_' +] diff --git a/segmentation/mmseg/models/utils/drop.py b/segmentation/mmseg/models/utils/drop.py new file mode 100644 index 0000000..4520b0f --- /dev/null +++ b/segmentation/mmseg/models/utils/drop.py @@ -0,0 +1,31 @@ +"""Modified from https://github.com/rwightman/pytorch-image- +models/blob/master/timm/models/layers/drop.py.""" + +import torch +from torch import nn + + +class DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of + residual blocks). + + Args: + drop_prob (float): Drop rate for paths of model. Dropout rate has + to be between 0 and 1. Default: 0. + """ + + def __init__(self, drop_prob=0.): + super(DropPath, self).__init__() + self.drop_prob = drop_prob + self.keep_prob = 1 - drop_prob + + def forward(self, x): + if self.drop_prob == 0. or not self.training: + return x + shape = (x.shape[0], ) + (1, ) * ( + x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets + random_tensor = self.keep_prob + torch.rand( + shape, dtype=x.dtype, device=x.device) + random_tensor.floor_() # binarize + output = x.div(self.keep_prob) * random_tensor + return output diff --git a/segmentation/mmseg/models/utils/inverted_residual.py b/segmentation/mmseg/models/utils/inverted_residual.py new file mode 100644 index 0000000..ede71a2 --- /dev/null +++ b/segmentation/mmseg/models/utils/inverted_residual.py @@ -0,0 +1,208 @@ +from mmcv.cnn import ConvModule +from torch import nn +from torch.utils import checkpoint as cp + +from .se_layer import SELayer + + +class InvertedResidual(nn.Module): + """InvertedResidual block for MobileNetV2. + + Args: + in_channels (int): The input channels of the InvertedResidual block. + out_channels (int): The output channels of the InvertedResidual block. + stride (int): Stride of the middle (first) 3x3 convolution. + expand_ratio (int): Adjusts number of channels of the hidden layer + in InvertedResidual by this amount. + dilation (int): Dilation rate of depthwise conv. Default: 1 + conv_cfg (dict): Config dict for convolution layer. + Default: None, which means using conv2d. + norm_cfg (dict): Config dict for normalization layer. + Default: dict(type='BN'). + act_cfg (dict): Config dict for activation layer. + Default: dict(type='ReLU6'). + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + + Returns: + Tensor: The output tensor. + """ + + def __init__(self, + in_channels, + out_channels, + stride, + expand_ratio, + dilation=1, + conv_cfg=None, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU6'), + with_cp=False): + super(InvertedResidual, self).__init__() + self.stride = stride + assert stride in [1, 2], f'stride must in [1, 2]. ' \ + f'But received {stride}.' + self.with_cp = with_cp + self.use_res_connect = self.stride == 1 and in_channels == out_channels + hidden_dim = int(round(in_channels * expand_ratio)) + + layers = [] + if expand_ratio != 1: + layers.append( + ConvModule( + in_channels=in_channels, + out_channels=hidden_dim, + kernel_size=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg)) + layers.extend([ + ConvModule( + in_channels=hidden_dim, + out_channels=hidden_dim, + kernel_size=3, + stride=stride, + padding=dilation, + dilation=dilation, + groups=hidden_dim, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg), + ConvModule( + in_channels=hidden_dim, + out_channels=out_channels, + kernel_size=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=None) + ]) + self.conv = nn.Sequential(*layers) + + def forward(self, x): + + def _inner_forward(x): + if self.use_res_connect: + return x + self.conv(x) + else: + return self.conv(x) + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x) + else: + out = _inner_forward(x) + + return out + + +class InvertedResidualV3(nn.Module): + """Inverted Residual Block for MobileNetV3. + + Args: + in_channels (int): The input channels of this Module. + out_channels (int): The output channels of this Module. + mid_channels (int): The input channels of the depthwise convolution. + kernel_size (int): The kernel size of the depthwise convolution. + Default: 3. + stride (int): The stride of the depthwise convolution. Default: 1. + se_cfg (dict): Config dict for se layer. Default: None, which means no + se layer. + with_expand_conv (bool): Use expand conv or not. If set False, + mid_channels must be the same with in_channels. Default: True. + conv_cfg (dict): Config dict for convolution layer. Default: None, + which means using conv2d. + norm_cfg (dict): Config dict for normalization layer. + Default: dict(type='BN'). + act_cfg (dict): Config dict for activation layer. + Default: dict(type='ReLU'). + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + + Returns: + Tensor: The output tensor. + """ + + def __init__(self, + in_channels, + out_channels, + mid_channels, + kernel_size=3, + stride=1, + se_cfg=None, + with_expand_conv=True, + conv_cfg=None, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU'), + with_cp=False): + super(InvertedResidualV3, self).__init__() + self.with_res_shortcut = (stride == 1 and in_channels == out_channels) + assert stride in [1, 2] + self.with_cp = with_cp + self.with_se = se_cfg is not None + self.with_expand_conv = with_expand_conv + + if self.with_se: + assert isinstance(se_cfg, dict) + if not self.with_expand_conv: + assert mid_channels == in_channels + + if self.with_expand_conv: + self.expand_conv = ConvModule( + in_channels=in_channels, + out_channels=mid_channels, + kernel_size=1, + stride=1, + padding=0, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + self.depthwise_conv = ConvModule( + in_channels=mid_channels, + out_channels=mid_channels, + kernel_size=kernel_size, + stride=stride, + padding=kernel_size // 2, + groups=mid_channels, + conv_cfg=dict( + type='Conv2dAdaptivePadding') if stride == 2 else conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + + if self.with_se: + self.se = SELayer(**se_cfg) + + self.linear_conv = ConvModule( + in_channels=mid_channels, + out_channels=out_channels, + kernel_size=1, + stride=1, + padding=0, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=None) + + def forward(self, x): + + def _inner_forward(x): + out = x + + if self.with_expand_conv: + out = self.expand_conv(out) + + out = self.depthwise_conv(out) + + if self.with_se: + out = self.se(out) + + out = self.linear_conv(out) + + if self.with_res_shortcut: + return x + out + else: + return out + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x) + else: + out = _inner_forward(x) + + return out diff --git a/segmentation/mmseg/models/utils/make_divisible.py b/segmentation/mmseg/models/utils/make_divisible.py new file mode 100644 index 0000000..75ad756 --- /dev/null +++ b/segmentation/mmseg/models/utils/make_divisible.py @@ -0,0 +1,27 @@ +def make_divisible(value, divisor, min_value=None, min_ratio=0.9): + """Make divisible function. + + This function rounds the channel number to the nearest value that can be + divisible by the divisor. It is taken from the original tf repo. It ensures + that all layers have a channel number that is divisible by divisor. It can + be seen here: https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet.py # noqa + + Args: + value (int): The original channel number. + divisor (int): The divisor to fully divide the channel number. + min_value (int): The minimum value of the output channel. + Default: None, means that the minimum value equal to the divisor. + min_ratio (float): The minimum ratio of the rounded channel number to + the original channel number. Default: 0.9. + + Returns: + int: The modified output channel number. + """ + + if min_value is None: + min_value = divisor + new_value = max(min_value, int(value + divisor / 2) // divisor * divisor) + # Make sure that round down does not go down by more than (1-min_ratio). + if new_value < min_ratio * value: + new_value += divisor + return new_value diff --git a/segmentation/mmseg/models/utils/res_layer.py b/segmentation/mmseg/models/utils/res_layer.py new file mode 100644 index 0000000..2585ab5 --- /dev/null +++ b/segmentation/mmseg/models/utils/res_layer.py @@ -0,0 +1,94 @@ +from mmcv.cnn import build_conv_layer, build_norm_layer +from torch import nn as nn + + +class ResLayer(nn.Sequential): + """ResLayer to build ResNet style backbone. + + Args: + block (nn.Module): block used to build ResLayer. + inplanes (int): inplanes of block. + planes (int): planes of block. + num_blocks (int): number of blocks. + stride (int): stride of the first block. Default: 1 + avg_down (bool): Use AvgPool instead of stride conv when + downsampling in the bottleneck. Default: False + conv_cfg (dict): dictionary to construct and config conv layer. + Default: None + norm_cfg (dict): dictionary to construct and config norm layer. + Default: dict(type='BN') + multi_grid (int | None): Multi grid dilation rates of last + stage. Default: None + contract_dilation (bool): Whether contract first dilation of each layer + Default: False + """ + + def __init__(self, + block, + inplanes, + planes, + num_blocks, + stride=1, + dilation=1, + avg_down=False, + conv_cfg=None, + norm_cfg=dict(type='BN'), + multi_grid=None, + contract_dilation=False, + **kwargs): + self.block = block + + downsample = None + if stride != 1 or inplanes != planes * block.expansion: + downsample = [] + conv_stride = stride + if avg_down: + conv_stride = 1 + downsample.append( + nn.AvgPool2d( + kernel_size=stride, + stride=stride, + ceil_mode=True, + count_include_pad=False)) + downsample.extend([ + build_conv_layer( + conv_cfg, + inplanes, + planes * block.expansion, + kernel_size=1, + stride=conv_stride, + bias=False), + build_norm_layer(norm_cfg, planes * block.expansion)[1] + ]) + downsample = nn.Sequential(*downsample) + + layers = [] + if multi_grid is None: + if dilation > 1 and contract_dilation: + first_dilation = dilation // 2 + else: + first_dilation = dilation + else: + first_dilation = multi_grid[0] + layers.append( + block( + inplanes=inplanes, + planes=planes, + stride=stride, + dilation=first_dilation, + downsample=downsample, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + **kwargs)) + inplanes = planes * block.expansion + for i in range(1, num_blocks): + layers.append( + block( + inplanes=inplanes, + planes=planes, + stride=1, + dilation=dilation if multi_grid is None else multi_grid[i], + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + **kwargs)) + super(ResLayer, self).__init__(*layers) diff --git a/segmentation/mmseg/models/utils/se_layer.py b/segmentation/mmseg/models/utils/se_layer.py new file mode 100644 index 0000000..e083404 --- /dev/null +++ b/segmentation/mmseg/models/utils/se_layer.py @@ -0,0 +1,57 @@ +import mmcv +import torch.nn as nn +from mmcv.cnn import ConvModule + +from .make_divisible import make_divisible + + +class SELayer(nn.Module): + """Squeeze-and-Excitation Module. + + Args: + channels (int): The input (and output) channels of the SE layer. + ratio (int): Squeeze ratio in SELayer, the intermediate channel will be + ``int(channels/ratio)``. Default: 16. + conv_cfg (None or dict): Config dict for convolution layer. + Default: None, which means using conv2d. + act_cfg (dict or Sequence[dict]): Config dict for activation layer. + If act_cfg is a dict, two activation layers will be configured + by this dict. If act_cfg is a sequence of dicts, the first + activation layer will be configured by the first dict and the + second activation layer will be configured by the second dict. + Default: (dict(type='ReLU'), dict(type='HSigmoid', bias=3.0, + divisor=6.0)). + """ + + def __init__(self, + channels, + ratio=16, + conv_cfg=None, + act_cfg=(dict(type='ReLU'), + dict(type='HSigmoid', bias=3.0, divisor=6.0))): + super(SELayer, self).__init__() + if isinstance(act_cfg, dict): + act_cfg = (act_cfg, act_cfg) + assert len(act_cfg) == 2 + assert mmcv.is_tuple_of(act_cfg, dict) + self.global_avgpool = nn.AdaptiveAvgPool2d(1) + self.conv1 = ConvModule( + in_channels=channels, + out_channels=make_divisible(channels // ratio, 8), + kernel_size=1, + stride=1, + conv_cfg=conv_cfg, + act_cfg=act_cfg[0]) + self.conv2 = ConvModule( + in_channels=make_divisible(channels // ratio, 8), + out_channels=channels, + kernel_size=1, + stride=1, + conv_cfg=conv_cfg, + act_cfg=act_cfg[1]) + + def forward(self, x): + out = self.global_avgpool(x) + out = self.conv1(out) + out = self.conv2(out) + return x * out diff --git a/segmentation/mmseg/models/utils/self_attention_block.py b/segmentation/mmseg/models/utils/self_attention_block.py new file mode 100644 index 0000000..372fad2 --- /dev/null +++ b/segmentation/mmseg/models/utils/self_attention_block.py @@ -0,0 +1,159 @@ +import torch +from mmcv.cnn import ConvModule, constant_init +from torch import nn as nn +from torch.nn import functional as F + + +class SelfAttentionBlock(nn.Module): + """General self-attention block/non-local block. + + Please refer to https://arxiv.org/abs/1706.03762 for details about key, + query and value. + + Args: + key_in_channels (int): Input channels of key feature. + query_in_channels (int): Input channels of query feature. + channels (int): Output channels of key/query transform. + out_channels (int): Output channels. + share_key_query (bool): Whether share projection weight between key + and query projection. + query_downsample (nn.Module): Query downsample module. + key_downsample (nn.Module): Key downsample module. + key_query_num_convs (int): Number of convs for key/query projection. + value_num_convs (int): Number of convs for value projection. + matmul_norm (bool): Whether normalize attention map with sqrt of + channels + with_out (bool): Whether use out projection. + conv_cfg (dict|None): Config of conv layers. + norm_cfg (dict|None): Config of norm layers. + act_cfg (dict|None): Config of activation layers. + """ + + def __init__(self, key_in_channels, query_in_channels, channels, + out_channels, share_key_query, query_downsample, + key_downsample, key_query_num_convs, value_out_num_convs, + key_query_norm, value_out_norm, matmul_norm, with_out, + conv_cfg, norm_cfg, act_cfg): + super(SelfAttentionBlock, self).__init__() + if share_key_query: + assert key_in_channels == query_in_channels + self.key_in_channels = key_in_channels + self.query_in_channels = query_in_channels + self.out_channels = out_channels + self.channels = channels + self.share_key_query = share_key_query + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + self.key_project = self.build_project( + key_in_channels, + channels, + num_convs=key_query_num_convs, + use_conv_module=key_query_norm, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + if share_key_query: + self.query_project = self.key_project + else: + self.query_project = self.build_project( + query_in_channels, + channels, + num_convs=key_query_num_convs, + use_conv_module=key_query_norm, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + self.value_project = self.build_project( + key_in_channels, + channels if with_out else out_channels, + num_convs=value_out_num_convs, + use_conv_module=value_out_norm, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + if with_out: + self.out_project = self.build_project( + channels, + out_channels, + num_convs=value_out_num_convs, + use_conv_module=value_out_norm, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + else: + self.out_project = None + + self.query_downsample = query_downsample + self.key_downsample = key_downsample + self.matmul_norm = matmul_norm + + self.init_weights() + + def init_weights(self): + """Initialize weight of later layer.""" + if self.out_project is not None: + if not isinstance(self.out_project, ConvModule): + constant_init(self.out_project, 0) + + def build_project(self, in_channels, channels, num_convs, use_conv_module, + conv_cfg, norm_cfg, act_cfg): + """Build projection layer for key/query/value/out.""" + if use_conv_module: + convs = [ + ConvModule( + in_channels, + channels, + 1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + ] + for _ in range(num_convs - 1): + convs.append( + ConvModule( + channels, + channels, + 1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg)) + else: + convs = [nn.Conv2d(in_channels, channels, 1)] + for _ in range(num_convs - 1): + convs.append(nn.Conv2d(channels, channels, 1)) + if len(convs) > 1: + convs = nn.Sequential(*convs) + else: + convs = convs[0] + return convs + + def forward(self, query_feats, key_feats): + """Forward function.""" + batch_size = query_feats.size(0) + query = self.query_project(query_feats) + if self.query_downsample is not None: + query = self.query_downsample(query) + query = query.reshape(*query.shape[:2], -1) + query = query.permute(0, 2, 1).contiguous() + + key = self.key_project(key_feats) + value = self.value_project(key_feats) + if self.key_downsample is not None: + key = self.key_downsample(key) + value = self.key_downsample(value) + key = key.reshape(*key.shape[:2], -1) + value = value.reshape(*value.shape[:2], -1) + value = value.permute(0, 2, 1).contiguous() + + sim_map = torch.matmul(query, key) + if self.matmul_norm: + sim_map = (self.channels**-.5) * sim_map + sim_map = F.softmax(sim_map, dim=-1) + + context = torch.matmul(sim_map, value) + context = context.permute(0, 2, 1).contiguous() + context = context.reshape(batch_size, -1, *query_feats.shape[2:]) + if self.out_project is not None: + context = self.out_project(context) + return context diff --git a/segmentation/mmseg/models/utils/up_conv_block.py b/segmentation/mmseg/models/utils/up_conv_block.py new file mode 100644 index 0000000..6566b74 --- /dev/null +++ b/segmentation/mmseg/models/utils/up_conv_block.py @@ -0,0 +1,101 @@ +import torch +import torch.nn as nn +from mmcv.cnn import ConvModule, build_upsample_layer + + +class UpConvBlock(nn.Module): + """Upsample convolution block in decoder for UNet. + + This upsample convolution block consists of one upsample module + followed by one convolution block. The upsample module expands the + high-level low-resolution feature map and the convolution block fuses + the upsampled high-level low-resolution feature map and the low-level + high-resolution feature map from encoder. + + Args: + conv_block (nn.Sequential): Sequential of convolutional layers. + in_channels (int): Number of input channels of the high-level + skip_channels (int): Number of input channels of the low-level + high-resolution feature map from encoder. + out_channels (int): Number of output channels. + num_convs (int): Number of convolutional layers in the conv_block. + Default: 2. + stride (int): Stride of convolutional layer in conv_block. Default: 1. + dilation (int): Dilation rate of convolutional layer in conv_block. + Default: 1. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + conv_cfg (dict | None): Config dict for convolution layer. + Default: None. + norm_cfg (dict | None): Config dict for normalization layer. + Default: dict(type='BN'). + act_cfg (dict | None): Config dict for activation layer in ConvModule. + Default: dict(type='ReLU'). + upsample_cfg (dict): The upsample config of the upsample module in + decoder. Default: dict(type='InterpConv'). If the size of + high-level feature map is the same as that of skip feature map + (low-level feature map from encoder), it does not need upsample the + high-level feature map and the upsample_cfg is None. + dcn (bool): Use deformable convolution in convolutional layer or not. + Default: None. + plugins (dict): plugins for convolutional layers. Default: None. + """ + + def __init__(self, + conv_block, + in_channels, + skip_channels, + out_channels, + num_convs=2, + stride=1, + dilation=1, + with_cp=False, + conv_cfg=None, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU'), + upsample_cfg=dict(type='InterpConv'), + dcn=None, + plugins=None): + super(UpConvBlock, self).__init__() + assert dcn is None, 'Not implemented yet.' + assert plugins is None, 'Not implemented yet.' + + self.conv_block = conv_block( + in_channels=2 * skip_channels, + out_channels=out_channels, + num_convs=num_convs, + stride=stride, + dilation=dilation, + with_cp=with_cp, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + dcn=None, + plugins=None) + if upsample_cfg is not None: + self.upsample = build_upsample_layer( + cfg=upsample_cfg, + in_channels=in_channels, + out_channels=skip_channels, + with_cp=with_cp, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + else: + self.upsample = ConvModule( + in_channels, + skip_channels, + kernel_size=1, + stride=1, + padding=0, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + + def forward(self, skip, x): + """Forward function.""" + + x = self.upsample(x) + out = torch.cat([skip, x], dim=1) + out = self.conv_block(out) + + return out diff --git a/segmentation/mmseg/models/utils/weight_init.py b/segmentation/mmseg/models/utils/weight_init.py new file mode 100644 index 0000000..38141ba --- /dev/null +++ b/segmentation/mmseg/models/utils/weight_init.py @@ -0,0 +1,62 @@ +"""Modified from https://github.com/rwightman/pytorch-image- +models/blob/master/timm/models/layers/drop.py.""" + +import math +import warnings + +import torch + + +def _no_grad_trunc_normal_(tensor, mean, std, a, b): + """Reference: https://people.sc.fsu.edu/~jburkardt/presentations + /truncated_normal.pdf""" + + def norm_cdf(x): + # Computes standard normal cumulative distribution function + return (1. + math.erf(x / math.sqrt(2.))) / 2. + + if (mean < a - 2 * std) or (mean > b + 2 * std): + warnings.warn( + 'mean is more than 2 std from [a, b] in nn.init.trunc_normal_. ' + 'The distribution of values may be incorrect.', + stacklevel=2) + + with torch.no_grad(): + # Values are generated by using a truncated uniform distribution and + # then using the inverse CDF for the normal distribution. + # Get upper and lower cdf values + lower_bound = norm_cdf((a - mean) / std) + upper_bound = norm_cdf((b - mean) / std) + + # Uniformly fill tensor with values from [l, u], then translate to + # [2l-1, 2u-1]. + tensor.uniform_(2 * lower_bound - 1, 2 * upper_bound - 1) + + # Use inverse cdf transform for normal distribution to get truncated + # standard normal + tensor.erfinv_() + + # Transform to proper mean, std + tensor.mul_(std * math.sqrt(2.)) + tensor.add_(mean) + + # Clamp to ensure it's in the proper range + tensor.clamp_(min=a, max=b) + return tensor + + +def trunc_normal_(tensor, mean=0., std=1., a=-2., b=2.): + r"""Fills the input Tensor with values drawn from a truncated + normal distribution. The values are effectively drawn from the + normal distribution :math:`\mathcal{N}(\text{mean}, \text{std}^2)` + with values outside :math:`[a, b]` redrawn until they are within + the bounds. The method used for generating the random values works + best when :math:`a \leq \text{mean} \leq b`. + Args: + tensor (``torch.Tensor``): an n-dimensional `torch.Tensor` + mean (float): the mean of the normal distribution + std (float): the standard deviation of the normal distribution + a (float): the minimum cutoff value + b (float): the maximum cutoff value + """ + return _no_grad_trunc_normal_(tensor, mean, std, a, b) diff --git a/segmentation/mmseg/ops/__init__.py b/segmentation/mmseg/ops/__init__.py new file mode 100644 index 0000000..bec51c7 --- /dev/null +++ b/segmentation/mmseg/ops/__init__.py @@ -0,0 +1,4 @@ +from .encoding import Encoding +from .wrappers import Upsample, resize + +__all__ = ['Upsample', 'resize', 'Encoding'] diff --git a/segmentation/mmseg/ops/encoding.py b/segmentation/mmseg/ops/encoding.py new file mode 100644 index 0000000..7eb3629 --- /dev/null +++ b/segmentation/mmseg/ops/encoding.py @@ -0,0 +1,74 @@ +import torch +from torch import nn +from torch.nn import functional as F + + +class Encoding(nn.Module): + """Encoding Layer: a learnable residual encoder. + + Input is of shape (batch_size, channels, height, width). + Output is of shape (batch_size, num_codes, channels). + + Args: + channels: dimension of the features or feature channels + num_codes: number of code words + """ + + def __init__(self, channels, num_codes): + super(Encoding, self).__init__() + # init codewords and smoothing factor + self.channels, self.num_codes = channels, num_codes + std = 1. / ((num_codes * channels)**0.5) + # [num_codes, channels] + self.codewords = nn.Parameter( + torch.empty(num_codes, channels, + dtype=torch.float).uniform_(-std, std), + requires_grad=True) + # [num_codes] + self.scale = nn.Parameter( + torch.empty(num_codes, dtype=torch.float).uniform_(-1, 0), + requires_grad=True) + + @staticmethod + def scaled_l2(x, codewords, scale): + num_codes, channels = codewords.size() + batch_size = x.size(0) + reshaped_scale = scale.view((1, 1, num_codes)) + expanded_x = x.unsqueeze(2).expand( + (batch_size, x.size(1), num_codes, channels)) + reshaped_codewords = codewords.view((1, 1, num_codes, channels)) + + scaled_l2_norm = reshaped_scale * ( + expanded_x - reshaped_codewords).pow(2).sum(dim=3) + return scaled_l2_norm + + @staticmethod + def aggregate(assignment_weights, x, codewords): + num_codes, channels = codewords.size() + reshaped_codewords = codewords.view((1, 1, num_codes, channels)) + batch_size = x.size(0) + + expanded_x = x.unsqueeze(2).expand( + (batch_size, x.size(1), num_codes, channels)) + encoded_feat = (assignment_weights.unsqueeze(3) * + (expanded_x - reshaped_codewords)).sum(dim=1) + return encoded_feat + + def forward(self, x): + assert x.dim() == 4 and x.size(1) == self.channels + # [batch_size, channels, height, width] + batch_size = x.size(0) + # [batch_size, height x width, channels] + x = x.view(batch_size, self.channels, -1).transpose(1, 2).contiguous() + # assignment_weights: [batch_size, channels, num_codes] + assignment_weights = F.softmax( + self.scaled_l2(x, self.codewords, self.scale), dim=2) + # aggregate + encoded_feat = self.aggregate(assignment_weights, x, self.codewords) + return encoded_feat + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(Nx{self.channels}xHxW =>Nx{self.num_codes}' \ + f'x{self.channels})' + return repr_str diff --git a/segmentation/mmseg/ops/wrappers.py b/segmentation/mmseg/ops/wrappers.py new file mode 100644 index 0000000..0ed9a0c --- /dev/null +++ b/segmentation/mmseg/ops/wrappers.py @@ -0,0 +1,50 @@ +import warnings + +import torch.nn as nn +import torch.nn.functional as F + + +def resize(input, + size=None, + scale_factor=None, + mode='nearest', + align_corners=None, + warning=True): + if warning: + if size is not None and align_corners: + input_h, input_w = tuple(int(x) for x in input.shape[2:]) + output_h, output_w = tuple(int(x) for x in size) + if output_h > input_h or output_w > output_h: + if ((output_h > 1 and output_w > 1 and input_h > 1 + and input_w > 1) and (output_h - 1) % (input_h - 1) + and (output_w - 1) % (input_w - 1)): + warnings.warn( + f'When align_corners={align_corners}, ' + 'the output would more aligned if ' + f'input size {(input_h, input_w)} is `x+1` and ' + f'out size {(output_h, output_w)} is `nx+1`') + return F.interpolate(input, size, scale_factor, mode, align_corners) + + +class Upsample(nn.Module): + + def __init__(self, + size=None, + scale_factor=None, + mode='nearest', + align_corners=None): + super(Upsample, self).__init__() + self.size = size + if isinstance(scale_factor, tuple): + self.scale_factor = tuple(float(factor) for factor in scale_factor) + else: + self.scale_factor = float(scale_factor) if scale_factor else None + self.mode = mode + self.align_corners = align_corners + + def forward(self, x): + if not self.size: + size = [int(t * self.scale_factor) for t in x.shape[-2:]] + else: + size = self.size + return resize(x, size, None, self.mode, self.align_corners) diff --git a/segmentation/mmseg/utils/__init__.py b/segmentation/mmseg/utils/__init__.py new file mode 100644 index 0000000..ac489e2 --- /dev/null +++ b/segmentation/mmseg/utils/__init__.py @@ -0,0 +1,4 @@ +from .collect_env import collect_env +from .logger import get_root_logger + +__all__ = ['get_root_logger', 'collect_env'] diff --git a/segmentation/mmseg/utils/collect_env.py b/segmentation/mmseg/utils/collect_env.py new file mode 100644 index 0000000..8293a05 --- /dev/null +++ b/segmentation/mmseg/utils/collect_env.py @@ -0,0 +1,17 @@ +from mmcv.utils import collect_env as collect_base_env +from mmcv.utils import get_git_hash + +import mmseg + + +def collect_env(): + """Collect the information of the running environments.""" + env_info = collect_base_env() + env_info['MMSegmentation'] = f'{mmseg.__version__}+{get_git_hash()[:7]}' + + return env_info + + +if __name__ == '__main__': + for name, val in collect_env().items(): + print('{}: {}'.format(name, val)) diff --git a/segmentation/mmseg/utils/logger.py b/segmentation/mmseg/utils/logger.py new file mode 100644 index 0000000..05d2f13 --- /dev/null +++ b/segmentation/mmseg/utils/logger.py @@ -0,0 +1,27 @@ +import logging + +from mmcv.utils import get_logger + + +def get_root_logger(log_file=None, log_level=logging.INFO): + """Get the root logger. + + The logger will be initialized if it has not been initialized. By default a + StreamHandler will be added. If `log_file` is specified, a FileHandler will + also be added. The name of the root logger is the top-level package name, + e.g., "mmseg". + + Args: + log_file (str | None): The log filename. If specified, a FileHandler + will be added to the root logger. + log_level (int): The root logger level. Note that only the process of + rank 0 is affected, while other processes will set the level to + "Error" and be silent most of the time. + + Returns: + logging.Logger: The root logger. + """ + + logger = get_logger(name='mmseg', log_file=log_file, log_level=log_level) + + return logger diff --git a/segmentation/mmseg/version.py b/segmentation/mmseg/version.py new file mode 100644 index 0000000..e090d9f --- /dev/null +++ b/segmentation/mmseg/version.py @@ -0,0 +1,18 @@ +# Copyright (c) Open-MMLab. All rights reserved. + +__version__ = '0.13.0' + + +def parse_version_info(version_str): + version_info = [] + for x in version_str.split('.'): + if x.isdigit(): + version_info.append(int(x)) + elif x.find('rc') != -1: + patch_version = x.split('rc') + version_info.append(int(patch_version[0])) + version_info.append(f'rc{patch_version[1]}') + return tuple(version_info) + + +version_info = parse_version_info(__version__) diff --git a/segmentation/requirements.txt b/segmentation/requirements.txt new file mode 100644 index 0000000..6da5ade --- /dev/null +++ b/segmentation/requirements.txt @@ -0,0 +1,3 @@ +-r requirements/optional.txt +-r requirements/runtime.txt +-r requirements/tests.txt diff --git a/segmentation/requirements/docs.txt b/segmentation/requirements/docs.txt new file mode 100644 index 0000000..89fbf86 --- /dev/null +++ b/segmentation/requirements/docs.txt @@ -0,0 +1,4 @@ +recommonmark +sphinx +sphinx_markdown_tables +sphinx_rtd_theme diff --git a/segmentation/requirements/optional.txt b/segmentation/requirements/optional.txt new file mode 100644 index 0000000..47fa593 --- /dev/null +++ b/segmentation/requirements/optional.txt @@ -0,0 +1 @@ +cityscapesscripts diff --git a/segmentation/requirements/readthedocs.txt b/segmentation/requirements/readthedocs.txt new file mode 100644 index 0000000..0542bfc --- /dev/null +++ b/segmentation/requirements/readthedocs.txt @@ -0,0 +1,3 @@ +mmcv +torch +torchvision diff --git a/segmentation/requirements/runtime.txt b/segmentation/requirements/runtime.txt new file mode 100644 index 0000000..47048d0 --- /dev/null +++ b/segmentation/requirements/runtime.txt @@ -0,0 +1,3 @@ +matplotlib +numpy +prettytable diff --git a/segmentation/requirements/tests.txt b/segmentation/requirements/tests.txt new file mode 100644 index 0000000..991fd71 --- /dev/null +++ b/segmentation/requirements/tests.txt @@ -0,0 +1,7 @@ +codecov +flake8 +interrogate +isort==4.3.21 +pytest +xdoctest>=0.10.0 +yapf diff --git a/segmentation/resources/mmseg-logo.png b/segmentation/resources/mmseg-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..009083a9e80599a1893591ff362da377c94a0975 GIT binary patch literal 44728 zcmaHT1y~%*(kSkMB|r$iI3c*RxI=JBaCd^cOYq<>!6CQ>cXxMphsE7_oU7;L-2d_I z%=ENWS9MpHbq~REGGcF#aFHM&Al^ub3oAfCK&QN>>k!~x?-9nE=C2P(I|VU8h|&@K z{nr!bn;g zg8DU$009e$3jzC@f_%L`LE`-<;-~X$(|CLvk?q2&^9no4`)eZuJj`WWU()lN!BLoE0 zzNwOmy^6FHNZ-nWPS?On&ydd9!uk*B5Zun7*QAA^y)Mw%!ramh7AUM z=$x48tZa?x896yQ=^2>lnV4u_YtY)cSla74(^}e*{u9W*;RqYr>D!uG+nZWh0{?)k zt7qk4&qG4;M@Rqp`xl-T*8l0q((YeWz0ybTtZPlrNXJ0`zZzMaSlL_InOOZFP5)E; ze*!RcHvKPsf9dv*l7BY)$FTepTkhAf1qs<2>e?F$zh1n5OdKr(`|D5X->kz;|685^ z()y3q(pCnhMlS#V)_14k^GM;wuU|HliUV~8wGIuV~a$Kqn zlZ$^%_6;H=*gz4Qkrt7$8xHydlONd>K>uUc8?Z!=2;D(5c2TQs?Zt~nC6`;Jo7>It z_4UfK+r4Nq`_T`pmb;^s5LqcJdY#=Y&A`~3tuZ`tPM(}8+Z-Z_**ns!&{c7E zuI3WcIu&vR1gLM`K;WMrf!?J2RX`}dr7PFSA-v!z!*a_et95z5#x*BzKi)8F*zJAv z1>1v|SUU}EQ78d^000IC^3M-7I7s3YTDuQAeW{J58KHJOK|6zyAOcT>zHJJHDy%c+ zb=d%ROFSc<+2~DkpzRlo`l7ymUP&*<0o<(*q-2?YMdu%H!-D(>#+8w^i_LDkoLfd( zC|P%E1{cLURIRt~JKWY^AU)xXcy^<*bBRX`E>UjToE>}lrD}ohP@oxsS-aQ|<|GMh zV__KV72X_w#`Ir!-{ZX^{1jHVymC_`n|FOElGxr=Owo+sS)mzviw@{>GuiaJnW(Ei zVlkQ%w+#%Zq-HqNSVNc7g95=4-uW#0&KV84KO3SpylvRu(BM?@yG zcxmFZ(B5911V{I^lEOUB+$bk@#jY;>?j5i=2EZ<9AK`0n33ay=z)g`cx-q5B>y7Zh z8*!A^dAE6)$;ndzj~Nt%@f%mb_%ysyuFCHBnW|2^dOUY|Qh){f+hH>jStJ@PuBOQF z0fMm-dTp)C3C&PO8svGO8&8t=i0^K5I`>18Km4lFXeZw;x~vmz{9Y;PXBA2RH0h}# z)Cp6&bCp9d66CZ=KhfW&HXr?UVp*ZS#Bk7iq*jA5_3qu)xH*7euJSDbt)K90aj;H&)E}73v##+v&|%n{)y9j|NVP35^(A7x|H+ zROQj{{zl#>1cdcsKbZ-ppZE>NqX?knc3ob7F#o~|{Acb#l_JSNYf<&5XF{6RVQv0W zH^x2+A~l%BDd%)|$6FXo8ca7{O@Q3Cy+Y=V#DI~2xLd?{P^Q>^Pm-p5_1`;#6lfFW zN5~5kRXaO5!4XltVFe}=K_qdZnFV$lzsWp*)lxewyqFStq)caFmQ~nR`{QxD% zA<0-kzgw!Q>}#@uLay9#;VE%ZIz;A*C&ZB*Zvxfei$;$tGq0~-(3|9Q>mE5Y$PYTZ z$71a!P46RjKsob0Z4}^6pJL7kZ%jdrc*fW^n2AAu`}e*GUwfKtz-Va&yW@9Xu;n-6 zKGWQ0e#k#2>cC-*Mck{MNfVTTd*DXKI6a8t1ST3QLb}QKHv?==+DKO(3YIkI@EUO7 zF)ccW_7r3LABl!lp4`-`@*fKt#;?g}|3;ec+gDb((wZSUN`oeGd?P-`nmFHta?mqX zG*P&U-(iZAkzN0=xR;|y>Ek_;qvb?_iYb`RipYaenPS|I28j-PmqQ05>+QZ0g5h$K z*;?*9ZvlO?lv zf$G2LnYr+cA({~lE*$cCFjc!bos;4mq zs@c{<9S;USQp(E`-fq3%SGI6p^ye+Ei1Y?}es5`7(^1lx_+8lm8S2o*&kMub!^Eqc zobnwT>=u&*N96kU*2aeOP1WO#zNyky(feW&X z(G#^yI_x@GPWNYje#>^ol%pY22Z@v=}qUG5Mw;qsh+ z%*Bzr2_QI73*H{+E!S|ttslW$;0-;gPcup;w?O`V4e-AJtvB#-tL$&Yaar#_19`l} zwjRk5A|r=ISM}@tT~cHk<24qbPjvWp&CN~-X7-FkTK{IxRKY<}_MlLsN^Vxr74n=7#uw)0? zhI0YUu}OcE06r9?47bJ{E;vVLY`lQIO&|z4IrJuY*8OH&ANWI7&$05yQU|_5v)!Ih zE6&6@|0$R1iQEy3#ctA;(TWJp&FBZ_jcJyzy*T#0*X#DJW~3C^G?afIPCfMZkIX5g zJtIJBcc|uF@+v`8Ju6>0qw5-?yz9r!#N{P;TQ=mAt&}jHR05M~BJ1I~V%?=9IE?wENQ*(*7{9;TfN>2yZT|}id6aj5o!yTEA=C+M@9(j$)dNTE*E-r^d z9vD|{s+tbaCOh*04~TtPop@&C2W^-|mzNWl)5>*OpVqgYkm7s9eS8tVQ8RM6#+=L5 zcI16~F|iKUV^fIBOb>{$U5I~Szm!M>6s?Ya7?RenaD8oY95j1=O?9-B`?{nJd$(uC zzalQw=2)<1_1|f>+Cwi_=CiK!_A{^hYtBYZlnvBFY}rsBa)*jkp^b-OD4XJYHK7ff zx_ODj^59hNOh!2o4$%7_-UX$@ML5-i@<>noV{IIDzIdQ*jE?0lr(XKSPre48hlv-*6vtd*gwpVp4DZh0#K z)Hj|qU5?0j+Mz4j7V-f((svDQeYKXQ=_=lORpwfvjo;%7X=rVc>^u9-Ke~;ab}}s#xW*^4!bo4v8IhM7OfbLr*Btn%itCyjw6Su%0`-v9~}`Yv~xc$HQQ12ZNYAV zkgeq=V1r}@cvl2I8D}jsGfwvF@c?sW^&LrzNzu7b=- z2D`37T%J4{(SrWG4{HD%$axlYRMKyPf)-XaGcx9o^Ew`0B}UJ*MR-b5ALKyT20z?eFI#KS62 znfovV3%5&31dsM1Qs!v^E9Y}K5{LUw`2ZDp_p?y=`1ql`a5 zkOsHZYv+zzn>p%NsMH4G6PkFuV(cS_f@!}m6&-#OIm?#???Wk_s>NYRtUNzNl1dS0 z5AR4KG-?%7jBUA(3^*ymOVdQbiCbR~zht);BnGw12`ZwR9-c-TMTNXi6IkV)4Qoo~oA zo6l^iWA|#^s%Fh5%rToVtEA)ba80^r(~0raBrDFg}D?3>H0!a1_O^2VrlsmqG;h3Ni|rLv#v&M;XF~3AHdoSJ?zkH$iU?G zA&T8~PD#!~f`!ne@se@L6)RKrR@j<#&D2Yz+h?Bs+EvhpH@_)5Dg+eXj->tf{!i9+ zZPYT<^+Cy(-3)xvNFGKe-yJ5%+@&2Sv|RB3a+4%59u8WlZ+{81O;8{Q9^iQRQqgSH z#{BSr<4F#j^-dAvX`V#)s@NesFS2rd@y2Om!$(ag{X6z#J{Rs`1yv_uW z#-0t>M_Gyk>g%)2(^w)<15&Pnu@aV4w#_73ejmo@Kk$^`*<=u6f$i!T z_Y*v+J^Z@OAdY~6?$|zrJ@-BR=LM%FLMUB1vM&_E{GIqczBsu2#?v-HYbThYZ+EE0 z+%O69u%XdO!Dvo@@8aY7ndQJr(w?-1-$hJoqxZR&Goul$HY`BfXI|l?0XD&h(Lf;> zqDcuuJ|T}MDla6_n*;|u#T@&2Q9tCKlzDYo^hk9T)S2;+t!paun`tjj-DSiv z?6qk@xo`hQe-c|RW0dpNn_L3MXYRYgOX1 zQs3Iwc3xm%g9AX?iJ8{kx=tP=wPvybE!pHd%kQy*Dc`m=k`&6(9{CuJy{YAOIKc^&P zAY)NWxYLHW=4B{;xEf)Y@(TC$oeAsRLS$YkSv&7k_lH}US0GL_{l=-0P*A=h)-}2> z6r6VJA@FefK8>T-_5kK+Gy}l8GcYhnd%APs(;nzv_KZo1BvXx=TDw>+G&77!mG&KK zxJo>#z67m|UN^6daw=r#U2}|X$6aY-z_-{I^w3kyNOcH4(HPMl4q&2D?!-t)upNOk zul~U8H{&aZqX2?Y`NmJ4dX{R+@XLP-^Ps@Nz+tbGf#Q%@$n6m#w3rYT+DFhxXeLM! z(-PUi?+e$kRRr32e1pnXt-1zgs=w868e#_^b%^}Lau{iQVZH}qU6*#yY@>PycqBRV zAdPznv|8SdPXpu1U?trVOS@@k(vMCDynpz}Xe-NDr=3&-dOdn_FQQE2M{U`Zb^ET^ zB!q{~)q30A*i%2p9yt^Kq+KJq5kXcVqxd~CUQNPKhmaQn?JFL35k$_6%t4UY-k81{ z=l61;D$U91&T=P=HZFUi(Za@kYhE(5sk zO_wdxJ&XDc`2J)hk8O7w=u-wG!Hu>(*SK$V8C;Rf0`)`3F=jI#>yy}EHH1MM3-m-V zH}htaq08Rd{Alsyo1{FG;Y8ns_}u3+L_0-`$e!OF!F%-l2y0K%<@dOr?r?C2T>GTE z$AzmYAQl=;R=;aKzKkq+LqwvY*{yKZfNc_q2!EAra(N{kC=3osfy0Ug{;`PLbNo3o z4&8$9Gq_6o=DN&RH5T26+dubau`U<9-06eXi}J@D$vORPoRuk$r2+IkVi)v>(_2ZB zo!yrxl<7=^On?bf(Z26?Y4llF3^lLJ>M-OZ7!XuA#M|^@OS(ODNd-2`;*vl1r4Xc% z{;Q+SLg=&QcY!B6NY=fjkRSdmu z;WICt*m)iNdu$sH5~>R@I$nQs9~^#vZ}K zrnS3LOobe@Hp~_3i##F4TeFxU!jA)?m~^+$e^p3Z0&G>MN;9mz#hi(D2U{9?`Z>T@ z^&Ibm9(9~9}pBBVj_E~$mQS?a3-A^o6u7DhDQ6ueJnDP1dZTi8q z*R~+(R zgRuj)o`iA6t-A#x2hisCsnUm^>;9j9GCcuccaO&T*Q=Fbrw(5bLpKO{c~y zfApbkaiSG+XuFX)?@WzT!V|RJFXWld#?Bzq; z1M)jX6iSa=2+M>i(qJ9=V&Y<$>q^@yMmvTf3C6xJ6-475wwn-#8s5JrHd3x8}IH8 zj|prEo65CzmJi|JxiFhwAu$TZ`78S0_)8FC-;R{gr1N*mHXwdqf!ttzvkoYkiX;;X zIE-hCwE30eaM{_^bX})8MQ_ua`sUp?C+oMPHLfy?`*?!_-Dg}efKnp|I-t)NtrXOv zImNdFr&r0}Jyn4lyLje2gCSn?aEjH5T={z8wl%)QrJqYH+tfjWIWb&EH_HH#A7o5W z6suj*YMa}$hAEjcTq|V-lNv_cw)n>M2nV7A5tg(@L`$SCPj7!$MbH4qw2#yc^gNEV z1vOmdU8kLi4#{EG;?9Vq!oEHXV(uq;%h}XYWUTJg+e#|N@@~-j9I0RGV|39LY7VJ! z!N%F4#^f%oz`z=`b15Qu!fC95b8teR@Kr^!xRy>r8H3ufvD*Bp(XHxM}8i9L>8$XLj zga^2c-|}dJzAo&wG_oLzh9WK+&f}2yw=h!C8L3xfw%D|gV}H7Zb*wMIi(|gCsnB@W z_fV5mMXW*5$R|XRJn1BpZQ59x^}0hgU7lOXqy9RbtYD|M7kuBiV=g8f!e{++&MhV` z0_@o<)sa>&->T2EpIxLmnCFd6UQ)Cy3-*umt$g;4C*Qn<{yiGGh2ZRWagmD5$n-)5 zx0)anSL-)3gTtPIc3uTUpp^CxuPo21@j~34CBzGPa&;z0a_EYVq*B#wWDN-ppjad! z6Cfh30Iep_j~iYlGAOVcL_{&_=2#f84>!MOVemB^z}dx`T2Xhsw^9pOekc$MFPah5 zEDM+b2uV}Be0iG)L?|u!P+KN#o)VmUas)D>h#l;sd@zI527qoBMj5xlESZJN$Yzxd{z)ec=|WI>n^c(SCShHyt47m(uC68h!YKkJ zdwa<_I&U%yiEd0U6oal+j4Tf{QvDh+#O(g$ji3?qqpM{_HNH6-zX|PN7w&JHXBqEc znBH5b`?vFI8E;?^poXGt6Hn66zJYlJX@xWuJF;-jhnVc;_Y4QjH{->0gH@*X%fIH- z&>AG{`ytkAe<6~pK~%N+^@tYa8)O3435!135s&;G{lmfOX5?m5Br2EbG@=sj9Wa|S zs)oiS-B)>-16xtu4H|JbYL}qb1u6BAiD265J7W9@$cGL?=P>YNDNA$$5Usk{I+PoaP`#-rSy@(!v*%G_eKq!K+d@#%sN*WEM;@ zjH69A5Z<6yUkvmLST0Lf-EL?)V^dPsMlvTfR~MTRdv{^}Wz>CqzCq9zY$r)P@v%7X z`hd{ZhKtejuKHHdq$)Tv;AO3on$l#Bp7=RhtKXd<3=avp%`qR~Vayqr3kMuS3+m@Z zg#|R=E2)kfUFs8ToeiLbST86~?M=+h9eTgmG0z3rF&TA_eS+>yqRM3 zmPf$Aov|lphHW z8atwNf$uMLID#l~!!L5_$vt#jzRHa{R2PS+(qCcLZP;j=e2c2ty52Vy;^=&(8_D#U;+?yCiv6>A6?aHWJ=}vJ-rlRFM?VF-o|DP2s_kz&-|c~fE|BBPO|N#yHlTs< zktWY!8EfmBJA&7Rxm6@Hh<65kR#bymf2+3aN+3Wp5^GwYL4-F;+Cio6u-1#sup*2! zWjQdM6K5Auqo1&`)ioER3;pC^>+6&nN9MI5Xp+vQn-?%N(jlDF|u~18FrhS$6jzr^5Tja7kQK1RCjf7N^ZI|y7YoImX zfohgB@8hMsMB}s+25Hm8R9UNarRmj|G_9FrA)-4?eC4S?k@OTPt}Z3Ssi9#Wa*^%S znBSZa$m`gyQvxx9;?$&FjE#Rqefe&R*ZfcwH}qEhcfE>>-j>EqxTME~rYM<)9B~Eh zkvnJLw<4%FZIch}`)Eh?!?2_OMAEsFCcu^B#lFt^>iP`1&8b#DnoF2!NV z^VQ`(xA1^nEQi`!eJ`R4ch&d^kmBkWB@)6sXu)Ex$SW)yH($qZaNklWM6H3C!!r*Y z*q~>oMV1nQ-3#qSAI(bjF>unDhI<+HW*4*P?a6uoN!qXhBDPhJZX${j>iLz!CO7RG zRx)>!66Q^}t}W76)8&;Xs^Zb*dVOxob{iIOtiu#QyCL2Y={Kg8>JD6; zsh``xf_rEu&~O3VodwFEi|S#B_dOBHyv#``6TqaD)B`iU;bHlSYCcAscQ-a2xA`ZIO+pRK&UliN`{}MuJlURv)<-jXX_5 zRr82XuF{CEOdA}w7x&{s2qU6miUq>GZ$OyCW(*u?zwih%E_7DJyc3wrc8&Lv$VM2T zvp2Bhg1Z1ym17e=mBL^-R=6B;o&$PyD1Ev-EnM;MzBgxgobT+E2# zx{I;=gpjzop*(_MBJaL)tVvWyBKrx}yYgkVk3`;O@wAxdJe3DL%YJZpS6zG7GXFY& z?gN?MDy^pUz1QQ$yxzLs?sODWbQa04CrNui5)a>=ovS%&2*t>St4~X(2=&fO&r3bQ z!)>=2r7l;FPtY`tk;9%NNgv+wj%}&rB|q%WjM(|?ikZB$v$scXK&L4b1rUeu#EcDI z`nUa3j|IB}W_RIp0$ze_jRUWD;|nA4|Ep8pGazh%M*T`|*4fj&9v*Dx;C6Y1U)@uA zOWuPZt|$Q2@%q-8D+6>F=?ZTOA!a2CN#;DMCK_dLL*8Tp(fjGZc!O?Bu2Ij_lj<-P z9Vc=Jb4rk@Ics2rbw08)pZ1Qfn)_#No3Y458TCx8>5{LUgQ%%Hd<_IeZQmC_hjm7T z@fwras7BTNAiV97T=#Yxuo2eB-9vhkenJMj%)Zva+c21*K4}_+Ygq@J1Bp!&Uc2s zeB!RVC#2wKUlZFQjaB#WZ26N$YV#IdlT2-LIG#P7j+CC9UVpFn2)}$irx8QE9_+&( zWqm%!_IjE>@mDFTGzk=vap1uo%7_79glmN&3G0M}(H-yJ~;N9sF^UpEU?9f`SB ziS$+nF2p-}OC?i|mpDV79}d>9lF)k{Bo`PO9>kJh!)CCu2!H3tXjWKj}+tA zJ+=V0u4IjopB>}Vi5W0=5dfJa1HG zC;X?tx*EWTF3$HeGlmM=cceQj%6!Vta12m0P;wNqdlGA$Z>->Mk`rxQfNO|$6C9d; zd}?rAU7z0@c%$tWwzVa6qhwyH-Ki)Vf{mg=-%#) zO%E3{*YHhlizshyfce5KEiPp>ykbt#KG$sAEYYf&X{iEtBPaZz^WDbJ-Ajx08(N+c zkFNs`sCnccljnH=5>m2l9z zcocAs0ouIj1?>;X&}QnnK_;`NWmMc#H3pg}T?LSS7aW*M6uSh8u=N zZ8NM`_?8q2id<)C&sJUuOjKbDAKHE#*%oC(C^?U`@ZtXjIg;geEBlorFhK^hX?`0{ z6{e;Yx9}!baddKY#f{S@>hh4BpUGg78~DqZ)t6)I{8)=-X<13eDeR6}fQ@l*Kc|*` zd`!pg%a;q8+z5xYZ3Q zDG;I0TWd&@y)m~Ks!(H*lbfI3;~|8Sdpw)RZA`qeWs1M|JQZs0#jpvpv0lEBYiXuQ z68SOBnP&RBlZ_zjePh&9R5V|UiD6);eMzg?3Nvc%I8HydhjsO_NTRc7-{gd75BX#7 ztdgYPe`DAg!3YLM)5~>Pb_X-WNN^}nubV7VY|xrqEM)zDu5A%WpiB50qYUlhU7b@t z-#r1|x12l)C#fi37}yJtoIB+!xkmVPX~H>fh7{>w|EA))s<&PY_D%pPj1Sr+A+yePJ|>XuRm)E z^Bj8!f)wjQvjqQbYwn@J#BQ)7PVJ{)+E1B&Q#xUXaqkGe&Jym+na9Ln&yK$$KhLJ7!Kd#iI_t3;Y$@hf7KcOPN z!|>srt#0}?9Sg=a$4;@*>ifbqiGAUxzE$=bHwMjD?s-nn079(QXs=Bzp-=*;4~)45 zMw*^d@Zon3S4)MijNA*-@{b>-RGox+J(?SSXxm7Ic{+(crh~9 zVfOW6xo!9K>=nVd_RRZ_iKAhfoVrVg5TlMp?KH3 z+Twd_h$w` zXy~KAF+cfMn%6&ERyrU!sVxnc*mm1M{n-d7z7~Is!`Wz6^k$6)3b7?PD$cv8pQh=n!aw$tyg0<2IeECBNaSgoxH)ir3B(>pZ&a~%LKF>mi%Ga?9dI> zq%3BOd6$M*W_PQs%|pAhQTe7UdGMr{lx4;Uy|B0KV!|H%mp9$t zK)gLYGN`1|JqCVCtRH+lQVvK498;F^uP<1+_>UY&}t!VA#a+3+xbFuXJ zXfIjpGk^cQT`pf5=|ZV=;b1OuBQU_jmW}lWX4ii zU_{`lQfjMsA?7odI$rLPg64Ht7b6cI?;LpGcmvZWJ!)(A7^(UlY^NeK_}~Dxdj!Qw z8G&MNAA%U;zUP7!aZ8qoW>;L)tES&c5J4r+Djau&(dtDc{>kx`hjFgREjg<0)54=I zjO>AlxG);f)OaBrLkbg@2NOed1h<#)aL`vr!B6o-E<&Jo<;0{*@eiPUPtqYawWChm zalcN4RZMb;RiBYBYzRm>?qHyaNrB`WP^a4mqv6hmiK3icfk__6jo`1sb#)$CwwV`; zDmwNM%D*v%+dX)$iI6NDy1FJ&4&oh>R0zCfQhSHv@?|XAlH-&su;z&G4M&btUDNZ% zG}U5;vsHDoF)6-B*jcZRg%MAjT9`4RdeQXfFUMaWpXG08_J}s69~0utm!WuRf=nz4 z;En~CyJKzW1^Otj!r>8HAtr~2gxs;*6#~DOl;(CLAIlcP&^ahl7S1b?EQheLhB+CfF(S(noC0)b*lm|7-1$EVE_eL>DeCyt*QeJe-77WTz6B=%3>KFS z>Qnl(v5>Dkpg8&oAd&QMoi(P1;C?^hQNHI8tA0mfuIb9<`J(-ug>P;rn8MRzAp6$i z#VKo@1M}3pI1$r{gao2vA8o+*%oy{&@@>jqVG3+rQf?qzu~8F2$-(w!8FO3% zCTeo#uEo#l?q!@YyHEKt3{(+p<`OnhHNL~;Td|!duI}Ns$QSYUj05F!=M7z8Ih*2R z@I@t0v`)4B)J}IOeqPlPmVv71;9?v{r6K+?0V}-0SES+WkpPuA`a?)%(oW_dq-`>S1VW5r2(bK5ftNyW-C z$+M4|(Hw`<@4pkp-EDg@Fwnl~e)*)Wa_82xvt=CATK+v?Mwfm6<6b4UKCJ(rjsd>6 zM%bdehc-7l;naHC2}6_$_oHNhS|OCFlwv`GK_hVS!d+$IHry)weEQHd4=R&G3oXf~ zLyk|*;s-nS%KzZ{SlDuKDMo}V3LA;{^S1n*wsF3^>}Qt2c-kZ#z`gw*d|qbH1;}0-l=Wg%h%oZ! zQw22(%DD`M#x_BAjFl7B)Mz9H*ltO28wDmWibG`OOqmi9pQGYd;t}l+az&O`%AfgC zA1_6m3po!B>kwM^vg7i~;kUXk=V1le9w0t_|F`|=1L^A>ISgr6aNjw`u#OT01J~e~ zvb%sE*X`NC8gz^nH*b3lp%Kv?Mu=a&QCLZJ6JkK`K;-N3L)bV&bA?`|>!-#dF&w;& zKJuuri-R>F?12U=1zN;#Tm=Z8)#VxIE}3JILlK(yBw!<<bQ{R z-N#t))i+%dUdHS-9^>{73&~O!dgsH}A4z_}sOB@wse~<~z(8XOu$>f>X0p$Nj=`Q* zetOTlBhZ1^u!lU21mn)>?gt+{Eq7a0X^UoiquU?eJn`#2Aezcd|1LNFKY7M~eD;a* z35H3l3?-A_ANKBPE?8>DO=}Td9R_Ij?)(n!js*VNEr`btCDQi=SjV-$Ysc5Pn)QRLR#6KAv*VqB(`>&r){!5s;1ve zxi)C)xHniuS2Z-XTin}=@% z32~{QBq-=DtP=9;N8$<;%?Ngh{hS?Dn99w88qcWbILVQnFj=!Nu;}QMx{v&^8d1_d zJ$rHBaFu166D8&KnC|7mtmv&vB24G{r+5glCr1lTQr!?UswmxEe~jEOFMiD?6q|M! z(%iJ2m#l3<2-tb4AlO61luJNPy=X(}nB!PKCL8|;B93l&+0Q45b!We-U5v2|&#%z& zo>APt9tmrx1I(QgpPTC3+e6D%igzS8o+{&bPYmB2wxEbq58GChF3pQk)jnRCTrlRG z5_lWdWV@|O5jAdbVAsYD^00)P^o8hP+?7KZ0jKIa{0Fn-4kd*9Lqc>Dv5Rrxfq-osCX4erwW8FwZ~} z4^Z=_o=-a0eO+zI>%xYX&imZ-)V=vuYx^P6tLSVwP_cpFp7dIASu#EKWzTy~*@49C zoz3__!%}wL2+m5WHD@U&QD5KeQnA|lN$&FLt^4@{`q>@+^3y~7W2wK7v{SvzvsUZd zXKLr?5q)mT4(2p&;?7lMkx_1IVC%&FYQu9l#`U8JCpIm`YpND;Ws zvJ{%I6?PaH15yfZCtxpSN<7W1p(;<}GZ~|H^++4#GSzTWmlDsM1(tV`%=gK?xw$Us-nvHXi|*yVnRud*Kh0G`U#W32Lj^?%xV*z; z`<6zr#7ih8Z|mHN&b;a&XA=~`#Xxw|qk%iY19NuC7kc-Io|pP1ySij-)c0i}1-x;^ zK4R8zxrR<%TV9D9cC$wK#UF9+W4xncU<>voD=hE$SM{)#76bF9hv<^(rG9fH&Cx<| z(T&h0+UJnPZ}lOwils4NZApw&r}1IsoEPg3%d&W9%WX1lVdsyf;6=G3LFbp4WB8v9 zw_3)d&~8gHk2nD|XwqqEu&=uYUzac>&lC=|-?ruz$1M}29#wU^@Z6uua8I~!cGMv4 zoF}wPq&bZU-NKcfWM8lObSMlMv`W@$bU{}*Cx9P8NYC2ty7NoYw^8ZxHB}k{r56rV z_b&At=XYkrGM}{Gsh7fk%>E^uvVvT3x1o}xN5Q9IIEqHQ0&{RA zz^It94J_Y^QjUYbiyG%1;S;zE>Do(p+_UkzRKR$*(V*m=bcGoYWVia)3Ah( zq{uLGmd2P!2ji$!0^8c-d_k#_D5e-!!eB zheX-ViQ|Gj<1@R?*}#)HmQ)X&&&jS~biLa|K`=7SuzVF|GIHZZ(dwt8!WXLo?DQsY zDw5PaQ9bz?Oss3ym^`sWwP3VQc(~t!-mSNH$(p{jr;fe2q|6P%OIOG1XC;9j?G`du zOG4YJG@-SF&A(I-J=yUF8Qyq(8_}L9`M9I}meAVp^&?;Yu@zs52-e%I7xc#vjO+Ec zOU8WOX;`=TXJ$SN_={2CM+Ue?Je^L0{KJ^xg(-eqH0#t+T_}4}E#1o(6VtHPepYs6 z5r2%kN)^PkMH?Nx(hD|^5^)d0tUTqn^ol)byeoljJF4zP%L3Z&--vKiox^)6>$jbC zs`RitJH0|qXec7$`i#g!`T3`ZZ;w>M!553d>ARm6einPev^=&^acex}JhyDE>ML0J zDKLj53G*F*W(7tgP65X7$3I(tjylOmW##WXBwsea7Zp54Sy^CixjB@RUB88Z*nnzKZJ)pk9g0^S%cSXJ^ z-?A4kDJ+T}1*!An+l)>hq|gB$X{rsW`84uHkLo_CMh?%{9SeOb^*9K{fWsLo`c?7; z=LR8OZJ#`|dd9EmUiq<&iBWRLwyCH7{{eJBi@(W^eDhmv&pn#< zeNdf=C;Dfx5=yz{*I;1#G(@4>dLbCKDvpzd!ELFhK1Et86;A<&ND$i9zy@` zL66Q%xmT~i=lNlLp6@(i$$ZLy)}@YTu=xSu@u8R;Q-2FxX?DEr{e;9yn$?}`=Bk?ROU(}D*^OMm}qrGSFL-?wGY&YB#&tx(2 zrObmUI44|~Y6=7%GoD?98PDaHq^h|;mE}6$i@p9#@)|K?%Y*+f(~_0n#&1XE-^1LB zn{dq@g&hsLX6g#cXFB_RX8J__ELOUYz1fF<#QpIrh|Q=Zny?I~%G160Z2m>U`U9() z*H8W&U)ET|of|co&vjne*mgvFSp;tDoK4jEfd7E-)UtBv74UC6W>E{gdf+YTscnF! z7WiJ_(`V5$e_(5(L5PT7q{rA!-kS+<@qAq2@CYr(GM4u;LmWyxOi>*HGFwf&FBf$^YxgrW`w<1`DjR{UOo&^blNvu$FT%asgtn>ZXQAlrj=uuo81_wgF)m5TCGHdWw##M5vOVp%!*@AVy_A1|%d;yKQb z>M-yWe5CFhP^9xKkB`6%<4^EI^iSwFFEML|n}+s(4LiQn@5>l@w(;p%zr&9s6W{)F zGtk}xm|6H>e>Ip&JKoYeelq@D_@qnjsx$@5@|LXpW?5nVfIu&wZ7(a6P5~aMsryLJ zq^6^=td!jq3E6$u5Rhqyd5h8VH zLfdQXv_h^&>JW(0dRn#ls8v!vYWfp|exG!|v_94u<0C2q5kxdLTqVei)aSYuJlbx~ zYmzhj3C%S$xHIA6xF%kh)&rH*HKCyBFy7<&BrRLq$MWeu zw2tm*$7P;@{?0~!K{(NOo*(D_>xU6(UnJIN?vFlfOIkcp{Tx*jEK?2|WYYNn-Mah* zK~Aeia*s@*qrDHqytlQ8iJt=b`=5UMVah#v8fF{+4}E`06Z((Oj{18H{BB3v))!k7 zEUBhR$8j)NEMl4CG9`ec{-z@jLqNCUgT2sPUb*WX0kpw0+_=sUHxo_%g0FKSUqd zTmY~|r;BiEJC<;_jBB0mm5d(PY7&6~ZZy<@`S~I6oNY*MfV|Rm>@prFYsV3;>jj=} zJS|r4EaX&oIK{{BKLsQE8K{eC>p+ng)omEv9CBN6-j~t!aYViAZ0>V8Egdo1y-<(; zS+}n%2u*udeyNaE=vUNsa<-{&hgLqKUZXnFoIl!R-##YGhJcpZei~(+(@CG8n6jz4 zoSS!hH=y@w@-hIyKc-U8x)tyuLmqQoAAoJlubkzg zvaHkOn-I$pNzJ5#-BWOCDB7;%cwbA`OpN9EpcX90jo4^c4cdvwm^#J~S3QDPqHMLR zmXF)BOhzxHyel$L{IR^CE>bW>j=)Q){ zbmmr#q~5mwh845>w4{FoaIdtyYu1cCzFXYC*XZa2UDV$-5vglHKwg(>`zQKmF;y^A zS2hG1zn8k!W+?RbL;n~}FKkBhF>N+$`$mi@gDgu=ZXEk5e&hM{*;aEnR48-r33oVG z_$riquM|li6P#d_>&JC~Kf27)H1grScpv8`6kssV<)*4HYvhA`Q&;iQL=4<<2++Pu z`reb0p&1y=!sDHHwk}P^Oa@9h8t1eK{+PZg^(Si2?1~dD%>$9L8#R3j1bw}_4PTVY zOq+q6i5Pi5G}*TCrf#TzU>|sX(>#avH37mOp54@B(+ZhvY5^Cot#L0zcN+7AHs5vB zQJ9^s?sQc~az8%sPtw>nHZZy4AF!U`6G5gY2g{*@E<-J2qA^I`L`hB7ukVGDl5{gZ zcExcV#`NJ7l;U&k+>M#e)_KavWQ$xpv0@hiLKq*j6R8 z6%kmdmp3Zn0dw)UxjB|D!SAM%O4IULOa;aUD|rEyy}z=-Z8To}tt)RqBQmSlc%+-# zg8ptgei!(`b(Ka8!SN&Q|J`d;y0ivlA`2o)Q~qh59hHQDe|n4RPy z@it`pubZ$($1KarX&Y)iRl!lnQESGn(F7G*9TZMjXDy4m{Q+SGw z@8>56$=#A=9pv_$fmOMDa25K<{boFXgTX}KB*~j7tqJ|18>mzwwVcKCe}RwJkJ8bH zQ#W%fzSZvs;zW(tKSo_YLH32wq83fR5UV-Tozc=Ko9g$>wP=^ctXKq;x)b@BnQyu~ z59GK0*4S*hE+mZUOhVF}R zc@T=#QB+?nW7(R_nX~7RU4^9{Pazr@^@RZ8yoml# zxmCvMqq5ozqt`CC;!%A}=$h&Bd5=^wu`)!fz+}rxbDaCiwkmf#mY&WaKGn4WE>Bs0 zXz)Z8s$s3#ONGocI54O_!r6h7ca@q8f0{RwiT?75K`wj7p$cn~h?x5)N!^<3^!^2^ z)UrReLJK$m<>{GAiI3oJmb-NY_TZ`4G^_b!?ef7fDZMg99}Ta*mdp;RB?trvEDB;t zBPaT2F)41%sdp89ZsqjmG+k$_xkq^E>fl%wTqhKRzWS0CJgfPc0n|@CKy?1C=?~(Y z2UzgE5+R&S8sdDMr7o5%s*RAx*xs%KStLVwhDdpGAe}gQwws2>Y1MhWzjFu%_>O)h zN;ag+Zp-L3igdf&HhbG9)eCZf{hMw6jI629S7<4;qJeI1DBO;h4(zhaOKkFGEgQ<} zvSj)#>aXL~vw2Lcd~f<}@(tnCw!2uft>#@M`JT!~@I*UEZ?Kp^s&cfrfmtaXUmJvI@ zK^jkI>U7HaKII;&e@S|QU~`4<*E-FV>L&~WFR!bo@>#44z9v-0Y4h#?dr)UkYWpH$ zQ%y?3L5jc^{)lmyUIjA1I_SIz58^*Qz>H4I+~9i!$2YT1{U=^IPKgf&*}NduxL#); zWt?>=f6eJNt=+hD8;7`pG$4o$w=+stW@3e!}dF)3#+j{nk}j{ zL(UHtHl@;bZ_fyvzB?05dUwCMFZdcUmu2E@U9P&@>;imG^q1qAJ*S@8RVI!<#0B^|jcAyQ zEfEgodok;e{rE~<4w+ZOpPS6+0aFW~PW2Of-hYSRzCTeRyANMOLnzZdo!rce(TjJ( zpVd#jc@XU^g&hxKwFRz8dl$7iBPgFg8~NA5zY`t@SJdTBCw^z#$8Q?Ht*bzAvl91Uzmd5 zNIcX8fgpMF@OXRE+KL zUCQ6sI%be2LK*{oQ`fSAvve|ugzYP)TvXpi znas0Q@82|at>*m3Oi4HlnHOVEOqNy#84+%`;VJNRehY~Hx@SA!`C0}haa>YJ|B*O>RP&ascm=Li zJqKSb!*c1HDR=4Wn(fLh;N|*Xpf5i0ZR+m%Tg2a6vktGq2XOe(H$iss-I~O?i<8uf zsq+8^w>i4y!u#<1fEzv59Y8J*&VB*^X^`%V@WJ>r?w>ctN)Oyj?*HFoP<#vJE_^5nv;-s+gMYjiF1d5)c<(u= z{>PeC$Y-%;$SPF$9?Je2-wFI36Lp;TB>Z>q!0=l9HnVIU?@`4`^L;t*9E`45v0P_Q zY<}fTx6J3Na;;+NZHRgggo}9zV$Qwz{$EVFOCG9FnMnyggY+->ynhg%_t)a{{#m5^ z_q>4Cy9N~?OQ|Tg;wAWP`Lh(X-Gi~)7wMLZ`8t+!{GGl6{%*RXvz>PEZ<>=s`V9D_ zt^?yCeo%8=QUxj3>-X;!3@Y(ks!4<&Mtfw+IRBVc_^oTYQFS8-$KfE?$Oa)w_@`QX zqhG(ra1vMy_QmzY&qQ?N%fnT3@Kk*~D*MYA{rV-k1HBNnWw>3PJ_^G06sco8II7-- zbeT5%>PUwA6e}ADmC71W%GC?yS8sCak~}XQS~?4IX{R~PePMe!u6hrfKP` zR!?sJQgrIZYQ?LN!;?XE9}XZ%bJi`sS*1Q7Ur(Oz{}YUs8r_hU@08*iVZW@J!Qp z!v!^_+=R2%C;DfxMwrWp^Aq2<>hBZpfmRtYKYQ_lU*X<+AyPXl3sx+hoXWEPicO?g znCqMdj26|Yk;kxa$LH$mKFDUc8SnG{bDFpQVmyHU2nrkVnGd&lVyAAYv->;od&b}8 zi~1m&;b!=q<++Rh6)Ssq7XDUv3;+|mSK|TvWr=I2Q?Y5^DljizY5@ z-ydE|%HG&zho(%IyO=e{l9SwnrzUQ*QpGkpjEO&f@>J$}I=znLY^AJvx{{iz%6g80 zXu9)2F30M=^F6`WNMAR;Kj3xWtI}4zkbUn}Un@Py+{Jg}2lNB!Da7Yksh?s7?87PQ z46C|z1rN0PemWnn<)s|;F$^}oP!qX|(_`l?p8szc{Jk1kNl52g=?g3Iu&q{ynXmfX zHr&*g;>Y#Qy0FMyd@nv2nC;Xv0rgqb+}g)6&;N1^P<~gF8M*oQ0mqNmRHsj2=WDRz z{aB?_=6f$jfu5N@%=ZMAjwN;aceu97@>#46uXt&!%);!n9_Ypk#OCJauf=!w+fard zs-&)MTGE5pR*wT|yR!b6+||D32cS6Z~FfQLKZ!1CKGk&2szPOMrg`UvEf zUxi!x#dYBEBsL0Nga?BEomfZi>>k`Fi}AaDEp#OLUH6+#Z52|It~B*~B|^$NQ?=DOM$y;c~M zK_DPQT6jO^!v2@WjOY2fz4`AoPpDfMMM@_8nNPH%|iLytd?c?2e$Wv-{{f{9z`cb`03cl#MN79E_2 zzKrz9HfuzGwfD6fEd1o42^KH-HojQjf&K({W1W5D@0xGm>E7+QfAB;oW#_7dWY)Qp z7GM0el&$g9dBlTfI%{UufzF79bNnrev4j6f3qN+RX4f$zr}sS50ewlX!9)VN{lO;;+p!s!tYKj8ABIg z;K8-;^z4pF_+dNV!>O=WPN_qhgqUI;glos4iAS;&mdm=CS#;AK?S6?DeDt27l+E%W ze_V8IA5KSfhnn%rlwNkZoeIt~H9Z*)8Du4bCIkH_3^iO7HnO@He{3;LT8pBg7|Vr* z<95Z6WIrT$=AB8UPbN9euSMas?U|DpLh2FDvygR0Lk#@I;^PISPpXaoQde{k@NG(^ zZRL)^#h49U7|e4YaV-i+>CX(}+-H4ZIUIy8H~+`z-`ml%GUPrTGC8$?@mE6Vu1>tX0Um=jZA&6BI$*g<<#A+u8o^Jl2zXr3sYpZFh zYMH-UaFE*E>ma|s>N>3>_-o*S`#+)M<#vwqw+U<8)$riVxmnMm*txaDZ&`3J#Br}D zaX2l$7AK#@TF@#di-$gkZ!1?=U4Dl5iY0WJaS1E-bG6O|XfP=`23g3Y|DAWsf+z7^ z`UcoFNCyS|e2KZa&p$E&e_xrf<=yxVdJXY!QJRh)COdRfa+NCS84t;uBy25g^7o22 zgUBBZGj_%}Iv{odaV*ovO!py_tcLu~5TRmmuIrHxg!X68G~k?sVf=a2K0Z9l+0!;0 zq~?5R!0eLV@5##GeI2H{u^Al77r%nql|L5G~GQd-e5YlOz3i3lp%96U%D#tV_xU^8qH= zowz(}ptvid+{JR+Ed`a?9LZ0ZRFl!i2^s@)KbY&q81T$4hp9S@FE!?@o9~4tJH4dK zVKP_85|U^i9FVDFyu@~Rv?N*j7(Z=HKw$asl?BI=wOLE8{4J2Kb*a_aI6P=p)vdVd zRaMKD2s$~;Y(83&okqrIF4+R&^?@{0^{b8y{Z-xVsY5@54&EPDkNY<6(fYp6yhERU zk<19E#;p#`JRSN8yI$IS zrn2RwrA(^Wa6`0AC)h?!IALHwK)Je><(cz2)O+8=QiUnxbgOR6Vy=WHe&As_rwez? zT52!UH$Y>JxnGZ_yn6=v#s>#^ePt+g{}J$Y2|P#AcHEe5Fm1)P>(fg}+?S!^zS=97 ztDy%Azoc)j=_n^reZD5d)r)IMoJgqqRb#az;{2Xd2d`PF)j9gu{c3AnTCHG2ZXMCk zuSW9HP3rxu^gy~w8>;O-ETzRvMV(HcB&i;(LkqrG@=0dLW17TkZIZe8a~8_#lFwpU zXsRpNh^^@VRh?)BDgBT^C0q{fj;x@$g^JWm6B@IY)ZxENShu=rqkC?J;P&c-MXE9a z?yD=8mo5DzmZc|r&{kGwxS-Ab7&4+0A7PpJ)C78ZM$+Hxc#V~FH@3p}icdKG!L3bn zIL^-Gd8yLBs<0D6?2EE-bxd9{#Njg7=jY_ZGn`px6dIX>GCnxS(rMXZ^5fbxWeRja zDSon%{K>r!iyrp}dTpCR0!3_ec0?{(-&3k#GtqE=P5nlYnWCwcV|t3yBa{KTf?b+` z`56jpO!Z73YeJqTpDaVu@G#K|*zX}fA3 zI)bOY{`jTqZWJKH3P1%ja*#pIaND%Q_)8b|7m`b{hh;b1$kRw(ooYMFpFn}R0`|86$s{bSWe#%_u!U`X;hJSW;9?#^*aH;*&nT8qu(fdKi5PXEijHXbf2;iCj4grYD(| zI7Fj*|GA;v4wNtWEnaKJOZ+AWS*;q$W?^ee&Y|bbPgKns9Vb0I=55%7&lx{KY_HLd z>gDCLSR=G?;)P$wMqQEeHm>f*zVCU z7PfvPnPV4|>+q__8XbP$Z@K948Y?9e&t(1y6D@z9pz@Uo>Ljev8s97ap+QU#;5Z0* zBs8($!BUw4lj`O7aqfT|4iNI?1A=1eTS{#imrwts*}=g+>4k@eEp`-3H^xf~c-%I==)UCJNafX5TNp&3)GzH{h#M>#1i zV{uQxHJIit8^*=pzVStgyC-y+F(QSKn=(EHb*ZY*?}IzUOPPL}qF(H!Tyaw(eRl5oz z?qiATMl15G8;Fw3mR3L8C7;D8>b0hK;l%6|2gxp0o()px&{+w4hQq)Xb|tJ^TQ%>X zntN-jB+?-SR3_o?duC}!Pa_i^u}$(l&xc<1hijSd6_A1GAlgbBD3Dc_02M}qaAN7I zQd=Rz3t7Fva%f;~bg~{lt`%b0tmB?e9j8ptO&vLi_Z{MkgqPE@GqQb!TDU|Wg9bM_ z;WPxpv$NEefZ&?G8IU;5wiu5{%_NlT0!U5ZSWy$)J=`W3r?EkkNQ_zHPN{W~uRUxo zl1~S9XVam}8!)4R=|9Y3Vxp0+FHYIRX=kspTz|5hN?V)2Ne9} zpN?}f{6KbaI;Ey2>X_sFK2g1-l^kbT(yCGq2q?v?cWQT;!vEA>sjeFLC8#}$a*s?& zP_Nc1E%UunoPCNTbv&3h`Q^HJ;CMGqB;AaeNz5$qAep}};r&2|X6$Tq6+`UPq%$d> z@dzg$6hypl4ofA9GDxyMR_HsEE+6(n?;0Fzg=2t|IB*!P8+C(z_%>)e{LGT}jeMM} zDdQ030SS$fg-GBSyJk6FfMxelUtYEsks#lS+kN7=>z<4U3iC+sw`f2oT=g8Z?(sDr zah%LroP4~u?v_6K+i|#+r{fxBEduTJo9}T$zBLAQw-&De@acOmzO?zdJTS^#{FgA~ z{qTHe)a!=eHAz3ly;9>cDK(?shFiKm&$9RF623Q2xhc-3^l2n76HmCOtP0)fiKI(7 zM|T}aXc}HDwSHs5x@D@V;Ne^ufjYD0vsfp9YwN5ZuI|#oaK@lYB_y1xvtLnA`kmqZ z+T`GB%$bv*Kga0Ijp=@m_hp2m53%e^sWjpoP@9K8+6Ld&Sow%-NS@1l9p%DQ#`3iSV9Q(f=?4&% za51u|ugRx74uta}I=H9$eL!~8eZ(_adIFn<&9Wi90>@oL`$n2{AGIqKJMMH`mznz7 zXyVNMf_&^X0v`IaQ9)m=MK+|1MA}da0K4nyJ2&$jT^%l>*Vv@{1GPcF{3e35WoJ}2=nL2P zb(GU<@_qIF9{@hHN7Il^wu$@1X`N@%9C98)UV5rY6P=8Q5tk9aTeSf$-`=qjtGF?< zuH6BU&vmq--fIqB`N9m?k0)z>-?+a_{QbZS9i&YL%F=S?nU211SpNWU-fL4{`2XAc z7I-_V>h3k?ocl^b5J-@Kh7jbHAVP8zC<+9%R&2Gc3SU23Td}S7)1TO4TNQy!eBi6q zYHO8WwPIV_3KkUY*|2?zk?DM+!oH=t|H+$bZ zXTR27d#$x+&e{LjFGyjD&fd+t`NhtTfbe*jUUy#3 z%$Ueh(uowzvh5^90!*fmqnRS^S!~8yRkU8VZC-5mh<23%olHn`ukGs0xy!a0g4sK* z*;cb{VJ}7Bk*8*K*r_1Ras>$4i6e-5=X)idQp=8QozAV!nyEuQG6kr~b(}bsV!DPl z_1L~(3LSPdefw1hVrnfmX^D3QWtE1kjAhM4ix~Mz9t5AQWyR7&bXDZXfbHzKCMsoC zL?b1b9cS3Qs%T4WG`&r0XLTw}xNd+YeS)rl9t2O)8II!`aDs+w8#Um>ks@Ju47R6k zq|@BIRySP84y^ui0oZ&l2*dR2rR>^OxD;|Pfg+_;1E!&BbU(y;n{}Nf%KbCsuf7e2 zA?V!Fq5(J>zOD%#`2LvHZm(qup<)XEDT4#A>DPpZp$*-<7&5O@=$YEY* zRZEynihCAYvSXO-3z@YiclL!+OS(5(*lXl_v65-K@BhmBr1Jo`T7*7NNimY$?wK65 zEb66vc5~&&Le^s4a$3gUD{^v5U2#%POy&bZm6(3HG@)iT`}@4?u!)xuCJ_eq0{q@CYM`}w%T@K1)qx0wvRp;f%0aLq zGaCJ?PA@Oi8!>$^z!>yt6FsUK$8{RhO@18KV3c7UKM<$oESGm*pLW{c^mb6Z?_>`S zUEMB&Uh6-wZ@+?o=V&z9+(cik55<5!y6x!mLXy^zdCp-?=L0fOd^s$Ua_w|t!{5g! z=Yk|QZNQuDo!my=@bXS<-uX9m;K!~?thrSwy_BF-n$gIaG)Mumvp#ESCK0p9ch6#L zR?P7iHanN)21MCOl@TYNN7l5{Phlh#iXp^VV5;^m9bMV5f!`!?2juKCazGI!H4FDG=427eB$(& z6y7sf2YQDbHclYYqGKhoOX@>hhpl?c;xS>OJLySl7+K|w5^<3-zJ}iTR`JdS`C=Xd zM8GkgUW=yI01StZ z)?YF-SW%bA8hK_?zHuDFGuc7+@v$`1;5sNEs`9*ES@+hD_jNzY+|2#e$g|iCKS!J1 z0%_JoH}di>y+pN2`nRSSp{vDczfIefLi2P>Oj1;o6bXW~bv~Ey*bf2#oX|445J-TM z6{aE3$FA$RagaChBs%H$fhMTQqmnO~mFneDDCgi>uFIA(A|jU6X)F}GN`54vQyA$| znAYd}(@7lBPWM^KpY>O!>A0@5lxJkGq9vVu#r$gWwdh*Y0*qR8Huy6lfPnm|tGuQ~oS=Vp2)l%Glu4CF zIJK_vM+tz$9eGp`rD5L}wFuWqpgclT>$;PCBvlGy^6V_6g+K#JJqQrc`a*na%^;nvq^1URNgVdt=t{HPxQ5dYNHHNZRCyge)Wd1X8U>JBQ?`!Obkb7B*~V5-|^1PUJJNqv8A@<+sw>eJ9uU;Wb+JF zx_fP6;+W>^YNbS~V?A1_aF&%-X*T0oB3E72=_#&hj&DI*C7WKhvE6#-d!?YkL-Y|T zTx$$U_o|PWFQ-20nznNR00Vdmt@|#i2 z9PqNX2h*5`4~}|f`ceSPy|C9(**%3C`qEvx@-Fem{}P6}B8B`zl2fOk9PgQFoVPV< zq-jVhvpl(2RQ?N<$-Nm5rvE;r&?^5>%6sr^l>TJr^bXI69-PkA=xXYu1>CdPNozM) z7hm}G{#5n`l}X4veJVl{Z6dH5w6`=t-x{eWD^0hlHAeew=?7A1J|cH7IUeOUr6`-S z2xjr6b7}NTS5vm7Qhl$WXNwLq`^?&*18oB#j%3bkZj>$AJhGU|O%bOI~zG>s#9sFBza=WHE9wZp&)IcD@}p`GyZg z*7-(ulR>^#n8BIsUO}F16o}J+Dq9DJ&1EXo`AkQGzGA@1e5LSW1Lv!TlM4}fxZ1FW zNvnQkgEh&IJ0*)OfvkgC2ijiSA0K3`ci&|=&fs`0L1^|%Q)#m%h3`@FwJZKf){S2X z>6SMwp~@YLGBW^9wY7P^a-#UBqErT&4F)J;m$;Cq3J6k+g(9 zH~ARnh_`dI)&lNXY^`ZiO%rXc%6=`a)I;%)bCVi)uSJ1>XtGdxawXiG#JRK<@Z^E2 zK5-+wRPNOv>kQzGe%n}Ivu=m@tDk7LQ1kS?^Su&*LIV;?XaPDQ(SQo(&(xn;zFA3b zYA4QJydR@J6Q!~-A7iD2r_H}cnDmX%CVUYHaYRO_?r(Yhrshe7skn9N^?OLS<_JOMpvxY)a?s)^!4)r&*V2=%gmJIY3`+&sk5b6x8Y5g z%KA&mYJ(T_TW{+)FGO)tXq*KcFA_8X}7B-tmIWLCXW|DBT8{F%$`xf3+gP4=(F z0_VbxomnfRf74XTA208{@`)SM?y%Rmxs<%Gabic=pB8Y>Vt=+K-(dfNuq>z{a#{VkDzDfi;wJ7s~A`D{GjRffrWC`YJ9{M`e1gcsVV$WDoDfS)TAS zaAkG?zgqQcnkvHDv=pf(j|%-sW4pGj_q8jSAxwD)e9aJS0BfLrTD+r+v@RM~v8N^RS0 z*iFlYBYo?}@r7ByNTGUzMf;xYzOYBPf^Ql_&Aj%=Kbzz5H(PQy81(*MgM|G-=6OF( zDd?fHclaYI%QjZb-|%J-V*RFbv)TggS)A3)+f&L#X91@8-iH=Sy)UyFJoSvCcfRdNTi(lW{|(98Fc*)Yr+t>MA}ra{$WC9A!ne~*dX(GUPi`8d!6X`=?Vo@whD{tkT?8zveWQ;C zX5q0iF?C2(pvOK3Cx@WyNsuZ6Imt9(>P}_h^oX<7wp_}|Xh_VHr9*xCo`~zslC{mt z-?$7TeK(>mu4$G1j%%GWQdrmv@>T8!3>_uTdI4e!V{C9 z1j)=VOlRWb^0`#^Tj|W8ehqSpb3-vPmb`a~{gR$0J3i?gaZds-Q#zS)9&@5D>C-)*6M{h|@y zKdDdPa+BbYrFaL?lGY0F;M#9#FQM8r$eluoIC%lsu9 zU)qtcbJ#k|v2YU}y61G~JYi5#$mW|=Su+s@k)Cw)qiDn}p?@0rTV&7NiHce> zjwbcRLNFc&qfoiM%E%F+4%!~m5yyFk6qu%~jxuXt70Yh`SpqKO{A9E_rnBQF9AJH< z1jk9f&|84>@W(No6hK(XaB{>6^$v{AmT&H7B&qRzb09IS6 zF1X)Vz&(rmO;dMUQBS4Y0X>p?PV|y(?Cea`SO-3QX(uu#6Ml|~ZGpeBBTa^10s8Oi zBfy$0Sd)C+82U5bS8C=pmUz8fy6T?BNM{#SG^N(`ZMBaN1Mb+9*3$)o|N(6UPm}Sgl0 zqd*Jb*(1*P@eE8HgpvLeA>*7#UdX1~pE6Tt6OZ*bdTT$3Y>=rsv>R=PD=W8;KvTUV zCA$GUU*7-_`Lm|V`#0n4RmHW1>5jt51x;*yKm6~g=gFqzONbQ!_1J|;G>d?q#GL$be{NBH{w81=_1@bd=lOEQ7}~x;M=d|0kN554~^-0liH zRM^E-p-p4u+47mN(aDq{c+qL59OLn&X-tGZ9Sd~J0dbl&oRpaW%dF$nKn!sWx*%T? zCtXad&*LPGiL<;q9QwXQ*7${PX;oJ5rez*>r{z&77~xbH;f&>{&xWz%CQN-8Hdoed z1T&T26J-*ojqr!L6-VIl1^h7o7l6p(X&EWyMWcrK4WK(>TH5-V&q}xoFRmJN{&Ku* z@;=@mbtM_U2R{qFzAl$gIB@uyY{Q>F{$+Hi{zy1l?f>#nOb>oRbZuCQbx^+nyIq3o7kJR|fnA|<$FW1{0UPmqeL47~ zd$HP2_U@LfJkMLyU$t2M(derE0>4$5X@r;IjXHm94%+$8u|(u<6nSf-#p=*~NBosH z)ybRn$v7Vo?MUabA5`JutcV8rjbi$NsYhiZkBx&21Mt=C@nI$D2;-Gin|MMSCnT~O zIYB_VL%CPx4Y%@^%P_3)69FpZt{^kEOURQzM962wR#InMI#udx_*GQXm8b?)s9D>d zg5%O&9HvkvZrVquF%eFeN7(S;;qM$xT)6mW$lb1Ky7{qvvnLCWz>zJvqa>&DQ8NMY z@m>dR;?1S+F@~v<7e55CdRlZltP@)NAmG&ZOpAQ%;`1jzD?10@gQUakCdP&#l*AvL7({ zcld347lcmxa?|MhGJJ1hN7V60@EdRqeuICC>uwnseq1KFz9uM0E_Z~efXi>z$Y$>i z_yTP4_taGF{2494-$#GLHRxlo50cj5M~gF!@_&Ti-JhfVZb1AN`1j+zLxdp)CWa)) zEf$qO8NY95BOg1st9fZFw8~9NK7WxF-|vDYz{!q7Eud2?Z6-7Q^V@dI`B!cQ**;vu z5jor0>YOmD9RgNzdBhM=HbW0z)9VC;)MFS?eMpMWX19KG59;7}PGzUx*rJh6mZ7EC zJ!zb{001f|Nkl%%8 z?a>}cU_RT}Zf9g?-$+8*~!a1!b} zOZH5?8Z>iDY?=vi^CIx@$8JVlmjifS+f3;y+QYn00(id}=_gm^xddl{PFTP_i=D8t ztAiHMzinfGjnLOJG6-Ir%6ss=&I>DU%7{tsCzcL^E?07=U^ol&2j(HJ55#?`_p!k#=zAe66n3siPrymu~bg`g||&gfm~)8x&V-SE%MFGjkVDWrO8EQC*d2&bpn#%a`(d3v)Q%5Kv#j3x?0+;i$^ zDX96GiP;jP?FYlgSXYW^HKv^?H^Vf(Nn_g>{Y)B3Oj=7aR1{{LYu**WsIi+aI5Ig(P)K;sfl#-V#<}8rb0`6H%YdKf0Squ21|BmmxYnmoch#uD3yC*X8{uceWB+fRi0+Ft5B!=bL^5OE?qpLJvdK>WPG*>7<(qbe#3E zUh5(QBSR8kzLl=S`Xfa82ARajgTIyu**RM#hUqHP>ZMOdtMkY!=Er!Ya;6@gt_iyH z0lZmSvq^9Xp$R7Sx=jN%W1neSZabZ{=3+YdX@D9YDnb586v%Q&ta%%qi!~TQ;cyf@ z4szv7dO>Er*%5V$FQW%_WLWD808PIgj%>4L&2yh}`9A??wrnputfxGivm;WvUGGD| zRyG(&S#Qh5ZkPF!7q!fV4E$AhVul9W+Bs)|E?U4ni(NGS{#Es0{?DK=`kc+&1>*Eay}zSn_ew|VodklTInz{ zg#ja5B?QC!wFV3p>rR6ixj+o7v!GoDN2-s6Vk5LgVJVnk^8Y}$oRKEz7`HHh(2Zzf zVlLsJ_5{ayCOn?(VV@LOLC6B|?YQ4Mi*zF>k!!vyXZqr5i&W^wYH@7thO93*-Zsla zXx$^Q>9QDM`hj&HnBaAaG-=MlUg@56>NZyH0JJMr9IEha!r$VjQh`4ACj|R0!X^z8ZO{SX33(nU{TO%@OVyl$OPB zu&p;Tc-0N>!F-z9>F;^1Gobu?e^|Ljvw~U4pOIHPYk|Qw1#7ehW?e9Q753dwGKUcJ zdmj1hwtQZ45f7aD-e#}|PwN7&{j^dpw{)E0SK{DA9>RN6%A=)$LT`J36XKi&CmhMR zd_dYuM5KiQp@X?afpTA?H=-=}Kdv&7)}-P$0Gvv5Er)pdOmc2gw{jb5L4K(($7Ya= zKQi9djhBJUtt{^+=G zYu`_nLW~ng25?+NK3vAfe)~~2-H66^NlWzr!0G1}xD=tYy6#}GPsf-k)b=R5E^Sjb zxQJejs#9B9JsayrS&IS??+8Fu9vIbg`P;VY5aVdaqMS0sf7#`b=ntqD;8THXJtlr7 z;YP)Ab(M!<2Iqw|f5&m>1$Nssb3E0>49;hw^d0EinV&>L?h>dwAt*tYI6h63$A%ks z0{X4IWzp1f!Wo%~^U^sc$k}4m2aAs8z?ExA`L4%>on8mN;H@nBuLSuiu`c>Ca@$Xv zKaDyKU)<(tE$E5v&kIQfN|@bd&!fjkpzST0!}%w{x=heDZ-PCl9OL&0PP*26aGCY$ z2d&=`S`SFM>@`~F#FKJV;nOdD0pAb?Vu1JWDa zsWi9yHcf?7R+AJz5-9Y;`O~z-g4XJ!8G<*(S3LH>$Az8Y!B@F}T5YE6FgMmW2DI95 z5`SyFl4*|z8w=iT*lHBu#Q;uh^a#~1+TU32Ok}Y0@!f$6SMGlA()e0hZN=!A-8-G= zx-1S~cN8=1Ujf&{D~BTU%;!Aq=oFTjgPZk&xG~T6?6~%Ih+`YywcrrJvT54B!@+L6oj-7kR4)n{0Ybfbms_KY&Na6%tWfPx6 zt&mxuzHQcUbz-9~@$B(-OiRVYjQ-bZ+<_z&(G8`3b+kzGo$80Nf=1CO169s4>Nb*C z$q)Uaua)`o=y9hQ>+#cLygr!XGpfakeEBTRd6*LAlF@W7lN9Bf!M{D=ZJ-z8-ML9U zpJhZs#Uy1M*L6L|V()R%0*BUCNOO~bDjb9&r~XMzfrP6SC;wO!lbXy)VVI`=4~0<5 z(2%3_8NW1*%9qAL>GSzM(+)9}5;U$umKu1>>vsC`hrrCLcka5&3W9bAjUKbsPBH92 zU+T)_)g1m)sjeH0siQK>syop(!ErNN6EeL5`M;Vs@(3+kn1*J*O4+JRyN;ySLsLuF zthsA7+To}kaRbJv)2~rJghrcA#U^d zwbi&pfX9*{9Mcm}gIOS$$>Gh(65*dnw4D*?U9Qu4xP9s}l3}OLT!k=oVOjs$SANrU zpVB+t3OS9xz$&pSUaQ#qFS^4-(YYxn!23>%_}bu;f_ zGDTlU;-Vb6&kr?&V#b`5tJ9(P4+XrUqd( zKQEGkvBlB&sr&044VI31tA%9FwCf&3r>xDa2fP}b;WscOR+|L?Xsby0s=gAkq3TYx zCt>JJ&|opHloGVz*OY8C zi(||lv<0Ot-cpSKlS%6nloMVW0fiR)cF>_z{1*QTyZsel-Rv#6Z~Zp+R>A}w!6;xD zg_i)}NYTzE)9ilh{Hz=13#*QRt{d)NWUjL%-nn*#%#$`5%%%gbt(*KiLO*XGZB_of zF+FYBAGr8{>Q#MfF;Mu$1argdm9M&|KU#(NIg&=OYsFpPy)QN;2Sh*qjH!mX(R`_~ z_2`ob^1vFEH1e3puHRrL-+O=UR_ki1pAJX*e) zlMw$Z(v#~HnW7^~Fs*GtBe_wgOH<(FBm3^E^h(rF>J9P3pr|r3iiY&n9-;oP$ebhW zpSfr$Vdl58=NBy^+@qC8Z1~A7(~oW3#L|g7kX}Hzs$(QuViAYH4Ct`Zi^BUfE_Tf= zo#4_x=<|keZ|ZDuu80Mly%%B3hr0wL-;bwi@llrdv@^E_zV~}{HLhNS(lZC&@O>#m z$pOUUF@&$pY7DculyYPw6_96xC*w2m9eTgG@U8h|xJN1;((T2rVcL4@ip6NBXvCM7 z{K!oBEWZjl;B9++f`R784C&XC9qxh;f6kA2eezs|%#g~fo|Dy_W~Pgbjn!MsJY@r( z{3Ts?iTb$bvD_xd!mO)PPuk5zw6AyX(C3=OyiC~mj50b0EM;$4$%wfTlw%-37^f8b z^O#q4|JY1h@-5YXJ@!PKC3(2r7AmQK@&-ueLkkb zb@uDedj)i}QFi$qR;8~U>PR>BaPL`7zK;h^tkas@ej#o{O(x#Vh#aXC3hHk7g7<4Q zwKVg*)iV;qOQ=6{ou~8i6Vb~_Dgh*tXoW5u6Y&6)Vu^X8FQgx zNUPZ3azkshsnvS0M;Or@K%pKIlo*Bp?fKG7#&_zKdzK$pOjJ1$Mp}C7s3#wpkB#cZ z_vtr$w>A)FH+GzD%7{OD+q!;n+zuAyR+Zph7Yf&r7sS6zB!>&PKdlk8@QaMG6CRV8 zqAg2_0FvMN5{OH(J`4MW%#wwi7e@W|ru7Z1L!GDL+!mI&W*L0Od2D?d-KpBvIPZC^ zdcE)crvSFNc(3HE8+l~$s^PRx$k+WEqOOD~Xq&u3osyFoRvuED^7Z6n+7e^b-1JpH zR^ziR1@1{lq)E`#L}Z-b7oyJxYe!cJp?<9cw{I2r#81k7qz!$#h+L1tBt(t(9!#b> z+TWIv759Y<3a+CNo6K@X3ReEiP#b2yHDMX7KYfO@ZH|y^u6T;yPn9u%{V*lh&nO~d z=tOt2TvV=NwI+w+jL)rB@!vE0aEb0BegiZ)JFhwmLAK<`A>3^F9CUtwwi9lK<)1o4 zbGd_-A>np2&>Gc>v^7P`UHvZR`7^<~l-wC&ux@g_%`#;ct1J_LGnKnMb>^x)hYx$F z{+S`Y ziYRUHO`JuFEca;RYqnF7nEZF2%zBPg^s9~xsUFxC?2EhR9DE5&h;oyI`h?fiFFku+ z^8Al~NysHeP6jdbx~m`#F&+vqmI1Wh+b}vB7uaO?C(V&raFFcJJN>5pbxRi)9)(Zu z;Ow)*v2K*zAf^F+H$^+zs+G<6#;v=a9aTuXA})Y4gF>OPXYuTD-gtT$>tsppUSpD` zb^=G8Z)T-X?HF|Ib zTq~`|UiIl1p^2JHL;nVwCviWk0Z!~sO3xWhv9HqR6MM0GMOtmZ-zJT%PXnK&khia@ zq8{1$X8dlY{j8HK^pTbVRzB0|grQR>AFb8PbX=ZPZgbpvg8Jx;#8=x!c0CH1 zI(2Wx1vFoKRWER78I4BLp>e?YF^SGu*|&bhMfNFL)^9i1D|jXLH=!NzIvd5Ny_6on z%W%{04@B+p6z3&?lLxo~eR^(|iXIozBgA1jUtMWxO)&J|^a~t?Rq0P^?@+u?Mwh9J zAsGZa#waMoc<|U>omWyW>sK~Iwo)WzE*lROCx%|Y&W=+a|KyBb+JVfJ5G#15uFd|j{JOlpDixsCT)Fy4a>KdgVA>3+e?PvjyH&HeO%@j*r^36TFM7(^6Dmx zen$lMxcPAy^dy0{9$l=$^7tfgly;t(30dp%FOWzWj9#-(M=Ck!@wQ!MtL zSiVm*(&MrCeDBMoGne;`&Nx`L$kpQYUKUb|1DwVarBy7%Rwk;@+3=6%Bj$RK+7nAW z;Zf!q#X&?hoVRQ{oQoOL+tAbdl$A~uH&qQ4RU>n2xW6s%#1N{W%cq!#XP`{U_7)M2@zClgZym((^wD_Gdv^^6)Y zMu$pDV7Gj)C-L25!9rx9Ii@#mWO8aVDXA;novLyqphWAQuQX7{O?$e?dpdT;u zhr}NyWFh@y_ml8yfN`Qa!w!ekC=P(;+W?m}*|8rO76Oyy*G&g;Eh-o2IUi*b`Xk?k zzfIQ6)FVXAE}?MHn=P|!#CXp1pxN126)b23PJjAjvD!ae8ipx&I51n)W0opkH)2G1E))4{B0w5&lf!n^8zb#2Wk1rrm_ax^JYDteD zyxC#AMl;^J1CYVJ(oD}q&+s>0(F zG6i1n2E9)!V|7dn&vOk(ye<5gHAWLD+%q{TR8gY}Z-oK(1ylDpggNUh93jkK0A#KG zHej2T#lBeE;fEhieH+2o6bjnsmVrJYMTwpkMq-Y*%?&_t5LmamNh8P%FOq-HXLBYF z%pI$X^uD?5qKQ4UoJQU})9G!Z4me!UJAIiJU8}w8i@C5j%0lIU<{r_zy{ua49-=di z&u`}(yx2Z%Itmz-c%lk@h~7u}45VNR|2t3x)JhW(?Ne;j6}O*AVABmBA!pr)m|m>C zLTlFvv{RQM6Uu{nXGhpcjP-L;gtw^tq`bzM7Tt4m>{L%R-NxC`wOW23n!wu$U;CDj zS8STZ=e{3=W`!|P@}<4|5jeo`MKh`QQn_27X~w_m6V%zy>3K1wvE$-UE?1%Q@8v%;O{krb?Z=4F$p| z`;i6snAK;q!1}Tdj^_oGyu*Qe#Z@ppb(=n?vE(987vI37_aiO0M-Sh!c4Q3V5;>;l;lq-kp`(P2fZSvg*q zXNorE0hn^-J26PE!uayu`v5$3TT0Y=b`Zo&K{oJ#+o};YBRK z=x6c1^IO*#0-k{e9IVX_@&p=0HD^?)D^`U$WVKfXJH|Im^W9NJ^4v1=xgaXtZF+j% zR5drn(BAILyAyrsiBO~L?{Lp*-882%_0I$>&b9cj^L0Sy3rC~vfo%RdQ(Ti?tJntS-PBe zo%^8Xbc`cR#4(i^SqESfkOdb9&ha2)Gh<5+hag2!#3;Xi|E*AqYV6S~$$WH)16hoR z;aUsr(tC&{VDyoKL;p52BK^EZ0GQ_( zlUSW*@JPga3`_;LFgjADJS{L1MTIG&TSA^K#N6U_%?;h!;uoz+9OFYE6P+9w8LK)PtDUY(4}R z@PYm<(i=cN$=%y7xT4E$+Fr~(>0bz$x-(N^=x)96&%-0ml6%H!<>!Pm^L94}9NMVs z>@RJPBDw~rMoMBA|#_E<3O&>y-@DA+-tfJ<{g0U(USo1y=nfyvwgvgOqU`eI4}d9r|yj!kdK zr%0tD#?F+s-sD}xy9}yRq45eJeA!-njtv8ST|#kgPznst)8Ug>JQ)(s%SerfJ6Z;ytkx@ z2oF&4{_NYRcpfZnBe8WwTc)t}oX&Oj&N3a}l6OL_@FR7m4tV2g=xdnEMh+MRUs#7v zGPM)-c7Yo;`wTao7RnBA`XFJ=JT+J{EQPL#md*V|5EW48%!im}V5B)=5%G%?BC=fG zw)Lq?k9hw}uQZGR40KtD10Q!U@*8U+l8y<*`!u^|wE=OnKiBNj1rv>O(8nCV46>IKA z5a9(vH;p$k7b=;4>B{< z)&7m|g-G8}AMx~vWJ=`+%s_%8q=2C+51N@e`i5b?%7}Nw`;P;fey?}O&zFCAKnxu15xc{F9-l`Mk2zMT)N2^GCfa zZP((}>UhEFLpst+8v#CAs<=|)kFT5uRy)SJ3fnmeRjl>4yrOB;k&}@x5f74tj{rjM z1bS@U*5xIAMY-1}YAD|qY%7^ZS8xsuu6Fcd(@Pb2f9GHS$;1LjIbNW`k@%Bf9Dqp; zWnOlcmP84#{KOBdg(vwq`94@i4M({p;zE@-Ezw-T9H6KP<{;Q~tR^C0#q6j@q? zWryb}xiE?6+s?NVs`{Nl#sDxu)d5xjBY4>bM)Qn>ScnKV8VPSgF@PU#+hE2sdR3VB z(FXn&FY_WqEFNX)2OJmlQ*{?XN^of~==3GEFqcUCAIbiIi2TV44n^3mexdPs>m@qx z%_~bbQQ0M!Akla+LZ?5lw-&WmiLsTWOyg3#8y}=!fYe{wtKobs90@T;LIwEJ zI+5a-ZHwvG>Waw>5 z5b||*bkSNeDtzsiBpHBIjmF3x$MNx9uNS|@kBAGQbla2wHN)9rod7gBC}}fC!Xgpj z(ZTTcoCNSrhsO4BiXuWJWzsyeL}q+mRIN)Vhynh{FErO5_C~CT#EHO_8YAL-JiZ|6 zI6Gl8m~rK#%GlOh@5jFa$$u4>q#22-;uX0t8F-F*ZZ^PfBY*2@DlMx&8P2KKq2*0=*;0_T|GZjqE%0}(q;DIlJeuV;t~LQIyYGX!(N z>ci50mB$&-D-d9+Y$CexzMO)$f>QAzv>jk2GAJjzbxq?IjX}L60Zsqb4T#}B zpWmZ9Yo_CRxOPE!y%0phw>Lyk5vaK-till54yV8q$;-{d@EL`1H!faU&_S=pmv6kM z7pqCvHGYKE1^0hp4bD1H-wY1nvsECgv1TA$?on)S-L!-&Nceq-#T)KJ7OozjzU@tF zr0^JM=s{EeX!V4Qcw(;wb0AC^%NsG@Eoa^$fDe!Dp8ht3D6`_)@=o>^kmxot5!$Iv z@jVK%Bl&+go;CV{qP2g+8^M;!yWy(Qb!Qrddn~w~LO5|f0RL7(@i#8SIB^oml4AHo zQk>GKCekZ=?O;Kk#MwkDWB+kBD8X=kkebQ z|5A5U1EhR4Z{NdBi*BKib|pAq+A;{SqvAY1du8oWFVG?K2WF)fV`w@67 zr-s`0Pfbo@3zsbj8<#*~@CC{7%01q>3&nZVoG+_8MmNgVRz>5KCKGs3;qs&F*$Oj1 zWHal3dG1T5`9~iLH?Cg4h)L`sG$*#LvNQkZ*@=SGg_rxMOO~xSMGUHbM17V@oIiX7({E+J@6;tYM%=K5jEcCMma+9x!f}jQxYM**z z1niUd=(o)G%jJSDzeWEpO8mz!h0*IQUl`1g*Hj^NV4Vj^o&2i)=jElP*F|x{$P;qj wPi3Rnk?XNf1`Y}(9`^r!F8^7JS_l5Yo|2c>r=j%n2fSQL^6GLGvSva54_HX`V*mgE literal 0 HcmV?d00001 diff --git a/segmentation/resources/seg_demo.gif b/segmentation/resources/seg_demo.gif new file mode 100644 index 0000000000000000000000000000000000000000..2f0760fe7a86912c07f9873eec3909ad41fe2acb GIT binary patch literal 951973 zcmb4}Q*$K@vxT!`+qP}nwr$(CZBK056WjL0wkOGCe{Y?CaQc3At<}{RU0siqj3hUY z2?#g{=qob_3Z@wbE*3GBGbs&)5~~I?cL0w7ucVB)nXHwqioKGWf|I7Ro1TY?e!h-@ zhMW;mfO(*irGcsyg|)qSe1&|8lW$i_41RrDOi2lClV)9B-=cw-Y0bng$p5oQ zfN+Wu>S|&l$`Y)M%wT~3S7OUJp z@$u19>O5O+U4QR0d($(fEu69Y*1svz-gEmiHm=-z`Wybfx1tRS2@VU52#<=4iH(a- zh)zsSNli-2NYBd5$<5BsOGkv%gmfw_FQ}}@tEsN7DN=80ZUJb!w{~_Zf_3-x4`?Ck z_>TmPj*m?`4^7NY&CmHRRYfnat<-OBZSU;v1>uDp{5d??KZ|wEK!GfVLOFjpeabz; zd3{g)IR1Kj-v0cD4aj#k&SD~mMB-F&j-G6YLMD^Vmwq&xz5hgGazK~O3c zEud|uU8_~fV;tKquH9%f9*HH8>!{mmwO*+)nCYnB>2yBsG2q4A#jCQYyaQiRZ}{^& z7z~c^W}^Wdg;bQ zdc8@@s7Uy${dTj<>AIo7kIQaCzqN$JP`Fy@q>=&bf@8;D)ujwDf z{;xO8gONBQrGdV`&lfATMvXrb-tP~8W|EYG2L641++Qy&1POfsLEfl}g1|7Pi-G{2 z2LLESW~=>BLjPbw_d}Kltzc}+s>4WpLTlhb6iz5~i7Kseum~)5Mb%L(jq>7A9OF1N zSseBHA!!14h4*nBPX+a%Yfy5=WR$huUW%Ws_)SKiE1bpZ*z-5fUD>Eb9qI%tR z+h>eI+bHs~!0Ui#Q$GizB8D>8%oWQZ%6XjTJl}oY=Dfhe^I<4d-PO3l+mH1^B$V%8 z%z1G%Wz9v2{#kekB(EebnH}2?rAXqvKMP(Gouck#C11k_z|9k8i#$yR(u=mHt|ND+ zW}u&T$X*@Q3*a(d++LJn%DYjX=K$%qm;D&J>jeL1q$zAZWx&RFXp-5OXD%NI#YQ%Pq_9MyX>a9tfQP@_yqTrqrg^}D$S z_F)jbSMu6kX;~LPOy#YdI9!|d{4^64YsMf86R-Cy2QT-sITt^O{%(>y%bqoOSJL{d zu$@ppzi>x#(#l1s<^5KR>b&lG{)nVfGzK^SgC>?`Ig$oD57mLGN`6Jop<47q@1Yu0 zi=b&$Tgf4Jtw}e*dAPMlzZSBSbymcM=~u+f_Ne0Q+s-)U%=7Nq`Yp*`)OP)QHw{zQ z?H~NEZ}KodOh&dXc-&k=RTbLA_u#|3fcKO1ojo@wq1rgI0q$wO2?+KMyz1q$_w3LC zVavtF^}5cx&ujItxt$H8w0adAi6<0hLA9 zmEoN|R~PbhystUiVaMtS8l>ZcvZ1UI)MHub;!vCfXYQZ=8!BGRYGF}=11uo8^e8DV z3I~vl7+{grwBw=_Zpo&2KgiB(M#UDDA^JQ`l1#*;nAOBr^6O1iQYA8H?$#CFB74tJ^EUAx^^zUk z6e|kFg2CVLV`U8>kN60}xlwJKBrys{DL|wKX7QFu`jkmanm(5jXJ~BD%1p`Gs+q{M z7juSpQTf%wU%pbciX_PD(Aaa>QveEH}A96$$mRA`fNBh?SX8AF@E;1KqVgUZ?g=KIV!GwR!tNIJY2^#?qn_U6(X)9XJLs+n)%#ey+x z5{$Y_`Q8wPn9XErCqrub_p8;$P%~Wjdx*a>?T_`?bOeH*6018vXyaqiOV?On|VZt=Qw$`oIM6xZiJ_(=gZS! z_1|i;A%K$LqI?bNJzPRwS6659Sek3FPdRtPAi9+>dOIbSIL^>TK|(cif6^Uf-{@)Y z*Do?%wqJ+*ZWWEvP8HK2IJ>biZ${=T2DeW*n}s22LUM8o5VW1N09)sdJX?EK4n@R7 zHiN$0c54g1Z{tNCwvo?m`M8V<$>OzD+Iw2R<6Uh_X_g!9q{QX`YLeK1j*2w3j8U1h zv`o_QQdL)o-5g(^;Xhj}3-IJ;Z^K?X`)c;?YPrH3Q3<0Pe8uQr%}V>QHO`!4YLKj^ zC}2&keb<97zEA-WUi#Gj7KM+lKW7@^M-Og7gt}!AF#zHUk7+$OVF{Kwo3vba`+ydV zBOva}RG%+Tg~6L$iPoHL!DY+{(OX*`OU@MiFYoy`WtaD>_;D*5Vp37;L0))s1nVc6)qmgc$czU16qBs2fD4(ZJ1SBV2>=l+axO2acxA(B5{ax^k}FmP3v z>g(u0;B9BwB;ZQH1`8!d7c6c!ZH#Rr_zai&9q1Fz+vk6u^@-#tjbA z_(o0weq)}%2(0Xxczt~dFMjtOs>RZbR@7xq($~F%LgxvL5g$So!ejKWM0x|0>mrFpp&9r#zwi)A!uKOa`d%Q!>vpKCf_Us%&Lapi&po?L>7NM z$w!zFT_*Z>jKaMr-?LWyxBzNtj)UhMnal1YT-?{35_Th$FVK7eiZs6ilCI(Ipzi7}^|S$YI!S44fL zRpXfxX@xcX6p#W~oQQlUmxFx|o!-E}ureEgBsbI!0|s6|TvB){hXe*x zg-yKz?9apKv-~z7p*pRCJ|f8a%nT8m!WGZ_bRNRj+G1)yq(>`-;nQuADV?x5W6vhN zleG-y#+h-bBbVM2Au0@($BLkwxQhKWy&16mCCao4M1*ute8Q4DYjiCzo8#RCJ6tAkL`b5xoLCqNxbgsV3B! zUa%4*CP@pAW&I3yhvO~|9fhNh}Z&6oV*$Fu=(-k@?TVYgQ1s+f- z^U($Q&tcSFyrB;{9TI>6t6T`|h`{PBSTS3}5Ye(X${4CBd!D?PYBkC?{VpB`7PG9r zm@GVLJpZi(GijAfkNhy2g7_`0nV9S=3F|l?oF$q9U=~D9I7d)kPJw#0>!+0F$1@;= zxUeWEMOqXPZX9!VNlUu@YedKY8YinQ{g-=c@+(hiV4EWTxKCG$-u99VZ*@rwhj+_! zEDmvgpH2c5O7!HV4s1^mylJo&}NCT`cL%YYYoTjO!mYh-ZFf z{!<;>9wxA(<^;nkAdOMkCd(ut5d11uq(y}{D33KAKNszqzePsGT=Qe)7L{DH-Bn{ZUT8AT?`4h; zLXqB%+qe*F%)4Dk0Bu{JE%3Xi^5T>B7zB}Kx-zULZlR{^Fr8SXCvaXSJ24jxtUE9@ zmfCokZF${v2wkP9$5oUYO1U~{stwdaD?uUBriTN%%C@0$s_|>PvO9y*L{kbpOUjA1 zksXi9b6pSi?Z0)Y?VoC)qbmv{Mih|Cg!DB7zb=aC4pZ^13*v6FG)c z^{n>^jo!u$PtGRGsg5`9B_%vaD`sv17GOeA=XH$K#O{J&2C9axnl^S_DzfRddfKra z56Lj7sdu58lAF`f%2@#PZ~>pK113d^J4M`>iU=9Xr&(c0c5T1Pu^{|3Q2eNDHZaJg z99BGeJ$kumjrqDNtJ6hk3U~nq+s(#uoca&2FppkkIsmHL4oUW|NV^K+NKPleUj=2X z49S+ECH`J~eV(C`3n9inHMQSmg&N1dI^zn-dyocrBcu7U$E1;)Gc<9Lg$us+ah{`53!;&d2~6-g)u-wXGC;NgAJddrpf7GY{Rys*Y~ zNSw&^jegliu+j3;rFF2UmHh1kYP>bJ@G+3A6x2$`hB8tUL!$|EE<_j4Q42lgkzqS0!#oC2ybgk)^>SqWrp(Pn<{$m1lXN@Y zxp$&;^TtXeCIw+X=;dd?24?5z$v4Q`zsT1w@&zR(H`CQHhWv^e=WqK=}?@Lo;SKC78D@q~(_B<&yN->~}+nom2-l7j^!w*pe5E?*CSW?tr`gu@rJHDQmkC;HAB+vTy zjfE`EM%YhIGtY_iX_DHGGLmJrymVTRn!u|m^Q$eM!VAbQ)ks4HI8FWvxfU9r9s1H+ zfx-r?edQzi&PLGm5=!FH6yU?HO&6|}g?(1F)Pd0NQ5-yt!zz&Nt<#TcXt3Cuze0Hx z+AbByz>J^Gwo4QN+u~x{?dH-p8p#T#My-~k$fWirmKpj}(a;8yzVmvOVz~8+3p2fzA_*Gy!5=$-+(1y0H zBNCt*aZY6}w+Y|78Wvs0sk!K~N0B+#&GRIdq-6*tJX;t5$&TM4;s)5agDNxF&Y=c? zek}D6WX|t225-#ac16;t3%+pfxc!T;6yvNlNNXY6yzJij-kV>_M>BHE`u7iSsDAgz z+03&GAnYp)98&>=Um=1R3@Mmsn9y&r;O<$l?omKZ<`PKCjYDh^j0jCaC6~zW+f^kR z{C@iysZ^-2E)Z}S{oN(NJplgeMIg^nO=^R}mLqXa;QJ3BJu_+@0DPYVBi>N%AJpvw zqL^*ckO`k5)h=3L(ZRt`+czCn>f%K$DfR=~$?kYCXISkRV`~V!W$>zuniw$EOvRW5gH?s3cC+$JBMn~P)b{jodwIUV_nrS}am!G(13W8ZfI~%tVaR|1=kHV307fVdNac(SNkinjLHN*b2%*a%nfW7MC@;64 zlwuq@Ap31rUx4wAPJ7?%IV{f>f_Mb=7DlW0SbW3EPXR}nu+{VrZuxhbrwNXt;~1-= z)}iwu%H2h%_v!ZjC61vfRs8U{mOk?doyHmj@Ct-d3P376tvn9tsJsd0w=jcX=1Az4 zp?zItaKV!3Jo!_54MQ>$(zgwFBepT42Px&IOVY%CF*RT5fp`#legZ1k3(Q+g4V(z@{hGZE@6l8WZtq#;=?ig8pJIA{;DOIRbv(OiImfd4wGY^6f>s&^IH=l zkb(Qs5_-y}^)mlwDE00Z%J!~#dO_A@rc3{>{KS9V1S`lJjUVNWssxZRgIc=}W%J{) z6X?Q||JvR=dv4p+G_!#dq}r{R$roqg%73N!$H}l+Z~ndHqsBRTBc%S{@onGDZ-vJu zlSdQaY}?Q+KGQAX?*rqsQrkBHoP#oz_A|J|u>8U&ln*`nB>70Izx7sEe-_a`sE)Y5 zbrz8C8xjLk`4%rdK7@XK1Rtb`2K`H9`n%6h{rfJsVZ7pzre;hJ~~p$9O{-mEpNSboNl{eGjQ#q zqjk4N)5IlK-q1yzpq+E4!?9fT)lxNy`)bBQ@^{?<26}!ZXbfZ;>CX)&4h|d)+%I?% zxCls4W-xL>PCgzX0TBr)894TdbCP)JH_}p}Yv!2a8SL=Hxn`h^N8c7>@=F0^2RiPw$N& zs)rF3S(T{|O(}3*Vu$Hmc8A9JIVhLR{K7^Yu zmWqO9r?w%|t|Z?SU@}o4&~w)?A5ElC5os&6JpJy@#`}6xHP^k~XtvsD;ML#naIdV- z;9_~I=pf>6f|zv*czOGX402V@PJBFlhJAZ;FlIxIJj{MR!|PA~drN&ldte>5O$=b_ zYkVV?Nl24rOMr*hRdp!Kv;3qQ7BtP9>y(YhCGV1lUsM=FW;&7=CkHC3Pj?^tn^=tF zxy-6a;m&f*!?U)N#pTaTlBeh|Pu&)M-%L?v)TQ=#lBz;7(sqZNB@U1) zM-nous5worsSJe)4n39L#!MKSPZoBJmd%dkM^E23L$HyVhmjS~po#|f89N}tjx;65 z6D8Em%!r@?$VE0`@nE}9m=aiMz(_HsQ4Y4^6 zzG-oxN@q1T4C=+A20K8?E8v(@NIcVAdSvgj&kP+n)=kcy7n|y^cWsuxPfo;Vy#4W& z3t{xphnHngZzjj!`NJ&Rp^tNq;lwfiRq%S8UX-AG3fBpvpaSJq?}e$@C7Kg4G{j5( z2qJNos9X8qXd6KlT4*1|(0InOBKCozYU$6h#K|yKo@8&NaI~dPI=dCvm%j>uNi}>F& zAkcqHDn(`-^l=m?i51ppP3CN2tXY^RoA(+8na~X2)#x_ll;~;yrJO--LVEDx6gJy> zlPT|J<7fWc^=}>K9#nOq7bBBwC>B^E^+t)kq0T`#>fh&W_rf&xz(EQ>V`l%)ro~eMxXwSuN_&2nX9Y!~e~eA(lXQ2 zZCF(%B(z-ew7LYoSs7zdFtm4SdbxU{*i9B88YXJM1q;J%JiefK;jsg?kVR;r&f|7{2XkTLm4&ESruktP5FTFD>uzct5M6fWEbz`n0EFH z5sY(TT0$^`A-@sEIWx8p)aPGbc1Sn{G{SOaY!a|1N?ev_Jp=G<5GX~*fw9gLZa(oz znzNz3s}FPH1;x(NrU;K0!X0kY4JiT&hA%+L@Dc@qMq{+yDSn)(Dfye{Gqo^p4ja)jS2Hw@+Ir^RR(0%kx99N z(Txb~#4BjO9^=*3%DKoE*ZjeeZD@^D!jTH)w9jEI;nW_*qVcs90ktdT7~7FFQVjLi zT`JGj1}%p${l+V25hjfRHI+Ap;#UiexKCI$e0W6LjbC@I(#b7X0#j!1m$b% z4odI!^aO%izr#EvdJ-)&%k~W0pEV>K!>w0nst_O`HUc9gFV=KzauQW@p9Nd^*#noN^dvmeUR{k z-X0oR4&bz*Qfw+pj|Q{NeU33P+*X^eu0rWso;d}aOl;K+CYKRU)Pu-qr%k{@k_Cih zU?^y*$5;!MXhh$kV%Ov!ZF>v8I%4O49AU4qjOqU-6TC>)*|7(MW0kJ<0u`7Yh9sUP z!gJAa^_H726$pP}Y091_unr`6y1OX(8GYq))K;~(1rfw%%4y-QoRc-78*3&3Y(Ut{ zIbTs9u7tGyIZVQvF}Jhohg$=OtVw2M$MAj^@8W~qaOUcS%53ph$;7Ozob=*C%F9E- zmI>phNbJW^)yjwKaE_nUlN7TpvO}G7OWw}u2}XUKN?|>lN<9nfeD#`poL?;`9p`6b z>e*g-!!bcyws+|zIO)4@t%H5H4t)&jvx-&jA4R|J=E_B}pC=Y+vDe9~wIs@5j5h~) zcf=s6WFsDOZO!oj5x}$X5f|f=p3sV}y=s z-rto0K8bBGkl2&fE|euaqm%tRnJQa$ycLYNI%oh<7=52iSbAXm2vv%|yZznQ*X_F@0iba-#MDi2?=U#- zrpSp%=B_kMFa8c+P1f8GXja3&J3(|p@10dnxrdbKKS10ke)qvcKAs({EGGS>DToE>K-Zb*(f zI*&pO&B2W@An7b!!~zx&Nd!X2lKGRhKxaPn@8~(92j!IsZ7ncG>W3d_p9qHbfzp*MFhRb!R^IQe3j217m^(% zfezATP`Hx)DhvXK8)CV(KI2l5W`%XjlZQrt%fshhGUDYfckHr3+CBUWA`rcDYj;!_PT zGx?Jo!d=bR_z;l+=nZvk-BDoug)zmfFC{X+()!ggX3GNZ>k<{ur|2}$?)a1J&fY}a*o#O#8$q(T7p4C(e{0$oALF^e9QL`~r4$J;>sa6r-lLyW~r zyoCID#Z0S!K)mn}ess=?fEp**5&}7ma+{1}kOInpN*ql_HgUZX8cEi9@g;o-vU`WD za({(yZ>;-d7wGnI0?poX`!_yk z64n8h+1HE^vP8oX9)^$l;;m?yoJ9A#MF5Gch$A(t4J2&}V@jJF|$ zIVX{^V9$%k7>?3|g+RbuYD{))%ov|pOSlxUMmLcT@nVMRL2BR8{PY9?! z*|D$rtng9_@9kx=bruH=O{t~EFKLHL1Ol*V0-qD`**MXo>HVX$qSJr3Z3 zMqv^z<^?al#V)?1{#@gtmkgh>6YQC;O#y?Yg9 zP+eLLgT+g1WYOJmwJn-|RpAf1mHr7sCw%#uc&v}UV^ zF4>M5L9I!FeEn4h%d}W;KJ4&H)rEo@z$JkJC(I37r3{MbXcHl*n=>?ux zS1i+X+dLY|{v8;!Q;5gac>v|*z;0f<<9)MdX+o3E6^<%5?I(fJ@V<~{!OU8{2k1y8j-V9dBrblwjEqq2-Se@cbEf3i`%MP z{(22!n{*%z__Sr7%_A?{s3PC@hJTyKGID)_f_lp ztRqz>7a3zM^#;gHN6g3rpZ$kg1O8x9s=d25I58X?dIWr!?X`Lk8GHVL6-Fu&8t@P( zs@})u!ZRV|`h%j+884B)Xw}>%7TXz%kOdu_EYhBB<@Y9v#wEu0;X>jSj^t?*aoM5Y zrR|(onb_K~>B?xkz@Xj9S)qCq!1VUWzm^b=9S(6J74;vOq=v#p9NbHs&4iG4>RBN<;ejIVKLt*EfNT?{BP6rUVFA!H{>s8jqa9R#g)NOR00=!gzeZ zMTQyAuYRaJmFoTZdJ+gr-bHhj5M@ z@o1JdwOn;a71A1q@itrJRE=cbkBhyWoYG-yVy<)&Z)_AJM3HVcV&U<%Fr;zQVK&EC zRjqvVPgZ?%e-dBZmmYTTB&!Rivv(XXH-4z(<8+mKT{QgKkh{+{j0%>kW?&X6Hv-n> zT{4-N=1r*bh?%7A)Np}f@b!dr)qeSu6y-vc+h`|Kh^5}B*ykwrC>5=$F>$ytEDyO$ zLEESOjiRc^8QE?g&$a5w0dS3!eI-9d%cNd*gg&evo#i6>m#X_5rz&wc$7mK>CT;H~ zra7@T#mR~VJaxyx{3`Dq*2(NC#&}ZJ@|ObMPAzuQGVLgrGe)V<^a|bl_Pk6Y+jd*! zQz3e8yYHp_aqGW%`>n+wY|b~a#X9V08lsik5K^ONtkH0QQh=afN#PyExGC?WMHbi+ z^W11~c*Y~c#9O>Gd@t6FaYNGh72O4$o9;VR670v*Ta^F@9_I|#h1E|Zj1MqeVAGLB3pR7{8Em%r~! z?iK#74>A4_6I=~7{)~V?DT*hl&Te?XC>?cupP2vWBAa=)slnwnCdCekhJLc*(K3vK zA&OtP$u4JQXyf?n#NWYjg*O1B@Pq{r4=QKs^4rfxds{bFKcA#vf^!}}0I%C;V>`iM zq4i&IEtPe;U-uqvQ`F#eC-X)KE>7WVk06v^GkwwUmlimAqM+7_Xa$(zOdDm5W}PZd z-_mYu#B~F~y!CmRxIDa#7fD7D?|`Z~vEKj!s6$=}*noV1 zz7Xudk$WgrfMyp#z{aYL%R-wqi=pO@`1C2y3RYX?2oh~upH3Qbt&qXIRPPuTtl8A) z>KFG~(9|H~215`@b<8uLI_ej5))sNDXd%WV?l3^KdK@uu5<_r?30HTy;|POiLIM>k zqTssPdPgd?LoH&<32d^P`veYnyR-&}hm5q>5nfHFn2{wqKo&P>>E#YCt zx-kRc>g6uZp`JI^00!kheF0jLqpZn7&}k?UB-^}O_`O7BU}vw~6v4l8JJwwT!nDPK zGO9ppAF&k#b7<>8KSZL(x!^enkMjbYk!O;Kyuyo^{k})!da@|x+6BEH)fa`00Fx(E z978`QCn_nqf4MRLPe+^B8{^N?{w02vOn>?VuN{&#xwiL`ohT4X z^Gv{u{YifP<-=pXJ)Cw_GvH3YaotSBplgtzjb6-3$K&*q)u#-_DfUbJR_Qj0Ua3nr zG(D6-tHEaDGSlU#`ATpWl*k&WIA&cO=MOX7@ll*iUaeaHWr$#?-@7iGFH-zwamDxd z$=pij&8myvufc`kcNX0rosrQp#=0~v1^w=y-5)cs15N-ccM%T(bf%KvXPue0|I49O z7C+oJR*tzf&xbKJ?l4t6n9|sWAsu>vi$qjrx#g}3t%M11=QW4k=1fPBw^!&2Xnr>LsAUi?j|_>f?yu6Utm>QLAZvgg3~d6mYP$IBH?5ej4V z-7Xx)NXUOUl8GexJdt#ee+*U#_LG7XdBamGNS4D#Gc zM!k2?{tI?G$Dvcm?;1fMQ6~LR?3C4yQTTGd?kS3cBZ0cqC|vRyomR$nbf7A} zjYYM^L-#-H^O71##6tNs+p+|eDe05{NDLC%#(iB-BlQ2W*nvE6Q#5hS~{Mw)h}C5_cff6ChTKI$11U_^_R37veo&D z*Nm;7s0OF);JdX|C&!KUl%~KY*H~;Li}_2AfvWXDŽ?HrLyN0bTYv{xLq0acAHI z>92~#oBmB%Vj}v^?Wf%vviTlG87;i(vsIpW^biRNTvx7~f8Ysjc8i2fi^3hn%*1iM ztEFF6ol$qM(RxZ`@3m%+@)zqXcrDcm*NbK_QxHt4lgX*pM4I*mw=0+uzvMp2Lig*- zB%ui56xLFv^go4^Xfvy;x4=h{c+DY4#?Rmgkq;ILy9f*2-jzcCI*ZANlje1u?RSi@> zpD1Zku($?uNDQC61gT@Mf1c#S`UDY-TNoof4h@USq^;yq6j$-Q2W==ovH~Wr>gGfj z`<*)wq#5^yjFb;)RL0fU6{|S^9H^@`_P5DSTseeFPu;4dhGHVF1oXJ|Z9Q^_!3hNQ zdsWqcLo`aK(PSjQ^!7@7)+|?r+ud)!s_<`m+%}6D632d4&6a>m1F2+;@#;%QH6lDU z97Ccwje=780O5~KKo%OV($2<;$Miy_=pK#Whs{76JFBH^eWl7&+d>5}R3hV8T!JF~ zGJH&078+QG{>aJO5kXrLs>_fkQEroeCcxw~1YcZF*e>76b6)RihoQzVOKXO!mR#0X z!L5-yR^9%>C7NnAV_#L6OPVQmDMnA?9E2Tl8lnofmhVHN7C*FE3UD_qaf5nv< z@MQ{90oWV+4dbeZGF3@ipJ8x zVb;JzW3t#?o9biuBCTJz+NZHz*J$!_b;P`Q+hdS`|LK`n+Jyz?G}h3;zH;gt|IC$9 z%`DqKjqeU7QVmS9s`(aiYY5qR_uc5;Q2IZ_y|RP4Dp{Wz2`rY{YNWb%|}-Ph3e5 zmo@e5*C(2GETPmAQ9tTR?s(74d^tJjy@~)@YbIYU*!5H3Y4v`>^Ry$ih8}V!po6;E z0_*1^o+&rE{JZ2MAPg*L&`gEg7<&EZbB>ky&x;yXdLMvp@k<|5TT^FUY`-*gFn_rS z>!Q#j%0stsMBS_z5b`vjDYIs&L0EJQu;eHYIE?zv>H$^%P7g&)r-j()OOs~Rit^@V zOy1~ojpy=P3+Cs#+&bCb$_QPRJ)b`P2UFsG6H0%RcmNZ6wi!tOsF|yebPr$ZF;Id< z%@4Mwdp*Kh_J(p=DiHYH7-9I}4`wJIBiJTBU^Tx?DzxWGGXl#APE+92r+j}?vGY~6wvY(*S$4N%JKctc$OS51(MPYI#p-1AV5l0^QN4inV|78;bqA|uIiH8HMz{ zP2H?i&%Nvq%EH{x;)*(WHNn(-zbG(#VFtSpRCIJ@QjQTB5^6W+n|&VwGym8D)}Rb* zM1|0s91pw@;WVUUhT(j@%aPY z5T_PRG?rpCgOl@@Bm~Y=byH%cLR>_Sf3;&symJlzXKUIM|L_N3a+9*S`>NuxTTmtu zMRAf0;LSq+6m>7(dSWqU+RVmODB@KnIx#V&M8y|eg;*9RJ-`lKDbv!9->;z1f!u}s zbcrCcRp`)&AV}3u>)vngCrrW6a%#>B86&6+*_JsmgvC)OQj|djoge-vI13=XMBX={ zC{4s!1UDo>Gls*2)O*FrR{}^vxhn#jRrL5d$e5*x&j6aZHASTv5Ud;S?Y+u$HlrrlFuRaY-@)qo0wEQ6tvf2fp9GmEUujV#g^ zQp%PUau`6TnuyYZlN#?+hf0(J;IQdc#vfOlEcAz?6K3~_2`hGyNt2KmBXpa0cV0~y zAxrAf5AU{!RFIFSwI`TN$$zlRo>2Cb=4aRp&ABKEkq=QR3`{wWra0Dh%(PEbB8yj3 z7S?cP$XR7^_aW&}0GI-1Py@v!8W4Qzkz4k$Bu6Ry^QGx)r9yY4yK4!%V+X{F5w;TX zlBWhN`+4&$p$91CROD;Mj2J*s$O>wHg)xm{q{lF0@o0ATQ*lLe6^=s_$s$QeWT}=3 ztC$noRC4Y9CDT#UsZ%nW`rX>(a*gu58J{x5dFil}9Of$u;U@~uk8%(lODcU3lPAi> zQ%VTzQ=QGwwtfY15^)gj3!t#O?S!zJU^=lDxeO z%nFoHe1Iw2lsWM}A;@QnJt7$?EMmWakrMr|8;jMiH7h#sb5l&cqf04sQYw`_F=U4G zfCf@c&9joG)ezbfMt94+2SXW2CFW^F`9TYtD>J$2Q~LRR`DIeO2wk;J48^M@q2;q_ z{sfxxc~RtxSVxsW*h(OkY--~_DyZS9;BBfnNh(XuIL*LFwTN;LUTV)X)VVbhuT;G` zl2t;^q`I!t@`Gcf0uF;UADe9K zs#Wp}_aAFsQE`4z>J-Ow>k0x2n7fvoOh`in`SObl44nj>@U%qX(4KDi@kcP&%ey8uyY*R2AYJLt1a0xM0X@ue$2o#hE5o zNGnjA{6Nb;Hj6+{O8JQKnoyGqmy@C!AG@f#qFDr5kXY&lUzwP@3sTSFs4DALR67Nh zkpvE^)v=X8R%i+yLUmTdaf$st+?k3Vn#6XyyigeI3(#u8(X`eC_@$W@S*+UA3Vvmq<=$?Iv^;C{^#3-!2i%G$z+l+bN^QNda@ z8%v+;0(c)0Nb5c8DH~5w5o_5xy|+Jgu~r!7vbMDKh>qw*x%or+bj+$ZyZBe`mY2P( zHI*%OgNUf2@%c^Bh%s{#e05<#l=fij+CoI_PRn#Xk_INt+p0rz-2xO8 zVMjmw;YP<$7SXY19mdTwWHx@PZ2rj@kx9=G#sY@j3)}m5mGhwzli?oveMO3P9{Q!X zdfYPF&UMEkRK{SOhJ%vxl_#3DL>#$2+nqU)$4sbwV#{exrb^GoBVlNB!^W1+GvF24 zvnkE6q~iyx$Tj51b*}kR?CK3HS@|C0^_JvT?;e~CIhhRii4Hl=~0)1+jGK*}q|#4i84S;+C6Pv)6n=8t7?aK2iwbB1Uc zz{_kkN+d37W8?Ub3Cwu||L3po&_y@sE2dQDpL)W_lBW5Z2Uwv9P`_rlEF@>fvNBG^ z?NG<_eP^<$P#+#RV;W05x9L69_;?aAi050BvktKoyBa1!DHp=Z%88eJ47%nY zUOQ?^-2Mn6%2sN;C>B%YA_z*lMOPxGuu@DkWH7u&JUn}AY?ysT=KC(QE&~RJ{a-+R z#6}{q<_-H%Xe}4LtH+)mD@-u}EyxBb-g9iClb^IoIPrBnBnoOlGg;}Xny~>AQ!7A( z&8p9oMh#-WhGM6gwky!CBEs%|pQT<@iiqIg-;ooF@nbIaJwRPniPVV~>WqSJx~ycA z`5H-*$c?miYCbgzpsPmx-1O)x>FYw?){3GZF1rSqhyFcFqbA_ly`dWvULlkc^kXb!Z>Jqxo{t&3^9LC zL#zz7;Nwz>eG|}#Lka0HAGYz=>hL3ibF<(iQK=-*-E=ks4H_l3y%EJ)m6Cz1-N zgTkIkU~}6+J-8#(G}L17g+6$g=l1KYbT^UpY{N!GBeKdqcZY}iv{Re^h$Cm)YP56n z`Hs3edqmG7&3d@gIxrv~EI7q(9(oF30H?g>m+x+CJ~~`+o__6?qer@rPkOvV+NGPY zD`)r#rnOpI2elJ?+3tfd*KkFu`iCpR!n?&^O*goUdwPufRH1t)LB)Z;_A>;9m*@5? z>*Iwk5h4PQ_pW7qMJWEMMt}w7ktcP*PZXw2Rgim@1`vfx`;r$)=K=urw545 z(*>OSi*L94K8eVeJiI%4qw6@n=etIr-lbzYIM?*QM-`lNcy%bb%&#ndAZPu2bk{fQ z1)v8rvPsW_@6XSA|Hu?1V#bSS{H}`+CT+uO6U_!y617)2f*NNL*B8bgcN}iyRE7tdzpc z)ZAopU{v6Aa@cZg3Oz+lRb6Gp6aj&am6a7~wY|kP&DG_NqSysiy`Z?z@bKtJR$gZ2 z&=_9&l@6w^?!C6ocA~Oc@%HK>)@E(YFejQGhw^3@s(?OESHI}$YMiM5O2OEuP_QEd zP+-7;WC$JEo zS{YO3Oq!S|I5WmfXR-|(pkb>SRA|a>;i$+PYQz)9b2amsTViel&}!VUP(c)-9u|HX z`V>64XJe#2Wg(vZ8u6^!j&0p`l;g(jfr^jpDk6*jmXgP$PnM1>Ns1BGzJ*!7L>$ZT z;&DhBha#pyvN2>kn?0N6`0-}WP%f}DsyW**(wNF!By7zKI}YX|7>T!%-qsX?^M#`z zVyr?zy>}-RyD+#w-*6kIbz`#b-A@jI_wo2~@n4dtl;1B0F>OMOb8ANIL|aSRR_aT6-4AhfECuajC1uZx>D@W z5mV0;k+13cL-COl8!fD*7x|RYPhywdY`n;NJhZ1{BvX`GYf&-q8%Wm+2Fppgee%+w zFnTQuB)Hx5lzOVDLd#G!aNwYST(yJORm+TUOnm!*m6j0Ja%h0-)U?aF=iu~pF*<{F zWY}wK%2Q7nK_CH^hc8a5G-IGeFN4Ydf=m}j+@cR)r$-N!3^1 z-6{+&u7EdQsr0QHq6`6PDE-j?;~HAk*%5o1E#q#L+OFQtU+e0bcupZ}h8fXb@U-DI z#3*Zf0z{w^1ZAk5-~?m`OrR8`QoOn0?RcwU5*C=&JPxXE6W;(028|~QnfW71Qc@fe zG$0f3VSpchIZA@cFupcaj!Zx+-{yw1IWic_CAIsPn9@|byMQT&R!f*nAh(s)ogynL zieNSfSi4z3Fp2{F#m113f%JIbfmf`?@F*y~Y`DxQ^66j)14o}Ec>!kBgI!Q+wt)b~ zL@G~QPaU>sIU@=YV5Pboy^2OYK?cW%vX~g?Xl1nxMU9AL5ZYdx$HFZg$bYUA78t2< zEErMDlOQmlCPnEBChW-n15!!P7e|?k3F3`zuFP9ix&cB2Ng$0JgiIR;M}_H8kCbKu zW;BGUvUHR~35#2y6;DBdI|{~OGwg>A4MIo-?do3CH2mQsw zDzs^0D4wVYhpn;|1T3ICZBa+A392Y zUj|c|uyM^XZAO!3HsYCwi004G^vC!OGLQ=*8V>jHK9>X$e`&!G{lMuV^mVj2`YRkr zmWZ+o01K1l(84}#K~gHds75*s)VT-TKJsT5qb6R4+v=SvQIQ^SgEfy&?_qO!VJe)^NCUr6dU+8DzJOx2*M5a4dv{6|AQ%?mIip;Ejq0+Jl< z4NQ6Y*GXAnXOPMS=_Wu~E6tUFq0DS{H-pw|?I#-LNk*a01KG(^*EE*BEOWc?z%)>` zoKDPuCd?I&oTApgs _IxzmhG1nWv6WE0$V=rMsy#=W?Xmk5-QeJExVE~Ug!1J zwl0qe@R`s55|bpkqUZxCB1xxtrC?nG{#3-K0gPk4TMVP}Zoen4Yz6iPj$NjNK=oyU zdzt6oEjR{+Oi>^<9GY4m<2SX&G_rO{3tMLjxSwmStw@n;SKZ=pXb(<#QGM%*QL`F88UGj9epTd7vR($3!q5ad&!}7&xQ&dfhFCvp9g07N@65bv|zfv=u6+ zresP^@a!!;Gam1O8GxVC0}rHKfp$kEmd{hTs(>a3Ka% zplS?|Km-mRWJ?9MQeojXU(o3*hL1PX_eic~x|RZnxeI3!!xOuJet=*wrm?e02E6#Z zV}Ggt8?!;vrqCeC$`cZuMhV2rk@$&p%qB95aeMn#HJw?d)lC@v-gg|?YjURv9A)-R z*~)w6?QRh~h7h3Ig(?6zz+Fg!1n5S7a3Lc*UgDQwh@ZW z&BT~E4EG!$_wWwSDR(ARPv#7?A|?qnA}^y`OOtjxdVK93`Q`6 z%4<65Bo|%X?XI*n7WW8U#LYwn9;XpM>PJ%Z5CN(Vc-8~%!h)Y}*2Qg@t;cC$BW%gc z)Bx;;7eQl6*{85d9^1y@`SD&(x1J=Q499pP6+``*?#V{4;E1P^Y&(If2iSSyrD-q! zf6$7^U~OoQHidK{!~Fs9midzDs8K1~3(NUFnqEz1Y7zWf>!%O6s~HIwM?ccu5PA4} zR?Z3yPh8DMYWwtXg>1QB_wTEvXLm=AcXRwG11#R+@hyKlPha-h>QFYK4T^Yzc%Bby z#RGoVz5VDysq!&DD@N=CTY8V(`cbd?SVZ7&)VE*Ys_$cRXdU*~lQY-I^J{>3o#sTa z9oTM2A%LqOljv)$4Vj(J_xWB0Ru8 zHUL5HlP_}cKA~8;czXgBggW5}Br!=pe7QQ)+qygyxIA>Bx`K!ZtDgz0H#Z@>2Qa%K zB*NO`zY5R*B#f@YiX-Y8K%aU--LpHgVK%8Kx$!UuL^Ll25SP9Hu?q~tgF>H0f{r!N z0^|6;$TKvI@I-CEycB!Gc_WpLYQd?i6?~h+GVr%l1I8N^##O_?94v%Jnm;jX!Csjp zGlDqPaK=S!#1Kltx^s<4q^u_Njt9h@Q4~3%VlijnFY<^*%36vpw1xx!Yyuz*!`Ex1 zI5@))%&eurG*`^7CnJJk!^iTI!$_*b-&nt31V&=q!#xDX7_p{gWT|ONFkEX!GdjDG zF+1mi#b2vNOJt|U0;);WlVhVhZ;YJ;d?3;&M{^9tqyR~vE5!iFpQd4hiFgBUoF`bM znef9g&Fer>yR9+2#eL()G6+b4Of_O0#)FIz9kh_vlbQtpfTje10uX?xbV{d`N~Z*q z*kdzUibjg;MBMv7C|E-C;z+NNNeB$NC=^MOWHxl%Cr@KbYM_pFT*=WB0V~^rv@p7O ztiY3$qb+hnoGdUZJ1`3S$BkIOUldBBEJhqe7BV0J10VpaOh!ikpvnR;0jb2y%*;#x zC^P>nNli#Qd#K0^(ga8pOKsGXN~{z`z#SbSI@c`Au>nO+6Ge;4j?S7(ILMO?5(H0( zN$?=Sn(POP7{83V#m9x>uXIQhb%K z=_dBM4|t4}x8$+$*-xA#PO#jgpWGqqE6m+EOk|l(=|q+Vu*}U=$VI43@choGBvH$B z0}q=cF38l7tJz&Vl*tWbs6GQ<>9>g1CIXaEy9)icdbN2ts*?a&g1 zO3h?PQ#3@H%DG!jyqvR=VMtGO>p$7MQ;f`0ck-=0MT|bh(cnQtWb09yIMhzKhghh~ zxlD_G`7J{5O*0ISxV+R-@C;kj$uWzcld7^&+eNeg;FGV3kx`A#bd3=PcvTWzRm$W} z?`%^y6@WxD&!e!pC!!ww&^dFPQ?9H4-0K@ekkM5tR%3k$8>PEHMN7TB&%B(rA<)lf zRSI&61yegxODHHi@U4HTAxIdVZDr7I9X7M_8GfA5AN*7pM3%3KPIhgziFsF86@YmK z(WVqpoGnqS^sKGK3S1S8f2C5QbpR|8LW+Y?CEOZ$T3CivENXaC1X8I8TGkV5R+%^e zAA;6Jm<4LpgJKogptGR?pwv)sx=Uokt-po0Soq z)!9T?Q}5*2Sp8YfT%JG(fqgBUIFgcYd$e2sEfDA-SlNqGrgJA6HCB)0r(7L{2v9uF zMSutx-2|9`(#5&bT>#Tn-O+`#aJx&zD~u`?J{_IdEK(FeHOqQ5RJzL8Ay|NtijKq; z!-FVR-3Z-8puU~c-RxT1w8cj@X`3yQfH=H0C%84Lds#c!6Hnq3cI{iRae-L1%)qUH zRvpo(Oy8)q5EQ+Q=JgxpC>dYHhhW77=^|FC)yDhv+%ixA7f65+6;W61Spha+SS8;C zR$l}F;9H58vMfBTJsYsXF!+6&hY&sa;TIWM2l&mj)r6FlNtKn}Z>^GnkXVYXt)0B#mUM$h zT~`^=TMez*8L1e8`%d#6TzfrW#Ep~es3mP+Vy7j7a>UKYmgt7=Wy2MicLQdfA|}cL-l?sIVsqwbMT`x)C28&sSTg`?Rs(Df=-B*M$5=xm z0|sD9MRImYbKXFKvCMS;4%J}X;mLMqih*SSSk$B!ukyoRRpN~>hmuQ~e-Nr76J&vzmo?7IM3}PuKOQ(7n~+Zbfzbo42^T{ETBocySuZ2J3QWA^k19b}xij*d+x;5dm!0 z`sj2iG8H-vq%!X!>vNsv=^q3d^(ElLRA-tsN~NI~DG%H#_wraS00QV04#)M|{&G|m za|F+F#3QWY4mS9_qBU=23qRblhI5B@(+uzMhX!$|B!F}rarXF!&y`%z?H0#S03)?e z_*!&_-G#*e&|_r&9I<7CN(Vxi?1)THHy>w2`gK5ID(sp@v?Rxn5Xq2;{>~=x7Kw8RzS1 zUDG=capKWY>*nnjWX=Cp0g>h}Lu5x$rb1y}WDArWG zbPr7Pg_6=k9;RXVbex{3w;Xkc+DU<+=p{c>E&KOTg#ZkIfUjo&r5X5GHD6ihZ!n%! zWyf`8SLg#TZLFtFwGGQC+W1zk82i3Xv|js=?`)8l=x9HAV4#lMw2sil=fltE#Mg2H z_}~Bk)$*O|G3(eXCrwZ#m{gTqNil3Xx*&4~$bbcKd(}SUQD6~8|D>gv;na76kQKI` zlJsbe|s|pcDc_ajjwy!4gtQO=w^3zejj6z4^fkcNEPNhR(4w~n;J^VefNMg8fexhk3DGEo$(1Y#ZF&harp%c% zYr+hXF(*F)@(}9OIWcHPj1tQkop|8MoC6A&X3Lg@C(nr;zj6EqvJjmiT#tI?Y7pGP zti}S-p+yS|H7TdshUvx;OxuQS&S(s4m55%BM)vycYvN8{f1KfXL;fNjjfUu&d9X08Z&HnmfDp?GzzygRA$OU|Y?TMHrk<1XxBQerEItqr6T}oMH*Hpn9BON|j0k2tw z8Lc#rRj|ZsS<1F$CC$Nx+G`V}YnCStEKn7i*%_J;0hz`*r*S(i#9YFR>dEJ^KbbOs znuKyUv7fX;$d_K@5rSu(#D_XZp*2-su)+&+GadR@)SUL-4o zwfdVK#tFIl2U8B%Ofjm)vYHfXbO1O}hf@@UVV|gG@dk^x_52O6rSd7Nvl&6#Uc451 zG3`r<`5|`NCz;LJYljutttr>UV!$G=SQdr>862RZ6|Q+aGI{R)O)tD$);rLbyj070 zH6ccTs90#ytR?^iz$xFH)OD<9=b$v~a87_WTXA*?Yi;qr3vT=>o+7XG=x^@^E)9nz zuMA&p0=R(tCc)WFLczLv?e)Jm=jtCMspVW1ctlf1OO*LQP{#fR}f>(OmwCgU-YPB#c?O}N`EcWtq$%ZM zPE3@lW+XXRUFdo(63{}d05u8#ONkLNKvM*m3IvQXO%GuJenLpTBgV07x>27E5eEUY zS*tzi8O4xZ0=N4G;9FM+K`S`nAv9#b3~(3&DA0AsKlsB+g=?H8_lP$79Z+#KyyP&p zcPc0v4SjT6NDeDyoB(VtPapgT=)5LABbF`;3FDCe8s-=|sD;LLg8?E577)o?*z%M* zoQV>7LQI+bW=WYTm~li9G2)f7l*(J8_bj%&P%3c`K-rl%}(i2_HGFvX!jd=Lr8)!v7eGieN#C zEDi`jh*5!Ab+}47>}ZJ2^)fi{@Z=o@$cAFdM|Z3OKylu)%tMOHnZ>C`2!`iR%E>fm z**u!gVpT=GkOfv=LDSU^#is_KV6AHygfu#+_jr8oRUlS0p zeZ(`K2XiT2#4}2*N(^N}n@z8ThAXsj;c=90pZg5j2tRJieS$2-+*HAeM2LVVP8z@> zRip)NK@y`6(4-SIswIy~mQ0Cbj0V8uqrs4Lnz)>!*HHSNRIZejo~r;>ofS)<5XzQ{ ze9hh}QxwU81*l~RNM3>ZDqoU-wS>4rDU=!?riRX`z?BJAt9ssH4aldygghaOy5 zQ<_pxJPE&TEM_qSEV~H_%Xr2Rp0GSS!4#E&z#qIr zj#Pdew;xMo04Zz5Xr(eyokCWD7+qIM9>ZkXfm9e8n4N6SX+ct&)Ret5*h)FaQU+`} zA>w;!eCcW==i0SfEV4+(5L?DuF^WUj{X=Mh$lXDtA*tYfkDZ!|z3!A(6X%85c@Hh* z_`VlHj81G90nLv*z*(JW&97MpyoW=fpaX2}07obkg`r`DjXNRj6$s#08_QG0HEy-4 z?b%PSBB}^$RjXQSjeT?=yTL ztis>XvBzA*S(sZGV2 z6%$gj(N5%}>id9pMw-A@& zR5g)uIzf5G1rV2Ka9W0NapQS{WHo*PXS(DE69*8fr+R-d0EV;(tmg`^$Acp>d&T5v z%=3Z6_YmmidxduY54{(BP55UmM@AW@VP@tH{Pk%HVFJ6xbKRg#(>HqLq#zJi6mLdX z>!c7sHxD~858oGI;U|Yh*Ap{^EHxN?vlb4h;sEgHhsc%z91w`h01ASKc3hVMXQzLQ z$YP?9b^+l#jJS6H2W0}DyV|F$Ofa56Byx(0l04Sh*x%Na|*jBMNPF8UQO zQ*c>EOL+jmD1!&Um-4uWZgqMBag|tEB))Vud8PnqK^Y2Bgz`ZG-lJSdR1PBte8Ic@O|!58!>pDu)#=oFrZS(yGflK9D=57wXm$!d66O$(A~HFabJ`k9tmsbi<4KPrMT!JFO~ zlmzgbjDm|u+JZhx5llKjgxFbwL;z9AoQet(F8L2vDo9xxOcA6Xyt1re3X53iUgdbE zXqu)4)0X2`G`DF)iFI>&D09(~X_9jgg5p(?`KNwbqyBlL?h2@WC7_t*e2mHeoPHUP zE(91oB#P)!VTie>Pi-nNx1$F&`t|CMc>#CoDN44?#u2*}SY}8ZEr&iQ=sA8yo z;=zXg%BT@(kqgVPXq$=#tF~>Mwr}gUCmO1w8nv?9$caC6VF39`h6(vo$oda7+ZI6*I8Jd~5dp3Dvz;bGq!8i%a?K;2M2m8u z;!o9jgnVmzlJm72L4&FLQu8FWgV(NAYqju7ySR&*`Gh)XrF_m;YxDTHU`uOZ+fP#4 zugNQP2_T7ftFe}fQx}`J&YPmtOT7*&Z4YU^a2vhTd$9GHw9&P<{P4GcdxAxJi;O~W zXfeKYwX!W)8H}rkteasVF|(9w3e*BY`1UXXq^ujnvo~Rf(KEWFduZNTI^j#4tb$+0 zXJ&<^VC!1Dv%9Ca%WBZ`9rLNd#mjX2x~OE^s6ofPkGj1B%eE!#y(z51b9+E4%&@EK z!kYTPNjtt^dMQK8n}TVJMq0RSd2?Blxb_mi5F>q)qbRtFzowA?d(qMqK3g45lSnJl z5S^PrCDk7We7*-)km;$w101fSWV8k`Pkw@GQ#-*HOu@RV#(lNL(kF&Bi)Tz%0yboY9No7RyCQ>RkeBumoU_lYJPPblVoCJHLLkrsP0;3rz}lJ6+S=8!iB$b!l@gH5r!9&5YN|>&(pDylo4Tr^wO=yUjIi&4ZlHgzcd) z9N0E3&W-nYgyz%OifCoL!>wc{p&}0%!3HI^!M(=+5kU;BN^MJx`_cGp0Qwx&W<$y> z5jEE_*Q#94Zb8LL6_oAO9UJu307s?^y?sUGhI`#t_ten4trPG%%xH|f)N<+GuFP;AsJt-?h~=VbIbv;S^rsPaD!2?O;I8;T%pn&iu}()+xPp0VD|=!E`ksyp2Lu&E*XCMTDzDF)vv zDBr&=PfI>XQG>6Kc_zlG$xw<#0NyA4U|m^0%C8bZqZNQC2ELD0j=s9eN?hh9xMgYH zn`=JbAMU5&r{Qs~ogZSKn#z2bnr;=&H+&(7j{&CJ$r$1gtUgdA;!KHeN?yVy*k zCOYHgj_BKNsUtwcBP8gDBpIaJb_uBe&I$8UyxfLlX5R`xs0PZOs4m1Zs0~#P;8*_Y z1O7HCX-``BQU%#)F~VF6&Up@AjV1-`wY}%9h$2`^;upU0$zA7oe&>Il=hB`L)UEN< z-toYk@_wbl82H!is-x)c?c5EbF)!&d&#<@IY!`3oow-ctm8S9D=^vykylnKXlJ8(R zV99LP{;mvb#!&ozCU;t;B~cd(P8nZ^le|hO<3KEo8#lfF9uWQzxSiW1e-YQ_!=5Vhgpc!wFR`|^&U6TLK*w=SQRP;6F{Oq7KMDusdHJ%fyQtQ6D=2xT`5=j@+eFP0rJuLT5Pu31 zKk>+w=4hYxZV&hU`SG})`@G+uInm}K|LH0I_ITg>fdBW@uF*Ds^Cu0?hTr^yp35!} zZTD2%2pE1X1Nr!ozMZr7sa9iPkP3&2o@2pz$JvtqdsxGJk9|V$%z^NORf%qKnb7xA zjy7sX;QFyG&_F;Sh(JZ-5t&pjnaB_T8kI^z)oP#$u!gYR?bk|}j>%>78Olg3)`+$n z&X&jNbNjokzC##%KVAD zPA)M~(3F(*lM%5`_HVZTc7YZ(H&vH+*A`dSLU>r%*I2KUE_pdYli9gB^R$;5Vx;66 zJ6oH(+uQrPK%h)0nt43xvRoP(8ieE>1dMCGKE!Pzq9T7{-@hN=zX`5j85j`2Ktgc~ z$2DZQuz-n(5Df;fc+sMP3>7w3sE85dfQb@GHZWn*WJ!`E7e;UiaKSd2Ge<HC zog)M=B-7JpLnsbUHZXeN;z6Yul{O8TbRvNeO{iA2Y6J}wduR??@iIV6R!TUcrd%Z)o2v8%&y=GrRk#?P?DkM9z+ zT!@%tK8ZQM^(?sma~{L0E%ZL&lXm|4E3j4!+|QS3@WX8yxp*lBLI(LL1{Vn#jPOAFG(@mJ z#}ef1!Vp&(krhqif#`=}kwmhrpM>JBw;LszF}Nzl1FA{%*4wg)u*?7hi!9k|IRjm^P(}|WRK!T_Tg`|RU9+pj zOa(H{GNhWS(Z*1J1B%KXC-@Pgp}s2;$5k0p64qF+A=61Hmm+VxTg&4v3N5+(^$0J? z2$P-w0O(>Y_nx4&*7?kWuF+^6m6qCRt*w@@Y%!D+L2e<%)=@a`^p@Pz{8W_0a@F-D z2}|vPmLE-9jFD5+K2k{3d_^6{DO2w~2+QS)bd{x7WnJ>%aK@YuN}qCl*h-MhYmy3J zEoScwFnZ;py)c;d?o4OTfDGJ{uRVE9lvPfd7j#!9^ktJ-e)-WfW&XCa$aLOi-JU6} z)Lk(DR+QI8evVT5Hxz$}Nn{+d@^j}u#Bk+r%Q z*fW&%YU5?iAlB`VcMF;BpOU#5Zus;WC^(65WFnF1vA(|$bC>3*7KgW7FR+L_Q_3;Ga(*C5Wd!s&kE*? z*ZEkqKh446U-#2tPmYDFr~&YFyt3b@1dzXN(91Hl3Yposf-AD^N-RsXM zZN2+~u;dn$_i;jl)a#(l#<)Q=ESsCd_LByp{S|kYH_+VHC z8Pc$5>Qf5)cKDEn{4W4MbYjC?M;1elV{6h_2Y_z436Op5Oz=UZc*Z2aDYmN&Uoh4f zfENXe6l;MGykzXo0IfE15QQd-WgTA`s5>IBmatUeECsi&#vyK(+X`mkycG=pHIC4O zF6`qLEQdMt#VB2F$f5hF`9zS2&Lf)yRU^IgNCM=_-Lty{*h?UBq1A+ z2Czp85p0{BO?b#NF>^NSAxNa8NJpAPaz2Sxo3x!e*|~%=e5Hz3e2+Yp$SX3Ou%J2R zrA~P&xMRwZc{mm73Ww^;zajKPvg~Qg_$f?KDASL9RKpCN36qkBQ<_dGBt`|G(H?P> zlf;uxz{G(kq7L&mCk0zpq~eujdL=kwS?3h1c)(G*GpbwglE^-q6)zh9QdMvAXHKa% z(81z!WpYI3VV4TfHvaUWhIOodAiLPmD6|M$XsBy8dy9w$GMb>MD2=$f)tMt3mAWyKN5x7mACRidG-!!$3-tB3UPf7qPBYP;wVf))H;1WQYVKy9$esB2l(EHtmLd8lX6 z*WrN7G8--!z(-_@Nw0bJAKH9iL!cW96VNnF^rdaFXncjqJ#-8I`>mpl#q?Vllkcv6 z6$)U<;>rg*7|BQWX;YEg+!i}oyitC#A}VX)DPz|{N2Smo@Zf~EQWcQ18PN!=ae@+< z8O>)FaZ}1$Vt%34R4QKaS+3j$*21U0Zyn#Bt*~DakW!R#&9f+j+}bZbDajL7G=mwv z&njCv(MpbV9gS(>MQ5<7e1K}uIveH=%OuS#IDx21J!(@YV8lvsvx(pQKQYT$twJ_3 zmT7(G`MTA{4WtK;cRlF2S`41VD)dWzJ&$24v2L@zbfYER?1Cy*Gj*o5w5dI9Cr>xB z*4Fm6XBlA#xbTJZ5CNa;v4+p0FVtu*^}0=c0vFt?&2C2ju|HrfT(Qox6WqQ}eC77z zVy9@cx`iigxx(rDiU8sFO;IY%!0-7qXxXuFT*R48as5Dh%NBR`#6@+-`QeywnItwMABq($l(HAoWJ}M+GqJDL-&+|AwJ=1&&4EsZHnr z-g)sreoqXRD|^IztzEnqmug>K%2~%c)Lpi9C|fWAdIUEJylaJRl>HhiXHm;xo^H0Q zS*tb2j=t**p)B>lE;wk2mphbJ>8-+i9*LohXja2DlC?~1MY!b5Z&~2D` zvo1(`=><~e)SsU11cTVFw-b&bjuLpVUwtVo{fJGW+j4zxa0cWi5 zY|HqjAONSYs8r_q9&id3>I~I#iGhSFjAkurTWG z#`zpDgD?)#BykHb(Qqp4mgE2f&(ICg@8#M~6hDy^-_FAT$rDqr1!?Z?R&A3sVg>;y z)|!O=^sp_SjPUwM*MLw6bB%obuM$MbFAmWci?0`Z?KQY)FBUNjqY$7VQK=$O`ErI5 ziy`^CPz$pW6Bz(u+)e~bF$2Fb6~S>F$B_;HfgIg!{mSWn-0vb3AkOSit%z(25KK?X zV*e1azI4kHjIb~Cv5%Nh8X4`Nn6DB4E%EXyF$)KB8o8wt&oC5A4fP^&B1f?zEfOQs zu^iKk{8o+}S22}d5K0*76|o7{T2Z!~5F4k=1>EHakuj=>5y0vp@Fu|*eGedqZX5pz znRe1A_pll_qan3%C5uw}C}|F|ZzGpdBb$;bpVBGQtOU*xCZG}yE$VZKs3F`jBjQmO zJ&I}~Xl{mLAzMKv4UYIU%eHD#2ukk%Y|HS5kD(H-0OxYgK2NiJ&L9~JvKSH}5hNk4 z(Ju!xy2KGGGZHZ=GBH0~s|(Z15MeC*OS>k>Bq=~DP8r*gJV(Rwm832G-b5jTg@H;EGK9`Z0BlQ@Ud zIE&K_sd698j}+k$UqHv^&P%d$kyk~HTb5Rq_;o`JIvP}~sEJkQfS z#f>l*tRSuMFW-~#;8QzkjV6#26v44E^V2c+lRx#-K1okGAz(5GX(YumlQ45SO;R53 zu(Z;l&LSlf67VHwj2C$grh<^Z{BIE5suP5=CF>zQ2QV&2lrBs3E?Kh*nlU#ks6CBx zK3DWNlgvf8&@jQ_KcTWlZ&XK*lSijAGOH3CL9+GCCNmYOqh7}+Mk3iTaiqF)nOcej z0wMf7VLZ>1HqjGH(UWrj_VOUvqA%AIOvCg=>GK0P8_TZ8I8M z6h>ndILDMpGqpr=G)+HsP0KV?K{Ywsv=6LOH|CUyq*FYob06*$Pw&ul#sND(GxK1i z5Gi5MMD*lfvo(EHzotm+`al?j<_?nZQLpq-msLc?6hAC=`J7cgyL4LH!c$3=Dg6^$ zNp)MjQN2#lBjGevZ|{FJGaq0zRT{2fcB59)U`D%;r0699i4VtuZ#5OJSM9Q))Z^Ep zl*`HvM(kl(%hOT+Bh^t~6ECgcT`#q=%xYmd^lv&G%S(la&xzvoHRa2vOvI0{ftMwJG^tQHS6FIp^KRaD#%_C!q-Uwc+Idp2i>aJTTb2n2CU2O&_EHdz7JN}0ATY*K$2} zdH?fU)edZ>=3|O9AVhbiE@>?D5DHvrQfsz*xdnCoC}~j?E$^0guc9sa)d>AoM49$& zPjvX?_F$WqaiOn%4R>*mi)uYIe;;&Xl^1}USAaiuA6|kDH#cnKgL(nNdSex&&i2J{ zuzx4;mvDD!?>56SSYNfYc7@=4fi_WZ18Lnib$J(OH7i=Bc7lypgb9R*Pc@e<9b}kn(<4n2gKig)N|rqi}|6n2m4PjYZZ}N!ATlv4^+!3A3tB z|Dk22S6#7$K9JZMNkfDib!Xw%XUo@h)59ifH-rNrd~?@?1=oZb)r*}LjG?iGt=7R_ zSd(Ky4c{1)`;%MqvvPCz+<2G_?l?yF7$I`bY{!I8m{FFKc$Q1VhCtMObNSz>xDpsS zOk%kX1eSNR*pTH{er@-XD^Hxhn3>0TVw1VWR;P`pIh3mz4NX;ZpAruFSb|UEhc`-y znFW|@d6`*~n1{L8czF^)Sg$~#eJ8nuCz+B_E`|{Slbtz}>p8hZ6?v`MpZ_^jO<9Nk z-?S9zcnrw--N-f|-0@=qaGYt`dn1}$2)T)ImOBf%E^~~X^=b%tZDxN}o{JVzLlmPk zx{>i2rPDa2R~j30w4astr2(3n12}W#cpSI6(7IRsaLyn0xDVqaSIfDXr|*5yIf}J( zd^Q?HNuY&llYU8KgyA`!L;9)pnSwhRtF8Jz08Bu$zh*+Fv7xKK8h`~lqEOjEXL%C1 zvTTooFIvej5ZbK)dHO84q@_460ggP0t|}}P_!tU2k@240K$s($q6?X>VfUo-d6e4#&FE= z|CIWz_@ou0TChP{o_!aw@1su~`?hx*n*XB>V*0X!8?1%91jHI#QJJ)p`!8fexr17< z2>}cn+gvQzn2VWw*OGu1?)T`K6$EyJu~aT8y1K!enRGjIo}01FTczJ(;;0R>fqS^) zo4C;wn{9dpak{zldy+OqPgEJbD^QSIEU6xusxSIi^)|Ee%xD`yw%ymHZHAce0n zkhPk(2ZMMYH-AM{zB_!ro$~wGbf873Zn~9Su2r&Jz+7qTHE2m|6I>q zIK%zCe`)>Jb3NTlkq@*S-Nbb%0Cm{^bU>Ip1W;4C@4YSYec$_CD){~1|NULx`w`&v zJ<_4O()ggLF%)HFZ4%(g{@%}?<{|D1F>iN+ zp417Oq`?4aMY?E>-j@03oh{zfbJnPNJnB1Mj71*gC0+12e(I|pPp)3;v!3xo51Y9> z-osw?@!5Nqg+U-uM0;?XN)n=;5J({g*1FQ4JRE8$RRf7Vtp$qAT5x zhTwfK-h7?E$gQ2}3!l;te(+_T@DbYkhnf}?g@4JVoMoT(+rReDtleLpDM7OL>%aGh zU-MHB%$!jd^tZXgqt!*yuu5K^yuWvB$uqv_ev2k%S|1eXL<`;*Mt%sYP2ZQqe_!94K`wlG-^_#J{y75#Wro|wsP(2^{csg^X{o5OTa8z zH9gmMx+p~?+_+vCIwS)TquQK#>C}5?#Fo;DNbJj{4;VaSh91zpuI^UD6C3Zf>v`>3|HH2BT3>*Dx^v^+y<4rseky6Y z#kldK#^eP-BKN5AOWiUZ>+EG~-BFx6$CESX=zTj#EqMYrF0VMI@bu@`n{TiFJ^c9W z=ew^@e`e)a`8j(A?Vmq@{|U&SXrmc8Re!!iFYu`~4Ikkpmi8 zB$7uKWx;_>N@ZGB-8cviYq8Z<&siQ?x#fhm#d9H;yoI?BMP^ke-*s+ecgA;ofY)Yt zBZUZ2bZ4g75=3w+$DN*g3}h#eK1PCOk2@L){|ccqDJtlpjRyLjkV7V!Af=NkX(^_c z4pSg$PFgk8g4_f!<%94@$R(;?raD_$WTNG&hZxodD5J%B2jgD}-9^O~Va(wu5p%w! z5Uzgy`6#lSEIMPd%@%s2v(ZK-?GTb`nklx|8lx?@v@qEzxK$N(ny6S}nQE)`xVkQe zV~WX}Hv7aYs=V;N8>WW$>I9sg)!F(>Kr>^8t z+bF|~Y7A_{8FT#cvmgr>X{6m|d$P$XFU6^o2U-JaR=EMdAF$)%){~#L%px1Bg5dr|{hJ-TfTisBox4hA-g?qD`3*Ysj z;FM&6PPifcY*@g56b^aCT3qqAriA|y5r6BG->&v#1qA#m0ZmLo)i83z_fau_g{dMI zOTnaN{Alq_1{glI%RcCm>13*sQ6=%N*^ z;fau#gD9ntggWN2mHk6lh;GG1C(aU;PBcixsNl*g{ttn^+~4wksY5&xQ-Pz{r4x|J zy<#%cl?|LkGTZ1(PwG*a|75%-CqKE9Lk_Z#-t6Xo68So3WDpk|H0LZ5Fn|wo11Ro% zCp_aRPj|Y203)y`KI^H^OiHM8s~lzy4G4&5LK6aO8c6bh_(gvbPCMGU6)qolG_mQ( zV-zhNMn`wi&~bE?5Bw%LLkiN&h;y7TLnlgIz|MBgv!yN#BscY=3TS~&dJ*%}bChS% zoidaru!LO1jw7RvBCDraBdX^Ph*Tg6RH;Wz>cuvA&Z(Z@oFc8%NVBRs3z8-SU9Bou zsT$U?s&l0;J!>mu3RAVRb!&?nNLpe8Y|rroPy)zpr; zjy6w=ZR{XUOH41!D_`GE%BrC; zsmrdyPHoT^4)5h!DvdKN@;(u~-z8jMb(+zlItI56{_j&C%+V$E2f6d5Z+tan;p1+Y z!zktOaG~MVa2eOb!S&!ao~vT$ww1l%buoGmEFALEc)Q-c@ji2mV;S=pKCBXlj&}@W z^A`5CL_RWv|B*aoBPZFmul=xnFN|R+XZSOgW!p)S5i0)9Hx&|g>=DMhI2#+d%!}>u zlAEk#HM7~hUw(6g2WsLDPno_@o->~BtmizNIK;%2N{Kz(9~85+$}48^J&XKaGg|~5 zLVh%)(adN`cbCa-u5_E>9An02`qC8rRGq1u6go>V%Bj0DxUww4Ee{}dp6zsFCoOAA zbK28!hP9?6jO$zHdZ`OAw5Wr<=MEE_&&58ru}|RV3XGW9UqvDZgJPT&DkdRxp%#3OS>Y}?gsW`QSBsE$CuUGif*r+jP7x#S>22F zcfJE||LcK=uFCJ;478J-@P#Y9;U8YMNgdw7C>;aQYUS>})1C2+f4k%S_Bg;pUT%Vq zT;%^QIY5M+EkYujkl$0}p95_SdnZZbCPz8Q`^|BZ&lbHr7dX&wF7)StI@ywyIKz{U z^rb63;Sd*b%!dtDX;<47AD{KkbFTHPZ@ueT|2n3o#|3HeBl$nbXYtcgNL8^+%0bP7jvD_ zdar!iGf#QV`+f7U?mX!4E_#eoyX|hLJL)g z|LUI2ydNg>1`i;`6Cd%$FFx`O-*{)2CgD_vuFf&fc_K@H`qsbvOrQ$TW)88J@0lj| z?+45XH`RLK?>_no3NS2A5&U_KvoL=z|Bjvi{L}=Xc~gJ-{hti(o+1j((ZVWN#O>K0 zN^D;PxgRspp90=r8a*H=MM4^369raaHA#d8VpABIkNSB+9A%&u`H?oQ9~t$L393ph7Mei*qF8Y(c4HQ65$4$e-M zUlP9GfkB`Qw%;60Aq0+~Y&~EIW)lorQEXj87H;7dD#1cw;ReQF3yvQ&nIQ|V|KTys zQ3LLhD3IYAs$mO46&=oD6yo0jCWZc)9{*Jk%>AF1QJhmbU>{N7<`9n@vJn_w(**(q z9^PU6Jfa;o;utX^2u5HUMq(Z|;GGZ?0yd%jAciNFqUP`#0cqd-;YcWoq9?jwKsDhi z#$ROY)>sik!u8+|D%~zJ+z#3z^TphsJ)X2ZVJmLZBEFy|dR`L9p(L(g2NH@a&LSF0 zV-iv$Eq0+bc2x9XUjce!H-h8ni6h7epHTGSANt`n_+Rr4B5Ek&F}`6yF<>%wqB8NG z0e<7yEm`mNV?O@l#w3WTvDK6no-UHwL2d@zH5=>zV3d?y?ycKD9o6+!|0FYpBROWI zqFE$B>f5$yBuJVY6fl5Oa8^2=q;oN3AXbCiiHbhHWJkiJKN?0v%A`%kBu?H|)g>fP z@?eGiB2f0^p1obms3b$u+np8TMRMd)HYHO&W%XelVnHQUI%P;+WmRS+R&FI#q7^}+ zBv=9kI|5ouYDG?-C0EWQTCQbUvZY(X*G__^PYNYE4rPKAWiT2gQexE*b_3?MWmIaV zU;?H`N+n?)=3pi!VlE~it(ZBgqg)EbLh7aK4Fjt&rdnd=TY_0l_T6S`W@h4K)jehn z(xsjiA66lxG7y(hju=B$rVy^vHYg@-2BvM+CT?ovU*;xfS|xA#|0ZbuX1~Q{WGaPf z-diiAT3)^;Y^KyhekODVXJ`UvbXKQyTIV(qCuG*8UEU&hB1I7{r+AJhF+Qho-X?mU zr+QLnNMeU!u4nCCCq{}SeBOyyn&ft(Br6DGYc^YRCdg%Cr+f@q!N^+T}jc#a(=IB&1 zrjzohW=`po?q-{*=vU?^mkMA@rC<87sAteu9f25O(?>2GSOm7+s` zeY9zp0x81HyQp5~`q9UZs}ms1B;AqN=J&s;Oe7p&n|cW|g%pDw9&GrP^w% z-m0$V>UAnYOG(~Uy{ViErLdk%U!E$mt}3DOs)r`4pQb9bI%~APXXCu;M%2DWvI&D!`iQJOOL473**5;-#o-y|OFBKCHI^{|A{#tdv?T#zri)j!eGttH&0J zzYc817A$p!sGf=}H;ydI#wVs~>ZK^G%Z`-8(ksoXtj*r*otUh>-Yn17EY4~y(E98j zAgslG?7pT`t&*(Lu4~UOt<$b-d9-ZI4lAcpEl6Q#&^B$>Zf)6BY@-F4)`l(Dj;+{2 zTE`mg)V5v77G~K(E!*y_%@VBJ&TXic-Pw9-X?o|r&Me&aE!qBU+fo|50xsPOt>FeP z;r4^q=A0VbE!sY=b2ch{7VgvXMA*u0?H*j!QW@!% zi(guA_0BIxuCDyvFa74P^%XDU=GgY$?zu9mOX@HF>ZR>Gj z*Kp8M>;c=b4tFpQpB4s#TnMku1CKC(e(w&qaC;sx3okKr@~@7-@cI(5%rdDGSFr_$ z84g>q7WeQ5XRzf3G0K20?uzit88P*4|1owtF~*XwyfU#Hp9B=gFd19z@=`GuqcIO- zFCO3V9`mtag0U3OmjIh3iRP~fzi|#D@*(%}H?j)9F@v=+5{Ii0#4&@4@{#N=C)YBgU9v56a4utV^%7PC zgK{W?lNe_PE$?zGQ{Uj8vN9X9GQ)Bd%P^5lFfTK6ZvOBzUo$m#@*V?oFrzIgpYbxA zZz3P_B8M|Muiv9^Gc;p{G-vZJW3L`#vpl==JZrPrsq-9v?ij=BE)=sp)3fiYaqFV; zIRmsgSBo>t@j4$vJJ+*8YhFA*|MWjQG)3ca5c9Jv7qR4e@j)BlICnHiOEf{>T|z(e zLK8zni*!ZnGA^&QO20HmgWE+9F_3nd+YGQvr?AAP^7w{yCc`wzmb8kUbmZc6P6M^F zM)XlHHBvWqddzh4j&V2ls~J0W!G84p#+p_8w7N-cMsr3=vouqWwbv>2R-ZLl(=Rd; zUoa33gKl#-&%!o(CoyfubaUHtDt6&w7iUYLx}oc| zE^qp@Mu`EOI;tN)s>|kMzj>&qGGD_bqZg8+@3}lnI|Rur#l6M`#Ko_w^MtjpSrM5^0+&6r`v9Zw_CFp`m2K@|4w?qGk2l8C%t$3 zuw#0;z^}dk|9Yzzyu|NSxi|LbhKi=UyI8|#i|4w8=et%~`oq(E!;6Z^mpr{U638EQ zvTt^Pu);I4c%gS)7#dZxck$uoSt+k7wgZN=01$pd{3fBfo-xzVR5ru)3p zpFGc>ywn@O(u?iBueN~vJE>N^n@>D)@0G|uz1Z)3&zn5ebLGL?{LmX~hjYDlw|kJW z`^L|FB}YBfOFiC~ect1}H52;WyS>jgZp)*1NtS@t)BTj!ysHcN*pI#5KR(p=J>pw9 z$6afX8*9-|e&WMDcga2A3uemSz1{CU{?9TlM%WyJO3I_~&f%-+#xA ze)j`GKsX!@ibrGeNN^#XP3SW^l~$=&YZklJcDdigGdu>Dy=1gmoo27k?{*w6r;F`% z{QW-S_x=A8K*2!5LBf;662wKtM#e|S5*MWaH#keoOU+EqP0voyET&Nx(lyi4)X>$> zR@PV7SlC(F+AD*jT-{yXUR?v*VA^5fV&Y@tWaL%DX6NU^XlQ9d0!rmP>}754ZM9WY z|L^W`@pAHP^KZ)B^>$u&c%t<4dHQ<$dsaPkxu$9Ae*hs3j8u@(BqjOqEmYW0)k90D z!ZD=i&|*c5r+jVX7;o2`iy1|R99c47K!MYo5j5z?S`mH?W6mU5)8tK?GIio4_wpAK zpzDHqM3R%{(Q-)XDK)B(=~I*`r3zx%G7`ceO?hg)C=nt^uM@d$B|BED(2i-~P>;X{17^H<=RU|x>rVbtH6>OdjvH^z z6jqaG&7nDu#)NC_Y1E2EuO{8vF=e{2{ot*{+P3T4f?exg$lJG5!-x+XcDH-k|8L{S zb1Tmx+wMxMF0UnD9v5^M&wi^@=WZ6Q+V7~3Y|mZ3JH=74)jOwso;>qV+C9mCU*A1^ zkOznh55IqiYWo7buQmLlaE>y|BpGi&21&XOi|ou3aKe&O<4-)c8q|K|*6Pni zCi~lEMhTBJa!M-F1Mx~~M5J;{Vv2;wOD@BdGLscqG}8by&CG1f8^v@pM>u2J@i}T5 z!E69F^Q@vsBEQn}&o>bmPP`ye}9ZVj$bkO4}={?xkQFjc3I{r(3MySJZYZ!<;P0Ic!ZXDw%596 zoh4Q1MEz}b9d+wP`sqat4jStFjHX)W#coavmyw&s`sy&@#YE}_|G(C(PqV=$Ix*pX zL9WT&26){pNlaOW{$Wb(xexqch)xb-&tSh1Bl(C(ulEjn+f4{sc=!PE5H zh_??=z;YQ2&*_`U)u#GN#w8~!X33+b~op38LwN8w8Xo<~2SJZpMKS#9nSP2so`F#&|1a|6vLPc)}2p5E+RZn<4OL zJ`H)SO7x2%`Cx&HG`&q}BpVd4hJqaB~^KN4(WG8NW@b(FA&-DIXL|7s~mUj{Rs>kQ+~+V?&&vQwVw zr02O%sX=e{jD|5mW+rQnV?oJ12PP7|m?r^rlD*&K&EA zQfLM>r$RL)H(yE-mmZZ1RxBqJa3@EgnpCQZY~fU`3e*;AF{)d=s!Ao;RAVqDsh_(| zS27B@P3nn09KEJP-Kx$fd9JI@FQARj&eVbjhzN(<9 zb**&$Yg_=o?{C{9aC-lnO9LOcp0t%MYk`T)C8iD?_SG%}x$88x#y7sLnZ%b?HJ)4T*h;^ezczJstyf)F z_Y#M?90IjP1i4gT=Qq*7UM6rw-9R{Rn!KmRDW;YEZ1D~X%+RK`L!Di10S=qguMBl= ze=Up*FI&XSrn9WG++JI!`?-8Jx29>F>vpRe)o-KDW;b?d-FEia-)8f(W{hnt-J%;R)w>yl<>-kbB&h=k9pPMILg= zlDr5=-iy7pO>AM~yDx|qlHeMC^M)rF=R4D{&u)O{OW37 z8okAF^fz*C<8n7Htxv4=lYi8zkOq9=p>BA(C%)WsKYZ6i{v?(UBdOm8SCj?W#v!#i z^TytMD)tQQF_W_GGNR0DaGU z;M=nIpA7wYQ{VQ&JAUNLNH_hJ8Q~0 z)u(T8>ZiZ)&quxbnf~|bUt9U}Z#>_(&wlsgKmX~cfB$bC1OOrV1O-q4E&wb500000 z1ONg62LHf-a7Zi~kI1BQ$!t2G(5Q4uty+}>L-tVVdNbcJcq}fH&*n6G&2GEdn{gaG z&(!bsJbth5^Z$N}#>dE&$;-@?Oa{=dZ`0J(Ziv`@irI19 z-rwNie&Xci=Hut-=sY>i@-c#RMxPgM~5J_Ncu?f;aInmmE#hDyO=T|1}MzIgLuI< zc+lK<0%EOk9&37VNx|K_8cs`Cx>P05nkHXwfqE3F?S}Yr+uN(1Th=29`!??3 zIH@Kjew?>+-xV9m<_G<`_3B1cYwx?=uJ`WX!*d@m{=0d@+Rv+BU!HyX_wLJ+pD(|D z`1tVi<2Rp`!{QGev>@Ps1QuxEfe0p8;0psX=wO2oJ}6;?6i#U2g$`z@;f5S`=>Oq| zAciR7h$NP1;)yME;GYJm)ioFeF2X2dj5N+@BY!pKc!7T(^yuRUi@C_6e;#NMWL+%w z=c5Npa=@aIMmqTsppq`lDS`>fU>z~pk2Nx=8Z%G7@&fTHtOi36-IhtqmxQ%X{DH6s%fU2 zZt7{LpnggrkRHtW=c$Eyi6oOhE{Ua(Q9cRls)_bk~_Z{OW%s_4W=xb*`Wz+vD4aD?UvYHE3C5GW~&{IQ343*xa5u->bat#sQ>P|?6&K! zfF@RmDXF-^`73JndV3?1;kF87y-GqkE0zSaO6sn%((3EJ`F5-Cna@TnZITinYFD1l zR-CF~QI=|I#u|fpX8<9WOsK>9nyj+5*2WlaxiH6^Ai5T&YctI^=Zy2tI$KIAlb3SU zE5Il}tlD9W`Fn7UOcHFP({=T_BGEDuys)a<8VWJ8E3XVC*IkEQ=$-=a3bxpMMVocS zMQ_~p+iS0UZNpl|$m60i=iM{Ue821W-+%}1;g3{);NgI`?n?K*N23g`!Tu)ha<4U7 zZTXN+{>rZe_X1n=)@X-JLFQW133}*muX%Cmf`&da>aed3ckCEv?f>PF8|baO-hA)v zd+)&iF8sVyI_aOJOW$bpmZg8kqSBP-SR~y8U(K-cvs)ZD?QDar`s#Ipf41D?cj-Os zYE#SY$(}F1eD>SN&pLeVPwoNUyXPPL{(|@KzuyfyT&L(|HN#oxUsO9)=Nz}A$R+G? zxVoN|5{54czAZ=ZUC^tPij#V;Wx`xdXsM6Y*VacbV^iqa!`tUoh-Rm#&w$UmRfYD zJQ;XA1L=~N!2Bg1_jyMkHAzazWD=G9ctBlQ$&!#G+%&gI#7kPRfq_(FBw@Is;ytpR z7!9WpyGKQhnkiOGBu{@rX+Bk=b61TZf-OC2x|U)RjQ?Y_olNU?zX1ttGx&UzKJ&@b zodPdu^n#V~Fh@{8`fzi&98yAkNKGiZ@r5rvO@Ekb$ps<|O3d=)Gt&r9ma-G7aq_B} zURA?c zwzU#NWVj|-)4t~PuekMTZd>qy4dAa=|H=}-hQ^ox@lZerl;D*5)hd8yQ@FTFM92~g;^ai$OcuG~77G41*Z zTqZLscPghP5ti1R8>K*b&#PhgqPN9?vhJ0P19ilG4Y?SP`?d z)T!PmbJfjoZtsiZOh3&Pj4KG*v7YnGXTL>{!<4M7{funTs|6XpO8cilqq|jiKJ?K# zYVcF(Jm)Q^(tYdn-f2aOV$od_&1X%kng931<`i4H(ymmqs?la2J9n(cyWF#^b9}=N z=+VGCnsD_p`!pjNj<<_VYSo6+B*;zDuAqyoHHQsaRImEfCw-Bqzr1O3zNyom=J1C> zEwNI!+1a84H>KB?QAJvpNCc&Iy!6>;{I=j$*<_}jvwCP|BT8^U#~ zuE1G4ZY)E)SQeJ0kw&difd%OSb3&cBvplwgqpsP{rub&fP1MEMSlu9>^)HV(L`FZ6 zk&QHLvx1Ddf-RV01>g8KV=iusi>kc^Tmw&oI4XgVYx5UNzVmhNrn#Zq=~UoOvZSxXooc&Yu%K;cg#R z&?&C=wa2~Y-3_>ssbnsvi`?&@A~~cvm(9sK>6lQxC}))y-YY5_?HV0*-R(YRDlwe$ zCY!s^A&>Wy7CoLwmvfY<*K4Kz8t_t2ec*!V#9{vzt(FC7)h{gQfg@YW43#W~&)#{= zpRIvKy1Vc5qb$&GGk-t1_C;$7yH*+Q# zBWCX_A6l$5HTZeY{`I&2oU~`Yl@=4k=d1qy_LV(m|9jEeyGM{Gr+eboKEgI>@8^Eu zM@j-W9&NX6mv??^mVOX8f&X5`K=zX;_LqUthb}+kUnQq+mt=s9<6seofJs+vCYXYl zCt;FjerUyWn`CVpRD2Y8fHgRJiw8+cwJG@5gFZMZ(sU)WH+V#7djV*JjOJ(!19wQ+ zXvcShVCFoDR5~98b0IW&5krGesDe)zTFcj1SmJydIEEYOGgbF0i}pqiBUvT5PILz+ z4TyWzhlQfYh051N+=fNww}s@Pg;Yp~Uv+eRxLK8DH8u5vP?v~^sEGI1BQxei6UB8# z$Y^7CZ62l`g-3V-bz%aTgSHoX<2H!3V}(hmexiteDs_sgSb=1xWcX8tutVJCxQ#Vn=>d;i};Z%){RcDRZhMT~Xghr$?$!FYav_;}USbg{UI(pXSLVtT@q zZ*r)2BFH&eMTn;bL!D@L$w+WrXpEl(gXOe`H;9a<*o@|ggG@tT(I|^$m?FfLO^;Pq z8I^}$D30TpjPLkard5v6#*P5VYW_%&%2fyPIYhf3ku*uQ-zvDOdzVg3oA39QiP1_mg$UO&`gV zOsSAW32i7iga1f5EJjI|Y*UoUwv2SS(2#xDV~0T>x?J4lhm1(Suvl-@|+qMpVJAK$JuJF_-ORjd?;! z@-J1QcwAY8!s(bXXGz1!o*ueFx;T*bDUd-rDEO(LL;9i8xqCG#aQ^o+Em|`dc}*(% ze8TgYr7|W%q{>;OI7+6Wh>#)LkpDS}Bsx`BiafM9RkVdf zdMP4%1#c@FrHJXD5Q<@ushVBtr=K%VgDIv>iKen?rpCycKKiH~>5`?{otYvzbgHKX zdZ(UZP(pQvqWPsfDpJVlp=*k$MtZ7x!l-BZn4zkst(skHWTV>nro#qAm||FAI5kS5 zssDKTZUAU?7FdM;7NHUfs_vPbj*6-*b)g!hs>=GT#)_d^Id;Pqrw6iM33D|@5-F5I zh8^&!g9?}*mQnjTs%Z+X=(y!$ z<`%8QI++_IC+#Y*$0~^Fy0B=Oims}t;g+mIdZdRst)6&wcAAJP`iLaR0rwL7-bCGvM9-+6T7Me3$drfV%aF4k~(FxYNP$}TTMn(x(Y|f)nA3P zpp_C-{28llTCs+@vI_a0GV8K58?jSMC+jM;4~wL=DX7;ohe8%Z0%lR%8I?^mO#cFM znf-cOFlsYr>$En@kWa{%ZCJHoinTgtw_|d(k$19MOSi}xj39ZNO?qxB%3x$GNa%KY zMPfWZ!mFGrt<`CR|M!7+MVqA>waPiScnfqe8-9K}ftUL^T1BE`o0DKTTZLsj1I4y$ z+f%`Hu%h*Bhss!W+qYl&xtm*@x%(2GdvonqtOu*IB5AQ(*`&1xNtIc8NR*`=_oQS9 zvew45p{sD-HN6N+uDi=onw!0P%Zk|xywpjp=sBsmcDRV8plWMFXB&rT8oWWacyzXq z@4C6ny0G1=mbNRsm`k-Od%eWFv&%M`vI<{8HZ`nbYclz*2W77M>$c?=X#WZ;xk>xI z6wE#MyQ~bHt__^D|Eox;l63}qu`}|Bwvx6^w^tY78S1#f!CO+D*23UBzZE>g7c9Ki zn`(Xgw>KNCI`n+qDThz}4%C z_R3p%Szp@Om*A6!qU*pf#A_=hw={dhxXZ@rx?-(q$d1f=g}lS&mSCAWL$sD688F5$ zggG4+$Bb9G=ZJNUT&|})$!zS#a-3za+M$PefY%4LgZog4@*h95f&VJ9FHWSo^7bT< z>m7vr%5&H``#ZBvoXRaLu_~K+y&K8Qtjec}%+|?t#RSF(qNM~pB;y2MN_?*B2Q{rM z%|^PysLIN3yk6uCOFHY3a2u($T(&COuX;vRuE(87Hm}6oJPQM_$NbM~%#3~7vXIP- z@w?6_+rf&AOD^N4bZX7mY#|CPJ+^X_gePg(t5(B2&RHAKSR2j6h|cM(&JYX3-kX~7 zEWodqWcn(lCahc73NZIZVfLKIm)j+z2+)*l(jMKIqHJ+I&ASeZ&_s=4#Cn~}wlG8$ zLlJEt7ByTzLZHD1u40(jy(z9sR^}mA|;ig8$_E%dYlZQG>R1*;DiN ztu5zr8rXPQ#D&>&UpPMLju@(&9ZK}D}L?QD-hbEJ=&rD0V`nID}c(%yk%Uy+FM)BBrDdI zOqxT)e@Z=6l!{~DT8%5sw>oXg5C^Dqti$HIWhopFG4k1@P1>g2+==$wK9bsBec8TO z*hG2Qh)vBO9En4gufv6Ixzp4LqMg-NYd-P!#t zt?k;cZN=!_-`Wj=b!KR`t*?nIkABqwKNg%L8LoBg$^Xom*RTwx)J-P-UO5zhs^g7G8jlM4qDY{D1;?;~L=Se7@m;{<%nQcV}+chu&Jq zedYVT&;b09_XF8)Zc_-3$i*7yIQ;2rg6CZR-v65p=0Gm%efZ3cD#`xs&!U7Jj6)*=A zu<}|G4bgB34D{4Vf=wPvAYJj|6CK3a9@_!0@_RA?c`z4P;_dRy+&o_LK~HMt?z^-; z+y~FJK#v{@|F-^1UNe6X(Y_u;HZZ-M$^RvM;CBg$axL{yd+k#%=^D20ub$~WzT8Ov z=UC0KgWmL7Jti$53SbRSY$fy2FdI#q>_nEhJreIU(!kF`6>lI)cvRxkz5C9g@yOPR5xr(Wcj$3+akxZrDNZ$B; zzt4zD^pgMfUe4*3Pxn?8@!Hz6gP#6$TT8&jvtKRB+l>v{v z>S@^V%HK+n?^}WV`@=8yDKOfV|NWak@C3h-$1memEWvv2(FfB>z}#mU7{K!QqE^T;6`oWOcgSX1~vH`F!oR)9ZMB ze9hPY^8oJ#2@A_6GB!3k7%MI`J|ZP2Cs!;lF)t}MBUcOq1wTVYNlQ)pKui5j-3$U= zUs+{mX=`n7adUNdaS;W|5^qmoM+ZlxZiz!$plWT&PDWQ%_4fC3%`T zlov&!V_X3|x|C^CTq$FwivOUpmT${~|CILiD|l(v!Ke@iLZm3-;bDfcs?_RbWfu-mqnQd56L5RN-B zWGo%IdY0~aLrxFXjhDH-oH=*z-SgR8#O}fh*|aUill?*OclY`L1AjkXrt_y!?RQ_C zNICE982@X@kK4Am;)1TcnodE`TBz=tD*jvPy#VP0FER=(6vs83bXpHR^Va)dy9(=L zjhEj7B2h&Z^D6{7lmAXK2t)sxvjV!zK3Gu4XfohnGqPq}(L^DMY(u`sGAJ(8lMG|^DeHY5%9?lhvTCoi|p1X04Rjg9SF&TwpUoNY_Y3@fE;};$7IAF&{y9Q$`Tf ztc4uir3Sll6RwqHgzwmIot*ULiC=yt37B4#OJp^wf`qBom}Aw>^0rE4Lk-@qd^wEK@VO9-uK}(H(9XM$8sc9*vU6s_O%SC{ht94 zCD{G<3nnyp?qkCF^!ou#>=1|)P+0r_P>)1e_`#X%)5(Euz;h5dZ!!xnL=fBHDy30a7W7Y;CbG|bH1 zI0dQ|9#M*A+#l391TZ3&uStt@NfMVBM-AoB3}yaY1d`A}Vp24@yE!E)p0VCSlh`<%y${ax`XOUI|a1H~AV@Z-4f$6%Sgi3QQF=jE}Im&^QvX0A~ zW+ZJE%_vc5i-zeUGibRZ2=Q>3_N39|?j)M>pfjC^>smnzF{5Kn6n=C>CMkPox;Igi ziy7Hv1?dzoaR zbxGVjVAirtU;sNn#J@GM0SYR?gbE9Ix^+=Wg)6LPWEd_XR&0`j>Mgf0y{K|v1J^K? zh=}uuvxbngZ@p`kR5li{4z4z;st08avR2_*l&y#T>^uqjOyy$1ct^Dm9y5&jmG@S@mjGlteldXOH`}F(s$AC1TF=qH6z^ zEF=rR%m^8>>KotN8W+5~3}0g_@e|2Pb|*4>qj1$q+ztxZJ$OB&L;N^k4O6bAn4+6> zyWzsrPEVx^!tQprnzQb1*th}yAsQJgMDvDdtmuXCL|F{phIp33D#jcRf8kC^n~@h(3fNRytC;~Z)^H12`r1ewZ3$wy!`*`N4YH3*9~Sx zy*lkouQp9F-E~+F+uTFmw_f`?@M@%szxxHQ+Ut&Q=w|d24reUXJqh=zCz_RG5P=9l zaCx-wIPr`R4$mgSqRM!2(;xwrkwv@=rFA~Or!pa&WIp+LstEHSu>I{b-v+urdZP-L zH{MG++d%*y@qZWEKa*i}({tqtti==~gkOAX|Gw|Vdy5}oPW@a&UGg)peC=_6dl8g@ z^Q=dn_RT$@0Hfdd8GBIa>HMh-jMqn%%%SI3o!S}&jTxPLM#oHWC;GsE*lPrO*l{n;YtK=ZD3d}j;!uIFwX^%Q2Y`w{}d04 zB#8kprUvZ}SZwP9>E}QeA!IrY1W^!|*v}Lki07>E=Sm^shOoV^P6d%L2{SJSeGsiG zY|6e+K-5L>#3CE4;qcxNm5z=7Nbn5fNJR*64P7wvmT(WLDK~m=#kfsW{08)VY7DjG zGVBm6dXV3u4-wA~0=ZCg&}{^x#0&wk)dG+ZlQ0CbP7|x|^T-emRpUT#kP39L8fNi; zszDn)6p`^ej*u9O@feFM84=F+zz!RUEn{La17YLC%#qsMOJ$r2O1jY-C(J!m zP#g)79P=?5o3XN%2MSjXG0toOJJDdEV;Vu_)YQ=nAMhd1NDLFw925W_H>e^thabl= z6*W;J{YMhJ1to4|2Wyd*P;v)POz+~cZ2D*$LlGkbAOu3d_DFynS405_kRu0B1pHC% zSdSp@P7+HlAoDHTlI%c8%w2v&zplu8Kyn$s1BmMJ7tatdLSPtIPz3(LiFA?>4e%#h z@);}d3Zt?cfU zEh5vAD8r8L&e93b?)jz>do&WC7Srn@(;QrH@7yEuCU3K9QY-ZmC?{ke!|(RQGA>(@ z(q_=(Y{b7(k``Ul7EKaKO!IzT^6C0dGym^>Ml<|Epf7jx_DB&T3o{W9Q}KinDoK(O z!{}2WWHu@DG!5c1Apd}u zCdtuuu(Cc4&^Le6JHInSB{Mm1A~sL*L*M@ryAE{vU?mLwFed#kK9w+(R8&5DQ%JZo z{O%J!_wzXYP9Zt5KczDGTnME+ML=(nbab>pp>ag{>@AROI&pIV{&E-tbK+X`HxKap zTvRZ*3n7PeN11RVV9_LB(>$4zrHJ&nvh);sQQh7yE1Q%BpOj#tv^$BUK4&!BZtGT>;?xv70W(gk^c#J! zObwL@Wh)sv^haUyM^`dIHFQ+Y^fozYQVa1Xt&>uD(^7wPPPwW^4O8T1kSxJ-fL5zK zg_Kk`6*D!?98I(;5!DvD(RNhT1&{v`gIaJWcN12-(@INq)@by!%%ToK^=_IWPb@ITnO*JbkiE%flGyvsOR^f_P3GyHiQ`RJs&1gvZpn+qY@{w@0SbIoIy0u6J z$?4Q)Ap2rTU%?d3urmvBG}ARS>k~u0l}_;!6IaV)vXwcxRZt_gPzTei=5J47B97Eh zFUb|NZqHRKltf}wMx|9zsLw`=GhSnsUxkP8rbh|xW@OazUjgzcK^9B3KwvdhO477h zSFqxcl|pl`LRJ>{AlA`*?q>m(M*YqGE|v$Qpt^W=W6SaZ=1&pgVql9zGn3XcBedd_ zmIN9W9OblSyY>oO>`r0!Mg{-U#iCN|3Uchc6lYnABlM7FWmaklvsU>uCWIDnalv2* zz+8=%1yS*K{^3n`lLVOdVcoW{5SQMr@xqAZH9WK_*)d}?muo3dU>TH1mMA{8a#ep) za?v#n+ZI!=#AoA{cA;=;Id)$D)7vhV7R^ZmP^wpl_E7zEbmyctLD4EBk0w{p6#p{w zmKFRwlVMS_I4JaKdu(oJO=IyhzsU1*tD_^tba+WO8?`Zc>$2jQ*Cw48_k8jd%=J2V z^8l*177l2EZk@tb!pakQW zC!ZHEXHqXUm{QfYerf-~0j~FQt5t$8gudD}Z^!dG7)d1oY75KP$A(pK|042m&v{YS z_FPv!*LF9W7WeWuaKB?X33uV*D$QhgpLq0hrsq?|jR(0`hUqd6Z}nFvWLO1wMTSa6 zO%VewI4_k}hmRN+dYCjtcymcOXiRv7Ik!`N0{SZ0&@1mu^(JSaj=&3k(?S7*99xkO_Ha`Jj$L0OIiY zWJv%6L^+g4`G))WS-E43akz{bd5Y(8ky}~puA`EE)MmFf7aj!>5TS~w)so|QB1B*T z1U5TS)P1zL1^54$luh}RNApF6Lum(@ME`d=DCuK;`E3sg3sY*FGe$EY^OyN3fGHVw zTga6&c~CEzIn>Z&C&2c?6&6}lk4IUQ_n4W#_>T!gX|Z>X-{WbFIF^$bpcqLs-cNz! zI308ueQgzpeW)jgd1;l`O+lH9DLRz9*q&XKJD{1B&$)0B^E(jwpUv1TwJA*bibL$I z;l%lhjnsBU@~H?=1kSetVmf!_hp6J&qUqV9ml>lWdN=Mjgk_hXxx)bxdK&~x=(5R2 zn6pP7Ak@fwiMIJOC5D`$F539+?Tl3-NC6HHns2w{v zemi@W*g6zpV`7uHhb6NoV3!yA4a1QHkau;H5V}(XjtJWnotr|7urIzCwhOts?WCzT zTR3sMfEX#IVzVopPm(QDsueopRB;~%P+;AAE4x~vQJat78;n)si}%^Sh2y#bn!2RQ zfPeq=H8e%Fgp9y}p|mYr7s@*U=5hh_@}kw7Fp?-rZ8*YRw8f8Nz8jlR%~>9W+hy50 zuhn?JS#!4)!EbMQ7BJkccRb142Ejo<1l@b9hxEnUyBK@=y;WneE4;*4qb9_eM5jkn zXgN%Gu)nvv$U`u^(VWD!{1(Cy1P~y_`LR-w+!mz#!58*iWt=x@d?v7bN~-W>jXKaH zK+w5d#5EhuYr+98K+(HE(W{8~@F$^b8*|@=zgPHb$9x^28dyD1#OK^J&sCTeH3JfW z4No?(Gt-!-JPA!b&vBSA7Mm-7V|Eo_&>a8)OrQXAeb;&21RB8C8{pT2UD$!$*DL=3 z0_N}7mA%jtowSwQDGJ@1r|%e8tG_+B!w34Cp;)uke9db?E8iTn+qrq=Tu1BN&W&5F zUEP`afKK`O&X?gA?5fZaF`;W#?| z{nO)pR6_lsn>8;f*UU-j0Nf`=Ssj!o-qoE}FqBr-%Q4<-BG^%&0ZxA3iGAgLeb-q& z*I7R1@4e-N-PVm-F1y^zf%o9AJ=5Qhw_S|c!`s`7!HYke=T7!@qnuzS*I7~9<1e%? zr28iopw?@>zO%6cOdbVJKILcr*Lz*p#eVDyKmf}A?8!dtVV>scz21F20Ve+d0c`%^ z$syV!TiDo9%&&IIvOVYxo#>0=T#w$5>T>{<@ump|WffoDp?=S$)~t`w6%4@aDIWnW z|MD;Y035*TPuJl~9_6_nl{;!+YzwzU%Eh*HM4{TfhC?-~HJ?{$XGC-Cpwno?eXFatI!(t^)`- zn+~zqDiS>$Wl^cQP@t0OgIc{}t=jCCn{`yelS-uPJ)_fVGiB{&zvKUM_`GhX-?5}U zKfL*8On?j$77mJBwJGA|F*areL`sGjR$5+SW@>^UIzE0@X@*Ef3KW>4o}#X@wyK&S zC^*b|DjeA2-X2Kc&icv%9~cM|4-+7mG&Gs(3)&1C zxt-sRf&=Gmskfixc5O5%)eXTw6Q*L#8b@Oydi**J*`Pr~l70Ui#(r(`glkk#V5%C_ zDt9hjsf$n&B0=#_1&DqlR`SBO?i`PiLhuo(c(I|9guURkG%C^n12cWPG#Yc}D$JKZ zdrl@aD70kLV4~KnIw?}7M^is5_jvXhc7bT)maFIjzp)(ZiG3oRw}v4^)>0X~@NL1| z&Ywd+V@TrOKoF-lj(*2*07{7&gCE&eY9^H|L3;*`UL5;W?cc*Ma5}!|(I|*UE1KRt z`)Shtk7HVX01`MKD&N4S3@m6Ur=387Sfx&b4q`Q!M)M3;&sfB*#hYvm zg(spo9#?<)wbuq=SaRJVA~G@|VFqybS9pN!n2BS~MP~oVWGfATU}~MImmX&HG1=sM zT*5g%g$k~_>iDRH?Vx4zx zp@1dTRS;w)A>fI|N`_I=qoJZ`SEwbV;8>-T{Xu$~r1G^lX{47@Iw^tn4W%GCnSvVN zmZOqdDwmQzBc`3Jo>Lz|q13*S`HCoFc9w`U(F67=Q%QFd^95tJ|Q4S(|YofNCeP zB*Pq#pQok;h_1;emn-kc^FG7rP*<`Xv&tgN49$JYBtUS*6uMf$&N?S7p}zwMT=c+0 z|BNrM2Wtm((9Zok^d%2N%xocu0ED!xx<--o6w^}Mqt(_@Oq;~p4s))iYO}pI$^r^l z^Jof=Q!~wQ512RHD2tnJ+*l6e^wU)g#E#d4+oiPxL^D32;Bpj1C*qUS_3Rc?`=Npg z7>qsRh!YGHdf@3?jIqWUa~z$=k${Zqxa#U1JKlDKNi&xB;qFZC@=|KE-T2jxHlnEicYy3Zz-#)^1ephxb(Fu?)B23L;wG{Jco1tGp&j{fO)c-s{jEeWq1FUMzheF zbsnXAB=PBtp+DOtXdiIp-@lW3vy}YnE`Xn^o&Vq^JlPG;OGq(Zl0I`P*>sO*uBzAb z6jr_HQEy?@Q%B?~;JgZ^$#NTvAV)B$!751XVHqGF1~NfHr2&KsqGQ@tzQ&!y%&%Ih zi;kiSAqiO|U`x1j84-(U#3M$DG)mN6Gj0YY0n$!o211gfoaD9y*6t}31mP8!m#i&< zLVH|Xg9*zxy)we#an}PQbj0Y0f1R;$mUCl;E;uYjIG}R~P}T-C*TRS7g%cynK(1!U zM!^8Y0XOUpV>+j?`aPy$_(NCkkZAwOOJdSx0o+~7;&s5i9f?20^9fHj<1KfA$piv` z${4*!hA}oxglL>X1{art6dcVRhWsEbXI0Bw`sJs z4T}sw4iB}%=qySltl6e1-bS(mYAJV_#AGMMNxNyRCrJyWB4MVLYGUcXX6nR>|( z_X(4ji-;qxQm_Jl3JjQb>Y$l=Su1|Bg^uu$Bed)|D`3ix6Mjf023puTNWKFMDk>c4 z4haeFtfYsRVAgl4iBZCc6a+&mg=<>cl%z1QZ*{ZT5jR=R|BXs`=wv4;P3g{*eKH#S z3CFjhK>%}9ke>P~s6Dr*&|v>Iv<)j*0Yh(j%QJq$kRAmgR!8{4d_qg0`rMi?sSr_# zCaaRgyAfEKxi4t$ceElO=(Qx$?)Jd8DZQ|(aE=79l#678x{odZET`%By)G@$f| zEomK^wM`r|trg{wCBBLUjb1i#wx|MKSE8dB2`R4=JCUDO8i2q;H@ecTMmJ}A)5z*A zls#cmb=~7P+igjekW!ZeRT)jW$l|T)P%5uni;$ zRw~6L9%VeW!ajtg*KjnpgN-eV^bEHZa z>@$Uc?S^NS5to%ahzdU^NEjwFsi6zg{&<*^kyY_KDM_X3Le^Aswl0@kV&1vX^Uxvu z0z1@;XwYW1$-@7(tqBz;);`C9qrD8G8x&cGx^E5|HQS0EQU# zyNg%F_=xgRFo$=CQ|(Fpw9VBrwbJjr+wWRigHizhGbj!)B9uH@LGIYVoeFHhLM)!^SHBu|8c3P z!yNB=cas0heZIt;X4mK5Okm$&U9(xg3+OFxu~cA;=Xy#!jDmaWfN#t#_+2ax4;T5? zn~p)Kp3vz{Mz$Qv-f>AXp30m*2+2)n@=u3A1e8!z2x?#Z6h-D^X#-c=c<;7vvANB8 zPm&4dChwdQR;T(KVDC&*VpI9ge@|L5?-YoN^D^kEM%VSWA z13KXFtwtp!mX>@HiNVv90M**y-0#d8@lb<&43JVFgc9|Ky-1HOk(fuPL+-pGvCynz$!^bXxT-{Db_k${Q_QdbTB;3FJiMorKTf}hm& zATfy^u|{Qr~6i%TPvLh9?;{%pbiE-g#4M=~)n!foPWwnc$(GGuX zLmD1PHn1TVJk_!h9E6;pdH1!Vnuvo=Otu3jAIbS9b@ndI(}Lx-d!pB(5*5(OeA9n~XG|nz-DnEX{T%YO zN2w{qRU)AYP5^0^X3~XDFn|mNq*nw0Q7=%ySKh)6s>xW80W#p73tS{1g3Q(3CTo_!exF2<8UI}3mJeWBqv{d8exQERz%_^0Ll_BXRs&=qabD}y5s*(p5l=p#dwlu zmYnAl(cF55AP90v8QN&#c_24QmvpM+IfUjfrom{+;hp4yFHEQ~bcF`k0s*Mu_5D<) ztWuLIC`|+cJvHW|H~@_MD3a2pT^5s%xQIoh zD0*dLU6iPY0;VSL3tnuCD8AgG%G^-xhjw{MyU}P-N)h38W{yf~X4WGB04ZjQ(wwQ= z<2fgizNHx0>4^x#08r_St^!m*K>{?zk!0ab$bw`AX^c+FER5+utZA;YsZAijG3aWq z3ZgR!8HTco;OxS&J^->lLX!f+FE}f+;zYDUD^B3)O9n!S(#8MwVIPSaO6pkWIU1%} za;F1c;iG)#dZflWJt}0Tm{M};sf8zFzFI1zn2(yvWe`uP5otsHmM+KveG=yuDCv?$ z;~N|Ryk;Q)G@Ju;!%)eDq{J#bF($2sAm|t)MuLHwYAl;>Y{w?Rny#O)5+e9DYqJKM z6g=xQsjOC|tdr&{0(@-D!mP~3ESuKs&DyNZj=;|HYzg%2&-!f80`1Te?Fb|QN##Yp zD5nUNY|A!jMrrG|VnvVq2yXF9cG}%`PAnEGY6F3%kbWRhLQ$mNt4qzQ@%-GZk;A*S zoVcjrhV3g`2!lXMty``h85L(0gzD1S)4@vRcLD$~yj1_Aeo2Wh*P~d3sd;`q7 zfSE?;F?ejw{%Qgc0OSm7PT~YG(Co|>?atzC=nC!V5^V{b0O@{i>Y{GZo-XRTuIj?> z>+-CO$jHv}i9?V;?DlNu^6t#~g0yb#%{14+kOdv)a>MP?8iPZb@`xJF@ITGV3OCBvY~s@4@a`E82m;By(~nd$Q^Z<|P#E z5XZvunlke~M!0@u_Ku>GaBr2QCrp8Fc}fujQsw%xTa^kQyp~ydcCi}sl;ELL+}iD> zk+FUjTos&XvMP@Pq%l;kuegvQbvCt1SX7F(O#d>I+;lZaE&kG}3?U={o@niHS&jYI z{gxS|P|9mCLn_K~;RdAe>gohn^+j{@Rp)V5BX0z-wO3n3L3A}AH+Np^HFa0_b?Y^D z^Rq;%BtPV}b%VEd-}PH3i<$~F3$!kJ|MhwY_F#W91L!kDL-u^rch810wt;eB*7ttP zNTIeUNCWtnhBg!Xw!0xn@g){5OEFBlG-r~py3(||Wmo_srBVX`P$z3gd8(cbfbDts zlYNTEO^>v-Nd!|&R{k-}jzySazMd7TI~E|b|*!z{0=__6mnpSL*i zzWAae`k^yAv^)E>Q~Up-mpAE3dbU$~l0!gsc;hC6dbsB>lk4!Oi+d+e`6{1MD?6a6 zV|n*R^DLv87k+mA!n$eK`E<#8OjB_HbJ$z5T~?I|1=(J&+jcBPK>5z9bcN@h`}uK0 zJhDqXiyr`?LpwiXe8#V}#v6LHd;GIoJjjcDwQILOU%R%W{G@L?JnM4>-0)xz^trG6 z&+wW)X*L+CuFx>flBCGQ5Rk?tpI`>Y4u$w|>a|!dU^wNL%7vDRX)#4g2 zeA74hDa@ohl*ztW{ja-N(U^sIbNyL+cf#wtvGZ=(!@k+a{=}1gu5MPyyM69U`^e9I zqDMR3Z@k?LzwrNf@&9<~SVj2TYde#htao_*i{2MiPh zlq)o}KurHkJSk~(9ASKvoTRL@bd0>z+~kDx>>R0>lVboO0sTxxT~%#mwH%3+9U-Z$ z1;NeL)t&A2{UzDeJ^d{Xb<sYa6$(}Vk%Bm@}VB0El8<&+Wy3B;( zA_o8WF5YB);ra!(D}*xx(7cT+msoK@bd4E5Cf7}$V|OSTSeDnogl2o?4i5bHS@dXv z|3V*>tgti_jvrmWei-4Rs0gra-xhgt$ZnKE?)2VWr#Hq74kAQY5Ty#6d@_d$h3-@O z^XSx}JGbs8sPWkst9}P>1pN3yEi;}sABTKWsCA5F0{mCLE4lT}ewFW?6bSnB{l^FJ zuf1>ya1y(O$?LxY0Xz_wXP}WrF$wRX5V3I}V+=8J99zws%iOsvGtD@orZZ|t%jYx{ zR|JhA6b}oHA%o6205=;wiet7LKcYZ`lYaD358v8zuRQdmKyo?kmei@IpPZZy$|(P( zjIuhKvO_~VCbtBNOE0?wGmY*>k!q<8{CX=*`_yFfzBb=5uueP&O{GCx1`D9D2q$DTF$^Dt6vhrqE1GTW@4ZIED5#ZEs}ZM7FUO>t5t z2G|id*}5aCRfs||$)q8P{KGdxj5scfB)`P6N?)H`fJ$Ex5%yOyv_v*cUX@+8S!SPI z5`j56kYkbs34P_yJ+;+#P6+(O_S$geGc*@)?JJkBaq%4R!Dt%2rZY_~+^}8^?@i5A zZa%C`#Chbs2GkirCDf!H@@oc=}LAv`@^@_zU9s*Z=7=R!)K(X$TLXb6Y~GZXMbCxd0zAV z_UGpcfdQZ~4Blx-4_h16k$BAiwwAjj6o6mt5Ruh3m9G~O|vi# zJQ2#9&hTkZT`E&^j%}Xk+!qu>DZ6%(il;AKh&d^E7`xu{GXRXhJ~^mZs?6pM2Nmc- zqjyjXC{z(ol`NEcf?3W=RiRL|WmQFs)vH>xa$ZH64a3Sf7DS@8jZ1Bt%4$+`QLZjb z!;2}{I##i)6s~_w5jgGX#MRssY^Iyt*dQ30Z0zrn+K}RNV|q>wl`=+1NGwtPU?c^M z7PQ}WD&KZFUhqyewBPlvRk4&+t+v;_UgBz3QQI?&SOu-T#b`)zyNgBwO~1o9S`ftw zinj&;xcR6nO$S?=n<5ao$#rKIEp*AMRZY4V4(I;@A*^7-;uWz|X^&#L8`;eAgv2M# z>}5+F)y}pU#gIL3jIWno8k3ffrX4VTy`rDW!q&B~JpqtoJLI6 zUdotyOkFW~U+iM#sW^}3Rd1kc%w8M6*U(ppuOH_t7GE}b(M2|EF78{F$<3C@Id*gj z3VceJHVUQ+<8p&9_24vz^UN1UHHpZ)YF3XKr>_nXPAlA3DpRq5O<-&wEH)I*x^c4Q z{WWFroM%2~VY9>@p`Ve>Vmni{*@A}lpuPY1JxsUn$J-L}wZoEaMsPdG+V*z1!9DJ7 zllxB8R2Pvi-HS{E?a8?N^mG-5SAAhQfu-J5s-HV+e$S)T{|0!h1!S)-i@#x)^G zaU?a;8PL*>?6CD5>^)Qbvw2iDdYujNLKFJX9=AfG<=bwOKYA;-jRmDoTXJnp@)=u< z_aQ#TX#yP(Aj+0j8@v)h>hb(q;m)q>$K0DdfUbD5AW!)?{d79oW*Q{|kQUF|C&66(b zVfMMdN+fEuZ}5H$Uso$9%LqFHsz`Htl6cd-kuteeHL@?cN9f_r)Kl@t5BGHGJ~CSzqhs z_TITYC-YwWmF3*mK5V+47|X%@U52Gq^qeAz+*L7lf4owK~aOX`Xe~}Q$b9zu$}Wg09*{a zYd?Cp0_1bP9P}(WnmXq5K^^}D!l)C%2CTCOl)llLKt!uRG|?99<31;RLMSZ1@RPzS zjKV3jLM+56---|_)3;T7H4_xWF(kuyn7_WmJ;O7z{9{2f<30DAom!hY^}9O{*&iY_ zLdi=&Jj^BM<3k0^!y^ntK}5m`EUhK9KnzsGEL=oJ)WSy0s7X42N#s9yfSq?B!%Wn~ zO@yKm!9;yr>S+uMkq{Uk_K1CS99~?xj zW58ZCKs&3xYTLF)bVMygMk-9kWmLvyWJYIPJNq#J2k0)L3&Ts~#B9{Y;~Kmbe8V*a z$LKmb#!IyrTtB;$k$L~4l-99^Kl{Z6+`~VVM?H*3U?jvr3`S{7LPJa+L^Lf8Y(_|Y z#uE5Gfutc9xS#Hd05CK|P}IhUgh&KB#Z~;hSvxw#JEDvXL35nLbELzM%*b67$z7@* ziDS0o!^PrDNzXzA6{xI{9i zL`qKlmyR4K8KkuY<4BqlAPm7fn5s$-v$71)25x{yTMVJ_3c>}Xy1qfld92B^oJpH3 z#I@u{oaClrq(Xx}ker8fq_;IAh!bH0W3fzMn@o3HPSCtOnsm<4 z#7F7m$LeG+w;V>foS`d>B(l@apV3R%q|NcvMiW4YsBFr~$VkNW&HwpE0c_0uX-trm z45hlP`z%XcYEH})OV9Mr%6v!agwE(3P0^%2>;$`;dCk>y%?4f22kpKDfClj-&k7~O zf@{SpLog<4Po@-yB$`ipv$?D!uB4NUDxkBaddK}FP(Y;2i4#y4jnM+7Q3JKk8wIE= zK~QlSJ1hURHV6e$yL3$iC=BCRNPi*0qqI<@v_rzYEA<>q^z6-P2+pevQN7E`xuVL$ zkkPASQT~Kc;``4r9m^Rt(-y7KG(FHA%}KZn(l>R`I1N$>oyKYmx@%n0hpZ6xqb|dm zK@ZJMbp)&uU9(OiR1@`v$2+kvMboN!(KCfqHI-BwrBobkOE&$-G>O9R{83KT)FHi} zRaqxY%u@>$C8wmFR8j~G#myqtTMAM~u~3cHP;Fb=RK#*`RG#lhIg_+lo*1R8AFAAN9^j zx(JdjRe+VjW@-Qgcnr{h&un$sEnV3W-CD0*M_4Qf$W*}O4A6A#Rf|>IiS^mG!cU+z(6j%2 z+sUO|xP@HH<-E+3Tbu+)pt;-a!`C_3TML!iCS`*rA|f&w%RN%U-^J9&G)6> z-$Px(b=`w~UPB#W5>8wfA&8Kf-3Mmjj!9tdZQvM|;2CaV8HN@LK3}+FT9E$@-I3Mc zztn~=m?_oOA|n2@{YBy=rl5RafC1KB0zOxW1yBW6fhu0$rmA5Ge&Q_VVhP6K3MM~j z9NsbZwhaE^+DzCrINAU82dxcZ#+}|chT}PAV!_! zPG)5;%-ZmE~QoiJuYo28>tkSEDse3YFO9tn{h22;UW={WJW^>+NDz;)| zX6AM#=64R|XM^S?beu6>Wui2W%!r*NvO&`Tp_NP=$27OcOnwU_^s7VB4kbX<+n^S0@#v@hfq$WHWQC%y|<(P)(sNUsT z)n$RO>5QJ@oyKaQ=IE~8YCMaolw4v$TFKqRf9h^OvN*+SrY$NRfSOKUoK&~W@xFV zY`~J$gZS#s)@!f+Y|!>>kH#v%KIHasTJ}9`HVntsX6?X3I{&4o*-jg|wry~>YQKhO zk>Rz_)@|(JYu^^`(KPL`M%L_0?F~RmQcUcg+c)Q?uuB5ho)U;jrsK(G-Kmc1?Y8WI z5N6T-ZSWrL@fL5=Chw0f-{LlII7#k@d@#js?&f|$_~yM+)lhQoFt}D?{l4uGl_wM( zZnh2Y0N3K+9`FMvR`upw!9F{Dt^jQQ=P?_{38(OJbn3X8AXLii%GOry*6tI=Z_Y07 z^B(aMA8(Ec@K8o?uyz>*U++eeL~0DUqBFG$xAAe*Qn3G}?b^2FZ|?7{-SLDa@F7R= zB3DimCvp`<3neW$Pa=SX)a1BllSYPx3Pd zrbK6Sul~F^uV5}T+@HcoIIQ$b$0Wwx^5zY5KJV}kf9Mal>?h?17H{)YS9DbWY*lx2 zuzqw%5A02QXiTT|zk5HiK{zo7s_KSm9~bnmwY5|?^hPIdVlQ%5FGN@eJ(v5fLf!0I zhjvRp9Zn}M4F~l;|8g+D_D(1Bgs|gQH+Er1c5(kF_i+wkb^{xHjLJ#+0*Y_bu^nLet%tiP3PCb70>vPN1T8!hPP@dg+~W_a`%-#I1CT+gr4_$N8;PG1}0c@ zkN@{$&-a|)d5-UBJ7aQ^kA*omjGYV2+q6o=935|GS%JatEQMo(-F9xzafJ?;B>?cA z*ZGgfd94@uuV3-bL-!$i29;lHYE}4zU-&!U)5wmi+NoqBj(VAgMF2W_04{d4MQ5+) zcLC@7tyeE*>0UE%ak7UiHA4sgk!_`C{3!q9@rf7VOQySDo^`5E`G3gzu;=>C2m7w~ ze2_QwLe_l20c^t$0K~2Glt+8Uhsu{05~XENO))% zh`5-j$oS|8DcKlVIeBS$Scl0;+4=vO845ZoN@`k)db*j)y6V~r`zqTyQfqsQo2$D! zLJ;sfU|?c$B{ZBQgtc_U-0b|!^d`+_Mh$n39cF~gMHZJ0{d5}? z9uGe+Ur&E;pO3%a&Z_^T8)Yo>Q#wZyCS#L6Q`QKfC&>m2UHuFqm2I>S5->!vE#{-D{r-oxld$*1}S^a4Epos(V;PK3acxl zfz=*ard7N5uWZ1xYlFN?7exczy6wJH}p9oNaBq_FCH7;w!;@_qXeCzzrXlFI#dc@im)& zhtanMcnXTPV1o>b2cd%uZ6@J^6z|k2)s8t_;2L88~Yb_ab#7i!& zxL<+%9fx9J0_Fr0jpZn}V;96Q$AyMMnr2>+Lmr7?l1MITWRv7Q=_DVve2A7>A)d6B zjD$HOS6=wl#oKOR`u6|g65P~iqLw!fR^pi_zR87|SSU$nXB2K&XP$WOsV6_1-PtFf zf=0T@Da zXK1U0zUnG^vBJ9HthUlBE3Ufc3P_^tbqOqU%U~&As3q;DX|l>L+nZix8Vjw7$lVAk zoYo3;mJ(UaGHADfe(PpyQHzYr4DEdcXj|1S4mhW~Lb}u`40VY`)BrIc!b- z)>|N{WsIt7ss;zv<`%rF+fYFco69h|>qguT#JWlh<-!?H%(2D`5MY42?h1umN6%*I zR$I`Rpl{3jcDnx}y|Tp>ufS=1dhN9&{zV_J=NIy+6 z)JC=&@-Kp!93#N?){C#pUUTZ~$^VWz->5NNeQLpKEB9AQ%r((*+;$EvbkPeq9d+Jv zOZ^Ald+Y6S;C~M;I8?s4Yqd8f0|j5oDnH)LTwYIZDcFpQEvj28ehM6$iXTXve=A}O z*V}XOy&VKV3tjiasJ{-o(vUH2XY7Czj(djfA|3!4$Y8#8mckp8Ti29pYI)edMGL9C zq^BVu$p=r5V%yNY4N~qu6WzMsw|}pF(zb)YapB=}T(!uMug`w_Eg64&zPNb`If88h znT_u~hQ9xk$=17V26f18fV5K`-H7JB@hMPz1k6VQ8@Rv-k}n~<3z-45Hjd%BuSM{S z-?I2czvfNH3ynh9v_40@6cTGX*AvR=&UYE^A@G6bo1hIP=#P2S5QaP~Uk`PazL5cJ zgUn#b=2%3t4tg+IAPiR4@Yf>}-6Vhd+aLc{RkPAF4f_fuk>K-^o zGv08GyK~nCWo9oTwkT$CQQ{Jl2b4T^2>@R}pucA-^jzpr*vIE+D{W-J-w!he#_k<* zjcIHoBOkd)RYWO$tZ|$ijY!8AX%a>*$;}coMJYVa<%k3e$8cC!qZ5+wiUbm&7H88% zHJKwD5|qt?nJWqT4dLjdRM;9= zAlYQ2fQ83y!qHRE@yToZaD>^u!{?wgAounHp zNzadtGNel#sZ*b-Ol3WFlp*8^L05UA7B-`bRs@4H_9q;z9*3b8)ny{bDb%!7CR6XK7rf+iseH93RF&t%cvS;>FjWo~wV0C>rZhOz@@HHL62@9O zrKmo9tSxzkI-*uqZklK(Wd}hezT#D+zR{~_`$|=%s`jq-%%osdT2P%~wRDZ5CPKGp zz1!l%u|5rFWYr4XG9t~b!Tqdd2eCT^{ED^1xZG-y$;o3r7rK}`=xqgA(Ci*oRDcxZ z3JoSO?DcH35t89fC2QRD9=EvbC9itpRy9RUSGx3_uYHr+FZ;<>XI{-t_Y^l6%VNbPP8g&<&xkZ)D5_>xT z%?9x41v%n{&iT=R){t+{i{3tK zI>Eiv@}@t%Brj9=%!*btnoHNwGSidEZHDHD*&N$98??l(p0t1vh3iT?>CSk*ldk;? zY(D>5*Z~=Ku#X*MW8c^h2@rH$>-*6j54qG}UbJ-^-BoG+@vx)L_7n&JZd=FyLD|~j z>BBNjY~f;g-Jovwp7$W?YUj6=qxNVB>%G3rd3eqICM05E6KQQq0o({6fViQM>x7eA z#=UmkW)}W#cc(k85dX9GD7n#t#-@7sRw`+oN$OP3yVcmfF3AyVNmGIt;RSE`!Ie?* zsra^a=#DtMIn7>+dD`X@-%uNWVbx<1`neyU-w}zZUnlol;9HPRtpO-r!%_j#?tW^0b${?Q1{r9Uow{Uc&s{zXo!>v)c1r?Ry;X zK3fx}ZLot!eZdQ_N&&6@^+}2S&ToJ7$S>&n+3)_Qsis^9U>-M>s@(7Ks5}h@Z^M9p zHPaCjbu&oQ^iV*({RuzmIedNVo7n&Usc(IgHG9-IfVhW!2B>yzmw3#kfK?L#6A%IA z$4tm$QX`dGtJZlVS1cz7c%b)YdXX@%Bxg}@egXXt_Sw?Z4Ie6WQQjaY)uw|ceMa&?G_qS%RhsE2*# zg1oqg8ik6dSca(>f~r`E;I%4)D1y-^i;PH%k4T5ONR6Rrjk}1Az*ubAm~6VJc%)d0 zWT=6|=!!syjQbW=G?hyKC3TA^i_vI{@py~Xn2Xn_jox?vk6!ePyOxg(B#gyKj_Wvx z{^V_jIDgTEkVCkPx}}ir_>6EUjq~V#lgNLSIFa_qjhu*)8u^JEnUS2uh2R*F07;U^ zXk;J8d@Y4Pog;M*DTnfCi}YBJGwG52*o!ualOO3~KbLVM$$ryTa>?jj1l5MjsEAJB zlFkQ{^Eiz%S(7>ml{pz#Psx#0Nt0L+L<3NgTN#K($#=2VVJQi92-1)Mkd#gskum9% zR2h{w>6Z9-F>e_=JZX|OWtVmNmFQ@Q07aCK79b^4e`ooUY1wi_v3e6p6p2}pi+Pb( z$(E1Vk&$VYkm;C|*^QRD3RxLbKDn89>6Ljolwk?~i)DE~v=*3xNtjK!mX(Q_Z3&mJ zDVI`7QRGIKp9z|v8A#@MO)s@rBq*3l8Ig5xnAF&qtvQ*J>6*!DoXQE3TgaNt22Wba znF48s1gV?XVvM6{kR(4;QEn1;!Jis_&I380E;k>g3A4~12^>ghhp?X=GKPFu#C1mkOn%0z%q|#=!g`a4t zmW|1yUgx6>3Zx~Pph8NdLCSU^nk!l9ndWH!E&gS1y*GX##Udmpl<8PS^X8CCn3|9X zqDD$tCJLq}8m2>)qUXqgP6{kVM}FuBGYYkx^cg)olVoP;k{$Yp|B0sos;52brF~kY z3ks;~grtGmpo98$>?t~y^rV0FdzFJ97bu^vR--ifWTK}HNv4E$x|Ch&r($ZJW9q4( z+CycErkT`9O~t5x1*yF`qels+b5;abLR(w#km0$fx0zj~m(dYMpSr&_9?^2ndLimR45uL4T{ufU40*-EIuS`_)}uM9Y-@#O#tz+5ppj;PeE z=&G=!5*=nyn$t>nnwoIddac+xq}+ZO3oEiB8xqF>vG-T46e+JO z>!bGSdKL?_|0=T@OS2bSv$+GEyn}`byO)&evm ziPLXXOSe~Rx3-pM2)C*8c(iSMwuiK|92>Z^$$&rCJ9mq?i5s-HRkRTqwqQ%I^17>I zOSzTnw`O~}`>MH^ySc>5oy0I&}TqS=F zDY5zoxsR*6jQOLMo4UT+xxXvC!Arbg7q^EyyUDA(gNT&DiMqf$ysay}uKTxn1G~$M zy|HT?Qn01*n!9=$iMuf+O91>ioB;g2;_JW*tiB%H!7jYEGW@_}JHaK)zXoS`=NcFkE59)e z!uAWpLQKTo$~!omy(MV>wT|nx0?fi0+`ozu$VptvwH(QC%C(dXzV}+ks~pR}%*n6Z$-%5*plr9JOsx?M z$+T$8_vft15ypf}D~jyO)ZEIeE6m_A%g@@(-R#Xwn16A+!e6VwSq#kQ?90R)N$G6P zm1)d3!i3)}&-0xBdM7rVxCq7Qe9qe_IM$raf~(J2vCXGR&joGJg8ax`tIL%v#_X)l z0S(X(EzuAiqU>AC@Q9xnZFNp~(5M-1$V|!{jmrF5%_V)!CatlC+tDk{(#xE}3&TQ7qRez1hS~qzx9ADb1du&D=sw+I)T5Mfu$vK64yxn&I+T;DxzP*&uExGLN-op$P zuwCB+?%IdV&zSw+1|GrU-O;=4;O70@=zY3J+uoMxE&Ckb8UA9|N8re{-1uG4Wi8%IIv-9ZlHU@heU?&58U;#qyXK^=!U&fDkxZ zxXb8s%)58a&}Y8qYEJ2Eek6fzoNo^2i9Y6b?&B2WjZqORScKI&c?3k;pqTaM_h z&grk-*6JA8+4TDHj)^e(>u*Q|oZfWGYOqUtA( z;LL97UHIX!ZtZjKwWe+Cmwo5WuIqV@>$k24b*1J<=Dl3mKmO_Y?C#22Y~W7 z-{t||?O2WT4&3kQ-sk{N@`|bLp03MC-SZ2t^#4%ttKsr5AM;Q@0Wd$X7Nh_ee~0qB zxp!^t@*4C2`t?O$@<;FADxdVqe)joF77!ox%2o07?(hd7@l;Rq!GP~6f8b9Z?P*`? z=+W;dAL%7;@U=ec8_f4hkN5-O0xbXZ6L0a+`S?yB@hzYDj;CVm#r0p$vSPpgyP7}t z;|%!hzJP>}^fZ0=r~j|IC-H8dmFx*m!0@ed@AmGZS!R#-&w1c@fZC(K_qLzlTW<89 zAKl1~`dE+l7wZ7>F8dD9htc`#ui*ax7 z`q2sf(J%d!1pZ#i!Fj*S?F{^dulKuu@`K;|#s9B@zWy26@jze@8U_Pm5ot^YibrL0 z32h=4iR#sw#cs7-?$;XzkHuy3*_=kN)oq3w&VI||^*NiGr|<23{vYlDiOB__@X#<3 zaZ$06@zF67a#AvZ@-mT%QZsAQvP&-%bX2qy@YF9g%aoPWwH3DWbvCyDmiE>*7k5{; zm-p9~cTgy*xR}W}3K@w2*g5z)8hTo~n)=!r^$w4b)B6)UJX_qTlL=lk3GM>pZ_0UK!5`U)*&(^Swe*ijrc1_5aL6L5w}gWSP>&VhZHxm5>$i` zI+5;3jx?DrlSp;4I%;frG2lyE`}WDS$)XW9=glT0~d0L@ro!S&E%&=m0ekD8hY}$+zk2t9s6zofRHjrsdg&2l zPb;lNnYht6Q(dKoZhV_>?lGrv_tyRU?q8yq7_j(9z~n? zbLPrF!`A*CDR^*x=Ee6qRlF2Hmys|bY zuMw`p9U=1&M6N=lPfIJCTWZHL)y$H}ab%QI#uyU$?KIOdBT3Qmx;AgR?@k=C zEYKD_4IT7QK7<4fdDPjPENdXk}Hw+EiaIjCs*STvWFHceQ%ieQ7Qi5|vAyff%T{m5N+f8xOPs1eE-d&8n z7u{{6(Kg_M1-|V^MjKsNK?SAwDc-pVKA2*IC%#nMY@70w+%7W|*4&Oo_O@SuzoO0L zl2z^~VQL?ScbSxBUOCm;3TdVp6iemzy?<$z`QnQImjs&VDix46nTC~68VZNwRnKBc zg^n8IIMV!?4sc2Cwp^@>mTc>t!v4BmvZ--->9m^|Lg^?u8^G+i0UmqOn`7hI>n?nj zo9e0k?i=vjT5OQvwA*Hz7^fN1IPk{*2E=5JL*{ee$@Atr7P^BZ^YYFy2PICVnNFJR z1wB9gbH;Vy(^MXj-Mj4A%?{e}+Ecgvii8;*UEb2&Rt9z4Rj2*&D>Oe;YiBIKeeO_> z$L4w2gJ)iPQD9UYE{NxZy87;^{~o@5*MwbV;fXiD`1H}gusGhKRLId}hF72d^DQJO zP|3S)U0j_}p5A*ZBU$u(@jvh#&?U$i%I_loMl0?C5P{UIiTyq#pPjLafb$?91lJcK z*M%cr9Tef;WaK`0C4*!mRAK!*s3vfojB^pJ(qT9#LmLk6gqX2m4SA@dtF32)J7nR- zn6*M94pAaVdsz<6@jxUV5sFWoivIYwzmdhThXKS9>p=6vEDCW~L_k{ug#yMeo)J7; zT%K)$sKqEc?rvs$qZGF=MkzVX0CJ?`9u0P*7ori6Lt8_20$E5s-ogbq%wdCwF@-}u z(v3w5(8v1K#@X>PCgPjpCNaqdg=r5?zMByzN!dvj?j#FTbj2FI2uZLR!Ua0JNh=Lm z%R8BafeR94dR%GCNcu8!l>E{f{b9-fUtY3cMLbFdduht>xL_CuXe21%X9OWgvx)O7 zVOuRk`^e20i7Yt%JBV4kx~xSVG8uBm_nxRV*^lqWcALBT3A#+b--rW$BSPi{KS zE$_r;+jb{V2OP95qOj*b_qk9@g$FgNoQ)L?IL~8Vlxwk6W*6gVMSp%20c|m;I|qum zk&^TvAXOI>QGTC1OiNTreW2oSWC80T8=0iGOa079oihQvbCwn zA;8kwiPELPqOEa->t5SHBvrEia$Kv!;#g-QvW*P0qlG;cVXyj(qeWAikX0&UDT}Pb zwAFtU9qcMNs|U3u5b9P{^{=q?0A>YP*S7e7toUP_68%h}oyba+Z*a6)j;2 z&|8z{@_pD_ZkpvCcRL-NFO{I(`tX`&ecuCl8XX(aUViTVj#f|#ybwTW5 zXQhV2Eta8wC7ecFU=zpxiX~WBZ9LnFy7|H9&ELxr0lNh@g%GHIU zfZLg3D~IyR1kUo7laS-yN*T<`f@we6+d2vZcD_tbG9vf8#lXVXz4<*OZ}uVL2K(5+ zL(cP__1xzI`+1Ro=Chy)J!nH0y3nbupaj?K53E=avAlsUS;H*Xwbt0q=H)JzIXzx2 zaNX|QyOK) zt$Wct<~R4!J2X50t9@UJLEAvZx7#gtuz|gAfA<^U{}wo*6*zE%2OQxCS2&^pJZ=o@ z*4gh~cbet7(ZdF7Fwj=Em|=a5eCs&eAt$z#6TaX*|5)TH&oZf{J#C4gI~c3(c&oWa z@tNaH-+G4B!g*fup8wqEL09;`ectV08y(aDC-nh}Nau}r=Wg+S_LI+8?v|(M<0{ws z*1xa}}4#E5}jccI_i?s><%%l58!zxN&R ze?NHu8jLptT3mY5fr`dwPSdB;;?7-1`sBS%`O1?KxP#xc9`{2QF0QTVw%h++@H+=S=#xHp9J{-+iVrE( zX{`R@Qw)3X?!%RMV1=C*dw{QH_ajuZm*&R}Z=u^CHmJG|%LS zPXa9v0}G0{N{X5&aOgfT+Y&7UA#emQ@Pu;gToCW&SV!hsPXT`^{VcyQrb&<^vk?utvp zMnjgO0u#gt30!aif#?k8xarvj`kc;=!y>nIZp>GQS^ks%?J<_CgDd+ zZ}r#=zX%ZxM-du6Zg3N!7NAd$i z9*zA-MX=PYdD71uVXheMtYxZE9&wQ#=g}VPamWe|AMcSL6V1WmE|^p>@nk^|C8qKJ zoGKQa5Ee6VAB&D5k?{_%u^%V0ArX-x^G+gZF%x-_MGDUK%8v|Vkk#VNI_gZjeD5MF zQX`ut!BWyCRq`cc5+?KTA8l^kI8nob@gS=q>$Y(wF|zJR5F*JdaXQfCL@*gG@+X&4 zCYzEev5|CG3>a{7!+5PEHRQrHB;sx>($vuW&?>&pFt+q>_!P)2&yoqylGoIdE!#3J z*YYjn(kXwi8iUR)ij{f{s2?k?xDEJrV+nk_1H zk`(&^9+UDZ8PgJ1ao2jRB`?!6G4nG+lP)1q#q#JXZ33#=?JDaF?$q%YMA9t(ijmgh z&@j`|Hg6L*bCWPdlQ)Gd7XdRdgR?eulQ@ghIE}L~kyAN`GdY=)F9)+Zfm0WEGdim? zI6>1MM{n)2VXmxlC(+;#a|qa+b2*DsGhNarw=q3?vplhrJ>3&Ng%F!qCdOj#in7Bi z^V8nIGV0I`Kf|V?{^w?9li;YcJP#B>6I3~6(Lmo*CCO7kBUC~sltL@iLN8Q8trJ2u zR60A9LqAkOLDV^uQ$zu+EeFp5k&Pr>p*x3T={E31V-!YbR7Pu*MsL(ca}-Bmlm`zE zLw}SwMsJrnLqt+40P)IzNjP8~Azq_Hz|uTB57L){cm zp>$Bqlu+R^&$={E7gYqyBM1NhA^8LaPyjFhEC2ui000C40ssd8z>07(BpQpzT#H1yW8-$JWih@n-1&IsjvJ0a2tVxZGwe^ zh8c;Aij9nqj)akrl#`Z~n3tNFoSUAVi%WTxRaS<1X$N0qtYxdMu(7hUw6(Ue4!2?k z6;8ZuO;N|A$w&Y=Vd3JzF zP|Ik{d=m=P-Qnum*jUS@&D;rjH}IW7XY|?)Q5flyLr{hwN_;peqQ#4XGGf%oapNVA zm@Yc9&@m&)lO?`cwmk2)WLuY4=xONF*d}49}I>(!LA~?m@{kM%(=7Y&l@giID3JEX$=}G zxBbkzwd>cgW6PdRyS7`~xLbR$JE84v4~jgf7_I_%YP-KP{>JOL@Yw-|ty#@NJ;H^_ z(X$X!O13)~@8GS5A1^-qcQ5!NWP9KsFX3pC-;3*p8a47}XeE!nuHiR@E%HU@Pj}@N z$P{XUxi?q_z%9oCX{JGiM?$Djk;jEpP#Dj7=DBkJUN_`{h);(ka%f_S0+o_zM{=bwP) z2_}_ff;p(6gHDO)qG5{WXp#>qnc$>Iq6Xn^{Sj#-rAtog>8GHED(a}DmTIb~N9I*u zUkN@M7+wjE#zCS}7J2Chu%>ESrJJ70=~?|XDw?HIP6_0!U_#^&RK7Or?6c5D8*Q)B zR!bO+WR&>Tie*({~O4#CPKz z=-zzy?Ki=H2QKlQg;I=h#*EsC>uE7R?IhdZPN}iYH&bo-<)*Gnq|BIxdRmw@d+fNV zR2KO%g3Y3(DYYE`*6*O|rX~qRp=gc&xzv%~uKVuLT8$2P`p}|v1mB9bK?`2vOKzm| zmaSK>qo>{Y*^cXF#8K^P^MzQVxW-pm~-vI%G!HLw4X<;Oz=+qWS%ZZSG)wtRT(P0k* zFirzrOWu~qg}gxmt8C6Yq#QTNMR?IsN|N-UDD`#8vKjDV_gY^ORmsX#x-w3$jOCgz zCOIiGs&c==W8D^2ymb{)ZTtD97)#kI8%gPsT$~^X*~qCkmX4ETbXBYh=%mTQqRlSIXtt_+zry%ts$35ys>L2RYTr^e6(&#;Y3%6{_Q!5|@TnO2~dOkKhBVLRZV$z9qD^eG@A$ai!K*i4~qf`=(-d zJIx0~seze->l!!LK{~?mp_@b_Ug|isJI)MU&b49&Y3GNyTJ)p3g5QtK{tN$X;gjXn$7E7{LRV0)`60Xi$ydiB5l4X|nh{61R7 z@x2MUn1cJYMPh09!JLw7&e*8SkOeTK9FV{sPnt`bZO)j(QXS8-`O?6;!5%X$D}-0v zJI3lnDf5yfWjPSa^r}>&oXw$TBb>|lju(yUtgmDx`hg$)lZGR+Pmlo&+a^01p-e>B zlh0&JDVC1AP7*h1vNkfqRLucD_t=c58s$yt&rrOkB5FCW>R-W&Cl$Nl6elN(T6 zHegZ#!)DGxR&p{-wO}id;$JO>!tmlj~Yqc6)JJjEFVJc@}b9CPIPI#%js}B}| zAs+9f@$!^wAM<8_ziFj5f_*kU%a&%zKm6CWbL2)VUUs}9_q58zz4E(l8mx&r`N@L& z@2lPs&>Y6eFtOeKGP&l7vkptngit{5R82I=FZ zH2mSo-rl83JNCP=F<%c3cPw#Raj&Bzawl_0W@%4Fe0|4v3Ai^3xPV$Dczu;9>2@0b z4OJ_@1!SO8e%69p=JzQMrWcEceM7f95tUQ|CkJs>2b$M`ESOWqrhe;ZQr-q3FobD4 z2Yq6sWA(&RmFFnRragc6ErybM1V~dq17yROgb0X$E{A}|r!H6&g>wUdB?E5NsDQ6jP+D|>#&}EJCxi;NWd^2! z5>|;F2#pCheScAnkkAmHh>cmOeQfhknxc)K_g#y$3<~fu9N2q|Hj!m_W^hPTAf|dj zBsuw{ZIcu^8>5Rh_$X2`CKVHmHu;NQMTiYEC~k-v1z98{V`>G4R?TQ(28oeKi7FTr zhH`L?E7(OShj6|Hc}Y2JRrejBazefkOqIfwASsTOxMLbugscczY6xBbUWZ=5aWA@7 zU^lmhLeo*AF(&+EcQ~n&`1on%1}5|Ok5%|ikGMk1_?Jg1SC$foRhgLWk~AvVl*7XW zP>FWO#Vhxwm|%953%4x>Fn>B&f6u6cwh*^55%8*JHqJPDKj_(g!WJZ*QEuz8c} zvv#ssC=6zDNOD%G2_eu$kN(0?i8d`%X_>?+Lhc|F&LEMIIe2|FNT*Yn#TiDL=_@>y zb)h+yK30wh=5aBZSu3gbIh3ct^opg{28Jz?~equC&31VUp$p}!f1S8~e`ni4o3OArECLX&09nw;r zVCk7`1D%oAnP>BJ8CRAdCxBmKC0xd8rSwSV$eoZAXgX3C&4w>~D2(nIn=FTy!dQFz zu|tYdS6;GS@$zwlvTh``Zz4&cE5?u^1}b34l$qz02InP{siam~irYd=_t%OZieNAq zl4n6WshKEYB3~4bEulb5@~~WqdE$oM--=cVrsW}g?Csp2r4g-$eGDSIj#4N zU3#T#^-W^7lmiD7PH6;F+FW}|sKx1BSvGlLnw=jiTpS9bE{cwlq@L@^Ba66(*XdT@ za-wjGqo+DP^2vnh*p^h)s*3Y`a_A&#WRe3UNG+91FC&%zkji3%8W>ONkd676M?i)M z*{c{Ssk5r7uz0NWSA=F~Mlp$P{N$;&cP63QhGa>1ibJQUDyQgEpPUwn`4^uGMvM(6 zc(m$eM9PzoLo&QNsci&O$hxGD`YIDCtfa68aWDb(`me%-el8*g?**n{dWm6~sj)~m z_R$tta*M&)tZCzsk@GO4)t=$=t*Odns~U`JA)bC%S-5psX(WWOa~uoAFY#Jj6pE28 z3pLVsD!~e+h-y0kd$S{?p$aQ@j)t(#nyDnqd@0#~Gggce+f58{4!R%Y-<> zCcK1)6Z+2AQ`T>YcF@S!u$OJ-DeH8i*aUwH=E;TWh%9 zDy~nPyLtJ9NfvRs`FCykF9=mxxvDB>WJv-Vy6kpQcm`OE8Ie`6LCjmdDg(Cz>jm4H zS(n?lW>3i@5S@qra$LJz8?{h@W#gp5ddS z$NQY{B|noFy1w*ko;$2+;0$MLy~4S`Wq1$&+km>u`mzQ3dYko;Eee_uRcPe98&oBa z*9lCpi!A$Bd+Dg7^P9rv#=AZ8mK$X&$ntk>8p9a-f3hO7aYS>zbsFB3Y7T6rJK3NB%!Cn7psG>42*Bh`=GLjOCxx|b%4Zf@WXjLl7vik z-W3*DNvWm#vys<&0OX<*ESd#qAGOO@54b7PM#*hBGnL1%&{e}KJj!T%K8@12AIG?L z18)@rD9A&(_B5nZX}1JwXC=rnv@A~lN_LUfHFSa;y?=1Xpu2NAoRr&AA{Css!tA{k zOpcR>lE1md=83_i3&!AfW3zZwrO{8I2EX21$^+Ge0>+nN_oC@}aDyw(d}%g9>`^#t zSTlT>P$#P*dNM%)rJcLXg-k}6+krNby^ZY10=>O}yU8PrO6&`EW?9gI`>p|bRrmB` zxcfb33eFZSOSQSUpwlrKsHIob$_?1Mpi`1&MZqlyEFM@Zje5Z~^~(wx%=>&(09uCW z5XJCe$iB+V&D=U}Dms|t#qDg(7`M8nWUB02u4Y_BD16cME1Ov~D0NXU8?C+*ZKHtN zvdWapEVx;>dR?89IWCRQLtxVXF|C+2ZPQvHmO))unYz<@i(mfK$wFFx(^|DM*NyM4_g{TM+q*qR)@LSN%syNwc86*~EFwVv*d=z1Ck`+5{c63~j;Psm=d%e64C}2GcLJP0HZ? zU-=U(9_Evx+`7GusRat$C%xDyCeNQ@3YPtlm|exnU2o~&*T+mq+*@V_Wj}a-C}+=u&eOg z1iifXD&5c>;PWW0=&M<7wTnOnm+{=DHEg@Ld)^M-;NN<(ra>kSVyYC*%ix4Y8m^55 zUVZ&cH9fby^kdnHx-=mEphq6MR-@nhZQ?7gWBskq9vNbitjt?JUV+QAM#4D7q9xu< za^v~hu>Iz69>3W$+g#Rc)oR}9v$Dy>YFe(1s3WwZ9mM=IK?3CDtTTN?^jx6H{IjOi7nix_;C*KM_yJh^M0dy@0rJdV*%JJ`U{Y!K5gShi}T zy~mk**vB$&VjDvLe{SU3Oh8cHbW%>`i>`22Zqr@B#A5EV>P^QzTipVR#R9&qcpYPB z+S=+=t;AT>0MpdkzT>j(e~0_Z=EG&-o7)@K=bcQN8>ZZQ!5O3Rh&OrM0Ty1Uc zjEC-!NQqQ_zdpQWX$C6s?(5Yx@1Q&EEaKejuHv?aV^@5Gl@sPCUgoU$>;Z*OE$pMk z1J`lh^K$O<*&f&rPd*k?sjBBX^qjdKw8Q6$@p#)Y%A3l%1@DOdxhCJXjUEd#P1anW z@103Dp03vaN6W}-T65bfj|a!<0;p0QBT)_i_749r>wHF>e%rKFnNII{W>ja6p83u}GAZi%jPQ!C?mj70U-gKe_{<%8>rSDU&f@MhvH#6B{LbYD zO@{^xtu$Y%m@mdZb8vQ};IkR}aPOY!?e9S8!<4=@Zd+ymyz;73g~`^W-&FW7%`CT% zvx-k3ukgG{KkmogZ)SV%BpuMjpEi(|^X9^fN=7Sg4g@XXYe;N;Je}f_*)%$#Pid4| zwO+CRQ>}H|%~CyK&4p5aNYLr?L!*&s#1}W3e6HKgYyF9j96JeI`T#?0l~(!F@3H@%Qz)K9I(Mn*)waxj5X6Sv%Hh)Ix?2DM8c&4idXI&wgD3wUHx7Q4&Fh6giS) zNjkZmjPRKYreHPsEoPAfqW{COzm;A#6Krv*4#5Q!l2&TaBxaA5kYiFQHO)GH4>z* zSYOA^+RsRB*;O}GloXC^Sw#~Q5>)(`I4N^u7(NQuTFzyKdpPBlo~d}{>esP9L@9W1 z_gA(I4;cQEtR(BK#>YI1jPiQ4>jOIH5Ra(ap!U-}qObKzS=any3lO#endrbk0TKLS z2!PUQtSrx*vn;y_jYI5|N5uIDAUcra%_wW2yK6d+KBCaX6Hy#xJMIKq@v!s%NGk6) zotQ>LL!4 zc`ij0-8(zg%pQFzF+ zP|%Qq;~yo860WhOP76#|N+Nh%3!5~?T_$Y8%rsjR|Gy5cG|tZ<0D@5VXfT(u!Z zHT1wVapclc%m+ktlOtAn?e#hq?Zh+BLkIPRrXXib%h1eiH_oFg^_x)N`qtK!HxnMVE-$+>N3vT7^PlIS)K7>Jj|^j3n0A;#}nu zsW3Lw<`Ey?HBKhz3$w(<;*My)~bh=W`eAx z%h=fzRrmbYu<@)o06W$;#B722&O0G=u0{6ko31$<AK z_WuGK_F!W{Hj)*OZrs4o`ijJ@ffem2oU7IYh4&tVCF?PSip=K{wZQik>`KB>pW);| zxZXelgqk|sRXR|C#;wo+-KzlPsPL{6Mg?I-u$c@$AhOf&kB7VfotX+JzcRU{ejM2V z>TcIVVQfi>+4&#EzGJZ(bz>E^yH*1Mg~bPWh;QncqVYzezWqfoLcLR9h7=}1RfVHm zF&jeVzLlY&P0Eck1RoUC)j~b;jE@tzp&hT#M(MSnQPL3qSrZYtMM=%84ML1nX>f%# z)Kvg~JzP^FtMfzv!eVt}tkJf5;zc!#&?1*iOxsf9Kr%{DcLRK&mvr=*(4Yw{fb(GB zgn+br5Di;$990-hc*l5UfP2MUR|Fgv4`BlGjl?5hB{2ymIV3EO8x$fW#!y82Z6t|Y zlMDqM6HVf15{gn>C0k%gJer{Dk?8QD%Z_)>c!KdWtQ4ayH>b|fh%X8Lut@sQ^hgpi zlb}be-x2>vP!pPmbK-OzhAg6}u{d&QZqTMA-Q-DLjVGTD-HZRo2}M_;MHR0!M(|oz zt1~&Iqph>xGgsM}Y5_@>{KDrNRhrNRJt!&T8WkP?6}n8JE@7zd%3*Ux>d`Pl^Q37) zhg{}DQ6Q4xqSHAdUb>k`jViSy<%}XH=gAP{`7jwW+A4bJxtOGW(MF|#r%Z7=pE|yA zWg6^F31=9zqRut1iL*#$vdY!#Oh%#vlj?@5$}n9yK(J(GYGXx0&ai6qSkr<7O2OGS zcWw-`Y^`8gWBN)UQHyL>tmjNoxej+k;y4if)<5P^-DO z_UMc{H9~pBah$ewv7hGM3cM_Q-qqHUzxD-KG)fa;!%8&{j+5^sJ)GhR$xc`s4NYc2 zo7)1zDW9@T@Pb=e-3ij?0~-*ezIKyZ1uloL7M|_(`kId+xmCa_?u!A%Yt8w^r+yec z@h61qWG(-tPTZkTn6lesF*9byv3zHTgiNS>-t?yDs8wr_07K?Tc*xch8VEI&7n$Gae;HJuxMi7f}%t(32dOn$zs__Q;nzrGa4-b8f9Q;s`>Z@>$QVUT8}pz0t}4M3{J2B;iqg65mOxFxI^`2}n0-J2OYOc>XPG zsyqkR{RJs~4FYBIAoS@qm)2YVymhQv14j?W&rD@rE$V*1rlVRG-NCs0|6 z@Eo9?X@5`lLRf;e&2v23X?QFD;R;`Pzm7t)AmWwiNA3!F2T4dm7JArN|I4GW8+JwB z$^lhhtltsO2w+bPDx$Py>r6(S_sj+=zdbf+h8{X=)_kAE@xAm8A4Wfwv$|=M?=5He8l-B0E z^6U1zATC^``MF{jX_f>B*Il*~m)yjQd=QO|{rB&j_U1Lh`xEoJ{O#X^*zzlEm@|gc zXPjXVKgo!jLH~&*09_dDiZIs83DWtF<6%`?#+GUk`qo!L*c){3y_p+-%>T z!~P^-(HzG8JTUlNL?ZnE1=LE$OE{_m`)>9ouq~Lc@j%0*2&=Fl3cM7smp*_29WV#; z>M2;O5-`+jg9B!qcFd;Z_Ba+blK=7;b#GBfq8`dW~ zl5hCX?(-yU44kUnX2~GvZVJFpuae9W8!-|kapF248dT{H!^pM#ud|H7$`-59FfsAq zFM$4S{{jxjw&>BCFa>8$6=4nrmCfnA?f_-6aM17!anUw{Y{**#kBibC%d~+1W8m5clF%y0OffU6>FY!<%4q37xB|&;ai}J7&%W`E*v{vSF%vJ# zUyg7(0qDv>JYO<^q4{}An^wAE*HnG1aDd~Q062;}0#339Z zu_e)qC1Elqk3dVLj(8-J>kR33JaH#6VE{cq!oMAF@)(zJ4|xcjM9k-WG9P75xqycg z%S-^dE22meqQEdq{DCOMWyEL&&kQnNa_0MT$v~z`3X5?pLEssrd_)@N%Alq&*=?aHTwo*Z=;KyY08!aND`p+%p z4G8<7huyOBkTlyeGqq~-6eAr|M>FRQE=5YP9#bPl!ba>7GBzvbnsAM(je!8l`bK9g z_Jf};&`KO`)m-usy=$%%b1}gWH>(gSdDHJ=kcxPLGIvcACZHF2sNaNB()3U$*~|Qv z6El(W4_(S8rP1Z=NtABSZN`vyXbI5-QXF$G%0>+JY;&)~^X|k4Aj;D`&olADVa?=I z4|Y;lc9T~`|4BH%=BwfoIW6f!J<^9ZbWqUJL`e%IQB6P_4pNRpHO*=@a>=baZZ;9- zh79CKYjaAzGeU`U@0t@e?~)H?jj)D;Ju7n~E_1k!(=9nO{@y?XKk`B^@nlfbLUj`_ z(Lw^7@=GyeK%MQ3473K1L@4NL7r$e{mt-cwb3?NIaN)UZ@D`T~KN(hXBJ`2f!}kJ6Ubs7QDxwW`Np zHVxHo|BXx>bP0Y`m+CZexUoB}pq9jvOZ#-c+Al_LHT?pWn+EmEP}J8*v^d|OZv<}& zy;Q*3=06uzONVvrJcCJNF|*!=<~WV*YV=wXW?afJW}X!X88IAZ!&D6x{5qg-I;t7N zwfr(wi~hh>pY$#oRY}KBRv(~TgHOjo6vtcz@Z42Vr!-qv$3@Fhjl`^8Z%+DV)Pv+P zKm65GZ6!nM=vnQQT5)z@b4g)OlUEy-b)wP#oGMcN;NwtCN-GuP=w6t{^ z(j;lOjB0(B3t`ZUG<1SA$P5nXYrk%Rv{5TU^KcWOkZxL-R zcdjV3;NWvyA1Y|KWtMKlXd8z8APd$ob`8o^(gK0a_A^9{GgYCh6ccx3FD5N;Pjj3u zCyiA`6j0#~@aqI~m=_qV`2)>0kLrbzFIyCo|L385HJ!|a$POKmEPIBCKOYfvn7ovfoL7sii=Ql)fbck(vnG!AeDHDJNcMf|G=2P@hd=i zdy`}X9MY9F6OiYYgx+j4NgLUW1J`pE!>~Aj zER%A^ZVu<(HtY_DOniFhXcMD@SEHR+2~? zRa#=MS8oIG1hLv_M6xE4A(gk(<_PyirEMnKxuVIZnCY0PJ2-_H|EF;(z>`Z!AQqUU zRZOp2RgxywnzP!A^AL=Y)aA_51o=8Ibz<}`Bb6u6hs$|(KX2`5u=M)!r{{R6;aYy> z+L&G_9MzV|6jF@cmWuVWGOW5i891p{~Hq{K?^bNWgL`(`*pI{5+p~Ss=D()p8S7o6GOZ$6x!)I|l$iRIH;~T)(NXyPMk3 zwUTx8(G%Rs<2OOGMnqup$50(M2Dn7j4t36 z*6Vvm#*yu8JJQc&pK@KJ_mV6oR6C)z)q}l&E#%X&tZGk}KASc^ zJWJ7f&;P9~*kHR!U2tt&aPc~|ayWz0>}|)k;4NJ&L`=LOx4DtM-1BkRKmBy*i{=xF zGYNa-0~{Fs5I^Jl=NpgE0?#=`zR^v}uzNUhow@+4>EG42yI8(>bBHV|g>V#1w|Z9R zRj@&2%lEM{uP`D60M>7DCyI_?o42p+;Av1mLZB}wJ7>3l*dQOP7z zv0|^Am=(`J?f@ zKDsS3N|+sbC^}~HL253YeV!?z`TYT!osyoGiiRS7bhN;>%+kVkeBfGscy#p2#?l50 z4-*#~A0sC#3wLmoIs-E4fl_k{c4B91Z)b0Ld4GRjbCdUtm!qd|-?6h3I%*`A7FwS> z`HfmStJd4MK?0Qa&RiuAyEydwH!h*Tfc_Xhd{>RaA84vpv7)#NU&dA*YWcE-Xyl=I zCiNY}wQMEJmM)Psa|X@S#F{oEs>`Vpr#6T}s(=gXC0qrbJIkRus&wg^J9eJR(H4(n zMr;&$b=Y80Qcj;noA}wai!0Nz|79pqn~I23#Y1TUDXNOas&;(-`|MfRX60BijYfEpMlV;nK39BW&FQsi=nyb>5fZD9yvakDg2othrD zs*i{WUW%!$N9no61ewV;nR&e4T5GVw8ZqdgcnlP!mX$WiNL5doahHX_unNou_bI6c zl79}d4MfuW@e1weBUWRM!h4`uei(GKWF)Z?wyu@^Z@(LoJ!Zx&mv) zuI4=*wbtz!U?XTg(GzWNDOZy$z^QStf!b@cy>^khIi2XY&XkySxazn@uHH1=c`L(0l7TBZm*W!76?dhq5%BurEUWM4To7N2vT z`eqe}rs#q-JUJ$I}uYv}qX1Git!}GEb6N z*)=M8EgKL%{}NowWezjyszV(*W>k+Kr?0_-pT4ceehrGVGqIwv)4l_VZABSypoQ)k z-2J^#dDE_Js@od@%f&b2g~Dbw!O~Rpq7R%IuPvXVm#o-?iwg=yOQOt#8;l=smKBTBOwdRK{58#ErtPn-3k*$GKPiC47tO?4cP<0`JiurnkgU%FSo$t zbqRuO;n@K>^1z<;C3@4VOW|OZ6$*-CCSluD4*S=_7N(GV4oi{)A~3oyqH#`L3ynLx zcOV2x!HY@KRs-AR00lr$cl4uEl@j&DEMY>9ev(_37F;DHWhJYHOC~;y!FMh=b_j zLg<4f(iM|HhEyd#1DO^BDsm!>d`m>f2}Mo;ZC)d>;4dtg$*jyo9wjki%osE~LB?~C z$wI;@ceTzkx>0s#OesxsL(du^i)Gs&W*qak!+xF)rF{G(F=(G7NYP*M+aGmeOYhpb+NTGJN6JC?$N};tBy#AD*v8_O9pkZ3D-s7Pr`=XSnbcOu| zwt!4kW)u6$AR>7QI)3R;$XcZxXL1pkdrfU;(5cP{v=gm=r7m}s;#NXYBCocrRnz{8 ziplg7s2B;TP6a24(#}z0!DZ@PN4D8x1{S`DeB>i9O3e?XNiY-)khW4c&>iOazo;+}dC;ZyhK7gz!ovt~9=wmi~_rkL2F&JX!S7psGWu>qTIHqC+ zCXbg8D5IfMiHu@=!dQo~ZRUx?Jlw_dH_iRco;LTD3*>dVkVjUjgnu$VGiHh>UQY3z z=lp;Q<5aIb2@dJD{Mjz9TP zpI+otRt4C)tm|&K=~S53gCZKiQTS?9JxlLK=iAu#EvdeLtzTvLo7pOJ1;9I`V=zfN ztuf6u!xh5ND--hMJ?AdC>8YFbnmY_-)Hu_Z-t;ow8M^@iIiklMUewsn>T7K|zJ+UP z!R{O8e1TUcVe+$@{cq?on#@^uy>PXs&*2YOGCIhNbdf9J>7sS||K2&iP7xBCpz%&c z+K0(gR`6Wtd``T*iQ9ErIX>(%m-#b1Z9R~(=BPBxx#Hoja#XI{x)ad4=OZ0USJxe& zcdz9V_^$W9>z(94#^EpXyex*XcgSM(x@LFFb>E-aqQ!kn+GLSImK5z6m&AKGXAZm= zmcA>fjJlpkAN?cCeeQO9p9hg{pM}6a^cxTV-h1@;qCQ?Dxi&6Ttq++Inz`WVk9_dO zu4ZW71tiH|$^NA?7S21!T5#UmZJj&)%4GBq0n*a-nMKisTc6c~FnM19wVvx);OjXZ zA{h+11YQJ!ofIS>w)I}uF-@_GNMBHAQ-5vVL&;^p8XpNs4DbS^P&vy7)GF`y7U5?E}!&kYC2@)Y^ zap66U;O{9Gm4Ka$#oQC(!{1bd3%+3gJ%@N9RQ9-q%Hf-Q*&&rR*zEnF=Ovj{Xq*DH z9St2Jp(P;(E+P533lS0?L}c9PJs-nN!uVv@B1X`OA)MQ-5EKsK7tWvky_|d124w}< ztaJx8v7mh+iUJ-S1@Isa?qR7F-Q49A;T21Vc)Gc}6k5I%RHsLSAp1R0iC|IGy zIg2DNoiX}@x?vX}79($9fdUx7myuy_abn7)U$%*1|FTWqjCrEU0NAU<3PB}Ky&wW! zU=Li(p`7uYsp+9HDM1z>+bhzedwiECq+nTP$;CNi-*wUI@dqOq;y@}P)sb2#>|J@` zV=kE$APyO3+z~oe;{`;dogBbvy`2vxRDRLlf;5s04#V*=!@YQmUL}Vd;zeFWQ$4B! z+r6VIvSh)?BI?!6-!&pZx?U@0i7v`oA^M^c0%Q5Tmt#R>267u|Afr*@1T#A0GhR+k zu2TKD4E#w7*A3uRUSs`zVvJ#u@*$t%I1fPm&9ey&&K)II!sIn>;Wp@s7pCP~#$*%B z^Sj{BDgDhW^VWPIcq-=KEG>lJM9-0r$ zCwS`IY=!}#*(P-w03iP1ZX&2|E@pNur1u#CGa@H$N@yuaXb<8Zgz}ElL1Yu68$ zQ#wF_7N|vf-4YC?9qyHVdW_&fXAN?I|435D*5u(}3eT(JACpZ#Kjn<;ixTh~n+l*2i;^o=N_^503 zCwDd)$MvOZ8u&97yA|mMN$*sj|gT|EfkLrG}A|cIu~sD=>=csKQ<<;+~@330r7ERahA> z4U@9IpLf=wo6f3#R$FJ@Os+P{&)^bzMjN{fjiXGCv<*ftj110or;0*fcpj>HH0+?> znmRcv7#1lRoL(JBs*;ZAfo^NZUh2noY;*yS$C7MrjY8 zT2KaWR*6Egq~d9!y(W;fsAO*hY%PGqu?ABMPD{uLt-Ie+*xj22U?r?O3YOk}U3)eI ze<{L?I*yL6V>RSQNCa>2BJ77LYt)XF*`n^VQZ9;6?Hz3E)k0?0a_kv6lGWOv?&><}P~KYH@{nAShchjkGA672-shXH<$i|YoCcO^5G}H* z42|ySs&*CGf!Z@L&E7t)?dGpqMQqfqE9%H*^tM%nVJx*yYUz@02M1!ve(ltG@CbJ> z=9cVkuJ6XFuK14Z|LIbhPq|xiQiNTp%x^Xeyy9-Gx+(rPZm6hD6Mg25Q5zZdYxOi2 zI=TrRGO*M#*tg=a1*^~W)=0)~aQNaRe%|FUPD$8WYhd2d^>S|c>Sd>{nBP?>q){Zg zzU~@d!P+N=;rUsvm$Qxx;6;nTj-xpXs#8raO`d`0dX-i zV;~Akvxche|Mdpt+uGKFFe596tCdRQcJ6O`AqjmxFwb)1f95fJHrqtSTr;K3U%1H< zn<&oau#D^=L~5Qfcb9+$fFx5gC6DqboAB0>jzi|nCl9o?R`M7tbVX{xCtF~(RWqoz zlkV!&D#~)-vNQXYRz(HP@d^zvpQ|$OA%j&gKZ6f419TNvbVE|Hn3`9jba5*;^h>9$ z7>BXzg%env?Kg9%?AdV_MhqfT?A~DXI%A<3elPJkhki6q{&upGQt~~|2#`YW_N6p{ zLNZQCGF9L7PG9m@YwVLe^g>&$SWEO|T4z|3tX&2KP7`EO>u`~20^fA!TKq&t;vV@qRWGu>}ZfLoJnxylr${xp?7 zVlW$Wl+^J!zhvKT^iV%j0CgkmprOC+j3h}HfNZb^2lQK_wrSE!ldi#B3|L03P; zVhgk-pS5k1^|>*0GMn&%j&&uYb;(Awn0oA#Xl-17_DeHwqx!a13$XuUYmTHO$o%yz zJAy!GY#mf}=?!z+0r$ZvHdrq+PCs^i(`v(ToCe1<=8|u8S2Sifv`r|2_)T3JXZL}d zRe?q!BLfMXtfj?mw^oCvpS*)b33^UIf`L6Rip69xy`Z)>jqn9FilAm&oFF7m0W@kVqq__HOc{Cc99Hs}!|238O z6&B$bg!+d|zx(n6jSI@d-J2rDq<0djUvh%v0$NRkh`ZbU8&2B=I zdMve$`i}FRD=(~|GHMI~-2!eWTa&a^Q+kDS3bpgw8RRN?Z&0}@hiGp*w=XHFlenoj zdRdSUi>%wEm%Hk~x_m41jMKP{KS`b&D80M9yu-xCCx zVHCA1y@McaeC`A7^so{{J;r-`Ko0=NN4i2Nnzz#R)pNbY`m>Cqe691kW-#`=yFBUB zyB6%YA)fw%zOaV&2XTGp!h0!Kw>B|6sk4X@18yS+plTzO!=%Sk?++g(FGH z<3qk+Q+27wa_RsviQNjoNMW>BQw54qV>R|CnIAPOA?Ej1+#z1;LLS#r0djYEnHQ4C@XM-3Gb?j`^LWW{OC4>YOD)fNS zE?>F^9PGD=Butw5PTdq(3}?=nHGTdB8dT`eCAo|qjfCypI;O^&3InH9nn5JujB$&a z6{}ROTt{uacBM%o2l1S?gQs)C(t*Q-SQC~SjCH z0$(SId0X{v-o5!D1uo8)Rj0~~pF7@cD=TX5z&?i_-N3i#lgg1zp6-D5v&Xle5$4;; zK=FW&2>UA@DConiCmNpYJ~Deg^V@h%e6ho2J@dJkByEKNRf+~U-x7OIy;=U+X`KcK zQ!qjaC!|mbq)>ZAo9;Gz>Z;}Ll7KiNl#{M2u1a*~D-*F1%N(=LLFYQC;-S$v(-k`K{FLn#~#d20f) zBPEP&r?=FqO-CfhWOGe7)eG(@M}Ap^L*(vM=DF5zd2uZi{{)meZG>~gIvRtSu@PjT zDT_o#tda4m0|W!8rU^EfKnjt5GVcL233{`ZEXDMXq9^A`Fw2S{%itr9QiZ~#K>#Gg z%cF4H3&A!kWDq9@QBgBgp@0;&SfDOM3L4?qva`c#96gcPBSiEQ&=swXW)x_-Y03`#NojNCvA$EGoRZYToIn-9{vNuBGydp+ zHH$3;5wPKC6!{X@p%Rwm-h*(gZMGaAG*;r=|D?6JW06~l6Sd)dK@`vIR^E0SPWW63 zTbZB3v@C9f1Lg`wlL3e4WzoF`yXKw^dNs{7AdFo~F0BCIkYNGU(=wYLdDnkc#k$nG z?1E1-fD9u;503^`6~Kl`t94Gbc{vc)s%`2uCv-Gc`fa=KHn!wml%v`}@Sc4xcWA&HH<~Ijlf393s|lMIi6YB{km89O`WW7;SKk-w z%$t-eKF;O7DZbW_ z{K$m~>=Q!~yY@dS{!U5!q8NQ@qy#O50ZUEvNGKpSocuBIY-bD*?q(%H;O)|hZ;sGtyvaBb65Ss=xwr?6l_ZbD$83#9|YZ0L?oj+DaAY#6y3K8l8l zNM9k7hdL!(rVc;!S~2n6*xfBvH2ryi% zf~Sk92Z^UKI!*vD)WjoBscB7Ks10T3B3kwEge@$<%tPWl93#WARPZf9lIAFmJJIqj zcmPrnNc7r7AsWVfwh9$a|Lji`A5h9+ZmB-V)iLZp)R%oI>2`UEa;O|cQn z@TWiiEFxw-C#3Xrp`gJW07AnWSB9eU09|B9RsqWdU1{~RTFu&LuIOD|LbQi&ea8Vf z0tyT?EVvtJFCbdg4i(hbppd{^WF3f-%C>Zk$b_k1V`(>XW^9he+sR!aDq7GoI8>y~ z#Um{zNTJHCs3J0_|7)$K)JAe83Z{iB%bufKXB?vf1GB_?gJwDRz1Uz*DLT0k zbgj4ufLlevKK>~90T*DW5RkXsGR9P*v+V1U*@Pf<#6u5cXeoRIaaWc+%aP)+MB1i944jqB3<)O0 zay-g2#MQV+tLt%c1jO%-H@yEAuxom)WL^D2)mEHnPy6_bCDLmj3)l4{c5`Z(a`uoG zx8J&ty>UJwtF+PBrkmj$xREpmk+G$ASFGL5C_S}Yc$(@vk2>eC;_qvnBe%KVJAo7t zk=SBct`T|#??_ZY;%|kkTrMpc80`V!{)YM#yKCyd4jh+e2#dd1Gn}68Eu%LTdJ%4& zq3E7MnCnh*8%0)NIo8j>4c9ow<<1*qnd~%1bvA{qRZ_J|d*mtK%F3~+Z7YD-<#+P- zhO~m<{~wAKm%%+Z({^qRap+UCP;+Xl%$vrtrlXok& z%S*lw?1Z}exS!Iuyh9G6a+-^SKnl|*{zp7ImBQrgVinj;|KwGS}?#i!KiMKXXI0no^@)*N6 zgajbOFA5Yt4-A!5YCAy85d|2;Ym_DtjId>kC!z926ig=;+#W$$Ds7vTJp;ML|C=Fl z*_qrTnr+CnQV1Kz^Fhcfw@<8=>1ekjFhUY2#Zz2DPH{8|$iRV^q7&Q1J*+<(QwDaR zCyOew%YzSkl%**kMujLn?)pVcu$j}7HJ1QMF5|o0djN&Y1!j~upqNL;Lo+|z7{BR2 zY?R60&_?DvEpJRjbb>dO6UP-HM|hl}x`-_sB#Z33!CB12QP_-|!^iUTL>bw~B0R!V zBmpH1fuGO-s%Zk^3Ai+9w^t0kSmeYrQ48O~MTo2@amyM$e7#7xJzVmo_5dY)QH}*v zHIAsm1aygCh!_V{iM?FL@@q!mi=B*&$)S)&jd7;gAcn)7$;W&J2s@{4|1?4G07u-& z$({^6>*I#eF%B)tNp+f`c09@(%9c~O!ny($GLt;W3q^cW!q}8e5D-XuA)Sb<0577y zSlkq{9D=kWHPrjR|4X}jS|V7(1#fDJeu_%JTERH{0;AJQ20RUup|P~Wv94RR>A5d#~K7Q8Z1F}G@qDT zL!#_76-yUTh$17L&Dc!E9~sD*(yD#TB?;iMg?t0rhyW%ziyVnPAeyC8Gc{84AuHp$ zhfI{%>w>Jj$5g0G)IboP8$4eS0245Q29Qytki({7KwSXBU@Ffl|Hw3_bIiQ}&+z;; z&FfLMJI^Gg1UL-IcEq8|6fr=N#Q2mMJo`Hwq^s57Pp$CJ$+0BF3r$7{N7DS3$Xg2` z0f-PtPz6n$D>$O-d_O44&HT7a-V`g}?V*XwP8X9xMG5g|NhTw8TFnfMs<77g&Hl zebi`O)-EN}hA0#Cc%~pszcnDjHC4g|&5H)biF)&Z4=7g;|EL{Ru+t$crwNGGcMa8e z4OIak%kCo1sM9*S6o7pNfPIzMNF@LPXjYzqoJ)PkL5)^~*?#5M zc!gI@eN>u7AL82-9~{q95C+K0fXfggC3*mF1%a!ylo=z)UeSP3{D7&QS{3+!pAAbZ zR4@lvfO+-ToBi6b4ci2mBOba}@Op>~Wr~Ii*nV|eur1X`O~80SHAoHHL3P^#U;vxh zo2;~3u!9ar_q@-4CDx*em0e^j1cm-UWeODTVR-N6~>tK+Y3!*#JiG6BK zGc`8~s7j>`NO0}lB$z1;@JFkqTB=>Tti_TDxZCQDU9dF)eXyzg^EWVPhq#SYXrf-X zeOuW5*GT2dS<~0+&07J;(JH%H@^xECK-h`J+k* z)5=9)eqd4z=~CiAS)6pf9HPc_FqQ9Tu>>}u#WUGJ1^7Ww<7HD0|9XS*Kr1A~0O$PxA@%^V3d{X-02VG` zuf^M&mC+nIRJ63uhfE>!eOS6kV)?b=CjMIjfUW?@OTOF6`UPLFJykaZhP-X!w;h0* zwOfSpfa1E?mcX^e-Pi=Sj>j#t$jv}#!s9=N1rdBM+IrH=)wgCs6tt3D1vtR0L@P|1 z;1%-NS)AYwen~=mNv1&;rie2=<>L6|*YaIr5|+{1ZDAH3)L>{k&kGZcDFr@du^-H* z;??2UWI|5NmwIZ>I3fj81mfp?UJp=LAR1pQHsx zjW?v|y-~XGd|5;YTP+q_QpV;2Fy%EC<=lnU7(N4F!LtcCMFfaE*`VcGX49vkkI{is zQ&<41bl!4>ULRu;Y_?uA4&^}=2E$wo3YccG1zbk<0zF;g_?3hMC}B`G-*&E8mq37H z5S^`#+oX0@J~eB2hUcUX<$hfN2*_Q_u;+SaOBfO%QRrub_F>N2<3G$~y!Pu#5acb@ z;E+45LSE?mOpXu6Pxf4=5aV2K{}Y-5lZp(BhNsqQ6h3KrjnPb{*tK410fNw^>e#>H$7Vh)KUKhB~ zD5hfib6BHhX?7Upk(TDmE?-op*|SFHw%*&Ru5H{+*nP!rv<7dt_UyO)?g_BxxQ=5c zm?JbUyHOZ{5%A(8oylo*JApP}f_Bxv-ng1H-P?lF#CB4JPM7E--N)|l&IK39E}zkb zDh{)xVVqmbM&T3IV&Mj5+#PR{=2Q$i2DyUiQqXN6>>eSg%9(vwH4TBVHCTH|gbe>| zP|hb)R6>6=SKm%e?Q%{h|Ay+z-to1L(WO4GFj0UCw_dkh?jFBl%vN%hMu4++am{nt3DAHq?kTT=fcGAObrxCuep51jJ?r#SZjKgltB%9{x;SJ|iFLk?;wpa3v;TDj$g3b#csQ zfZ1>|veB4-JVgwEJV9*$3#jcaM(QTMO4o6Kwq5c_Pipv)@u^e+%lINysD@AeWFB8> zdA?aSG0Q5>b$6y+iH+M*&Qw8F_4bu#k|u9>wg9wl_GgCxQ7>c7X6q^k@3MCC1-S0B zM(?rwdj1#~eq|3`Bdc(nZP4S?S34MeQ=yBlbf?>V=Sn$$MKW@tZua0AD!3LbPp z-(c(j>7nv(1+S6$%=e!Xhk#Y-M_=_4ZgL^mfIzKu3Ln(H%?U#L*$?;SPWNjHqU;zdOq*CNlx^_X92EvD41*=8cI>J*0820#H6pwWqEc+cKb zW_R`~SN2D5a%<1_r5}1&mg}dtj30V>HUwicC-?WhY;rF&znYlkIflurNFL=(>K1XVVk+yxk z-|?TaV;FsZHQsPUQe`kd1reZ1ARq~an93w_R457prn3osHbtPS^>L9rw%o0Q00@i6 z&}g|ic9K9skvGzfcL!u>7%jNx_xe6xPk=>{MaVTc0cGO!#OS1Ff)d$Kpjc_rFzF{o z;DXS(|G?lf#z0yMaf+Inx&Q_M`kD|68ulq$YKsf2+v$?Q@e3R*EKD&FFp#lP0Ws3- z>?Rj2J#CmpeT|*1z0J)vVFdmyp5%ZO9*&f*?$F)t{th26KTlt8Pj7Inzkh7t{|`{f zx^e;?6g+4!p+Y784hVE;0O3M`6tq}8BvC=b2pJV{bU0Dr$A}U~b}YB?V6qAg8G@Xk z5ugDuj>y=wd4`dIe8vVSP?n(rzX1kr-dyCsh=Yw4G*rZ*A*i?^REJz3=2^pdqnX4ek~?e$47QgZHb=idXyc)uKU>8J$fOO;UTu5+%iW z|L!aprAk-i$)+mrx|}OBW?7y(%lZk-gXqztLx;|AAp%fl&YB^(Cx-$DYjR=7j%$0{ zxpBPBfeR{i9lG4aiyJ?VJh|g)EU!8TE=~x=EsxC#^B^CQXJt1|1%pNDi~X* zt=>vT2s)3YBgtIdRY`|oRro*!^tw<+j0f9>tJMRIo8FXXPxwrJV_Q{}1bCKqs;{q8#^-ok-XCc%FH4S3*j7~Gt9hC?hT zo#jzy5}_PTv?oRjHJ&Hs6JuN`;u%ByQ&0{AN>qH(UM+0ODX(_GrpDw`*d>=7d)OWR zYDYT-3bJtIX z+SD7^3jQcWZXKBgs(?*)`afzrcKc;(>r316m$vRT7r=d`Uls!V|5x4n|L=|h;fr5k z(%aoa)Gk8^Zc7Vnpaa`MI8L3!Vvkdtpg8t0hAAO7+wl{fGzK2bZLB)r8DV;qkgotm zByxPJRZU2jp_Fk789_c@nKor^F?qN0K=nXjYyO+ANt;?zOcD(j_>PM z70=Z+W)b5W&r4SV=XECkap`~r%o~`>Q9yC@a2gDJpd%mo$TsPtJ$bW}<1VSe@*ECv ziHlqZ52Y}XY;H~$W84YJvq=Sb13(!tmHzq{0VbxgEGByo{|%Lqx*7K50F-EqQCPG{ z2L2F;LbQkrTi2DESfQ9+uv&(m$TS_^ge@ZAUb&$3mz7}30(-Dgj<|R%6c|yO5~2k- zY5}fUT+MhYa)GtnhzfYh6OQz(XB_XzK2pR_ZEMrxFc{g#x!Es}f()iTDl$Mr7BZ2+ zFjlf86rx_Sl;tdMqOzW# zOm&@(_ zJ#tPf59rJcVm6_*&486`=Yyv`@d?K|(s8XmdeYS1|N0B3xCpL*oG1W~!_b8yQci@7 zfI#gE*uXkclC^|WKoXb9I(a8O6@(5;H|fb&{^bZOJt;~}D!0r|R;89j=;&T*6J1sc zC>hA5Xl)iLM(&9VQv@QZh>DQ~E|fTwPApQbuj1)e!P zq>@a0`836E1;Exe#&f!7t!`z^iq`C^^%SIVD_p-zSJ?W|pvZYBdFhc*z2@~xjXW>^ z1PkBzMsSkv@dr^xmDv2IG9{NaXn!wTj{jm-z${cv1uFX&pV}0(cR1}WODo!*ESL+bh$NiA!5Ao5=6aPq$X3hRI?eBpzmbwD737^+&L5u@s+*2Cr!C{gd$U#FX4Cg4HsJ5D1EYEE&5f`J z*;iJdL+uWQ8N+kakO$~m?r&cptpk9tfA~ak135`I&K|%Ew9@1801zCm5)}&|E6^r4 zy0Vjb-w_UA0yT^f&k|BLN(@i{-k>Dr%tnEJiL%j zztw>zkWbS!Pt-+qw}(C|7-TKjb-;%*rVT48qf*^P}I=CDT*Z(MRws)CCdF0@CPzHqEa0KmV1ka;$iAGMiLULgU zPMBazJ>dYP1#M|$0OLY{+k`2dCj<)^baNEF*ir9C32t#+JNDbh3Wy~gVp@d84fpO_acoy*%fwXEI@BuB*G&(me#aDIP#l5Fm8ds33Gmho>ND_4fxsH8sbzha?kxWFvwdxQLk8Rk&bj zCwM-kkb?DQT`U+~SqFpVNdJlXn2GAx8JzfCH@I}vxQhz6WId=a2k9keS9YbS1TLV8 z4+)VGiHWK7iFXHuwxn`=cZ<_O8yiPv>X#c?h$hLy(OjJdZ&1~U@rZ)b zg`4zfn^<|3lGv47>6QCvoJoiTQ$s%i2_vMqmJd*np}2NxcZIzcYi(JCa;cr$$(^lX ziI5dZd5Mwtr8$t34jx&U)d>$7Fgs?KJnwlL$qIygsF?^3I7j*rjCx8mdK!D8YJ&TrZRp0Y;q9n|}JCgUYQ#qLm_gqKPW5;Yx%1@3LAsc&X~iATX8zeKxBHd$1pXfBKnhV#uG65?nwbto-CQ z{bR5QI%2P4ntV_NB*1mlN>+JFR@rKT@^-BZpsn-Ptt%_7PX(^TIj%8FuI75Kr0A}z zDlqFhsnc1hs=9m+r2zFRv_ng@uCWUtp+Nc?gzlhHegmt_bFmaVR|u=7^FyT`z^1KI z7xq^lqywz%DF)0Dfagk-#}$-&&>`srvKZQXz$tau3P;(RvMfutg^Hnu3bW*@w=%1Q zK|!`<$2|6PKV%1A6DOVQ>R0c|st=H(MeDeaTeOv^v_9KIAm^T)CACzmxtoiup`oA) z8vh1&m|SgY6g2gp&~^-CYlFk!hWT=V5BqIy3#eCTpz*k;wM%umd%N&Bx3jx@zq_r$ z3$DYPw|u*|HRz&5_^zhP0--VmDkVlw*&HZi9HY!W*(Cn^tJmy8v9b z2W+=^tH8#~z_9>oWc$3&%Pz??S1)406|8)aq_c-hqo%63w-K-1E5akZqu^)Nmh><9W z@=HEf{54cettP0u@AyvGWx!gzb;JwGT)ejpoTy_dc47Ry7kt4HjKR^X1%^wb9gM;s zTr-VJgQ}DOaIDKCEXU^+tIn%McRZ_KMzws*Pc;0;77)nG%*@Rk$nramh#UZlyvUem zT&wo6_I4s`Mw?9hw$*ilxM|6=8;@2@$^U!KwgAbY%+92ok6!%6$+x)k%>O^JjK)TA zz4Wlg0y8fLQEa;m(2(29SoEH#e6`bvu$f!6c)TZejIoVM!`t*!&OFE!P0?})0pmzI z(oD_#z=zb*w$XZ(H{)DgQzRMs%_Yd8(`wGUS=YIQEQ|)=A4Y+PDFY)b&bhEnLhBEzC{guY4h^YeU0Ol7JGe%onZITiwi5mjUkU zbS`BR99?u~5@$?a4On&Uf0Z z@tk}aOwWhiq(Uu(mJ9T@P){v#~!>G5>B;Y|)|J%%UC0 zB_M}G*OOlC0ZkXy@=Mm9Bh53hiM3Z2f8BNGW+XI`j+=2^;%wJ#TP-eK*K2DE1-#en zY}=RtRe-&~%^lb|ZLXu~)2i&#r_2FSmTC)ymg-Q{tLhP%O4;GPsX=TEPo2=34bhkq z)qKJLT?20_h{bQcj@yOFBnYh=AXb^4c&V&6BqagGfF%jsMaSj?VH1;Rn8gNv@}N z&B@0t;aJSv6z<%In$uezT>BKp)Ex{MPGni}N)1H5s+`@5-5VeL6yR;*Z!W?WW!{<1 zBvp;oS#9GK9U><+0q^bJH;%G?i{32WJwFcQh^$Sygvgr6iVhyv2!3zB9p!Vow!B&3 zC41>03%u#m;8c#>S02h(p3})m*kbpd``l|+!OFv+=1!*J5d~nm4Ck}Xy(YXVgzi1; zi@VC4=QQq_3((oU9@@fg<0UWxJp#|A4kM^#=vc!EL9XaQyxX8$f(zb)OfKn_p3-xT z*V>NPj$Y;A4(g(w< zPn_T*`|X}C?hiiU<6iDse(vUO%B;-pgl*V?r|PS2WUcOes2(@(uFoO9??3Oo%#q?S zl5Ka+>jO{lf}8;D6#>Sc3J#C!jjHU!0L>P9g4N# z@=bo%O0IREj^yG_?r+cXTrT(Hnxe%g@6;Xhp%(Kq5AW)p%6<=C2Bc8VS4jBY&$15m ziLa^st`Bu?)i#{;NI%*(ZaxNoQmP&GnZI=3CI9gh&y^r(Nqd>ukOkP#sJR18lb=qNul`7 zuZsQN69OOk%slMGUgJ;i<$xQ{j@tXmFb-(_t0+3Ix-k0qL6ct}`!C3dX8-7tj=+|_ zyS@Fb>OcE#Px~v6`}wc`D_ZwJ;1UiAL}M|+(I}ZqCkMKOPN`LEbSiCfE)VY)9F}OF zR_*h=y;>`dNd*3u$LDp0P!LZK@h~ybEn%eOF;a4pvJ%n~GgI?& zv(xhv6q6u;bb!==faOG$b=9>M_Ek2Pw*Lg;7B@FzcgmwmLopBUV$IjM7x}n2nOV7b zW49s$A_1ZL+S-}`fV&$2341|&oZ-5h939=Dls!NJI%A@}{XE^i9{(O)H@Dy7_8-82 zx$>Dan6IBdf&Umb9GDQHxP{arR@7$^<3WuVGj`0T?_)@NB1wiTdD3LIAu3s}blDQ- zOPMig&OG8`LWX5KdG?%Q2s75;xo-9P z_0uSJ>r&M+JImDxh_}=-)Y%NKGYvuy&DDYwFHv~XDr_8wQb-X5YbkOxoWRsPchM>+ zE`{^3N5T*vLM(U^pF)S5Crgglu>Y{<%cK2*{CgU6Y1ODx0)}X_b!^zPYu2WHTc+Tf zMRfDNnIsJwCV8C4}_mv3azlUM66x6YKrNO>n3$4G&UqtNzQh%^57V{o$wD`P-G z(KN~hGz@UHGTpLlg5^ci`#S>Gc?SKiIdlAMMU!;n<>7a|s#u|k|?gHz) z+YZPdg&a~V#Ksz+EVNEIk4XsL@&F!Zq%>;EDP`;l8vHB((ZepiJP1Gl!(+_A00`i4 zq6xiBNWu)&l#EM^ChCyRI{#NA5l_?ZG*PyRuu&1v6$1sdx7`dq)VD$nRiJ_`%D58J zNN3bB$4Zy7R7VA>A(GQhgY;CZdDcm*t@L(z(!NydTUAx&poCIOIs5sN)(`U$Q_RH* zG!Vhh(yVYpf82D_GF$uWjKVpMT{g~U?Hq}<)L0X!usp5hQ_yS&)t1FZCH)p$=70%? zQcTUwF)9ZTup~54KXn)0cRdj>zyVH$hE*7=WJb~~wTz(FfSnzN_kcU8=Q1dwfzY_%tk^I>YkcJQFA)0SLxw<{kVcEDs4 zQSRE|#tr4%Pu@Lc0_axbm&b*7dECH_2i$?gm0z6sA012D=x^2AYZtSvhx&Su3`Cgo zHbE{Ods)E-9rZdPlsJ3V)xZ2{*Rg$H_S7x1g{%I7vI?WT{@(8%N z#t|)l_hB0Q(*M^$2Hxoa0}#&y#lsl!=mvaUYai)KN4|zN?N<+69sJzaIudHkHYZe} z{8-pRK{1LbcS+z3nhpk(0hi@pVYF;t02xpk#S28X*f~7du!!jxmyun&BcORp>=aei4%?6yqkx z2piq?E@dxyiww$Y554T&}(1Ifl_{|wKnlZF^daw zWFbAI!T(p7C6q3lu{fe{H~z6xmoa2`NbMU6WbHgk~H5c$?7y(}Xh#if)sW8D-&_z{xUkezYexEN2?`CWUxZpqB`8=R4)eQcm5H9ZKU% z1{SIsB?6S6&%|j*rszePBy*V&l_)kDN<}JO43NQu=uU^))Qfggqnk_uNU55a;*At4 zKFQ}eT#Cn*e)X&LG#C|(m>)1+=#bWQs#24RR)lVoAiPP`Zsb~5yWSPAd6jDjc*9rU zK>x9!T~pm!DN0!Q8TP3dRcutD>djETqzLyL&Sa}nZZt4$fo6<{`neEN}~pv_NL zNefo4z_b)IRhL?&nO3$I)wN7r)<03hJH6f(x4G5rZgY#s@7=VKp_6S}uUS*XE*80r zRqmjsT3P3=!3kH*Y#@knSr|=&0h}`JcDF0ms~y&tY9;P)j~f%-p7*wXMelmqi(3x< zcD&`4uW=Kr(B$5Cx%s^v5*+p183w1n(N*Pifugs)0q;fAC@ptc3ey0Q_q7oAhtF7(}aZ>D%KG7xu{{wrGjAqI8nz^zzBd@10W?cuuWfr*Oqi^iha-Vxi=uWq~*)3_SH5uAYmj83I>0R%= zS{Tnr8bAmF!0$Xe+u2FpGqr_U?Mf0{qN>LB!Wo_|kJ?(>Ue+>#AFc%HejD7BeK9qF zP3|=xDcZw!iM=-+7?RUD-~1j$z?-e)c)vX41y}g0uU+$zbu{AP^jnK_jmFJF&Y|P> zc)QnKbfV93(lHO8N>1K%r{A^HcOHem`@L_g=UKfUk@k-b&SRN(o#tV;`Pggs%~Iyv zvfj>-6!u2}wLjWlcYOUKAlWoX9$c4?!rOcP1?9((ZwEFbqv>Avu6S31 zZC3nJU*CQ2?^e`>pSr#Qet6w)UG*RiubbfA_}~wp;gUr@Rp&w#GVqNIai2e2s_=Pp zzf1H?F8~Ly^aRiVF9G#T;rwLd`x>zO^vteO?&^>XNm|bRM5y=L&(!9N{oc>o{zUjf za8USb%O-CBP7nqA&)2N)|5^~Nux}lhePX}F2277P;5vxHiFyTB9 z{+8=^QV?&5$VxgddYVuPvBU)rPzni<3Jb6buaL$PQ2Pw)k^gGK2EWkt3@_lWjtu=x z{QgP^9|-I+a1Cb%sp5wSMNkf*>Hts$;+_u)^RUZSu=)rH2Dk7*dg`sfkPsbE&&p5? zvrg%LkP$Cwu-I_K=1>x|K@TgDCzddmoKR`-Lfk+P3qR2cLva8TuyZ^kga~3Z{H_pJ zvFW0Y0&CCTL`~Fqun~8!4K*+gt07Hp5e|DX4vP>NU849n3wZFb1dYQI^Kc0_v50Vi z1)njGK(3j*XAlv96|eCD$IlG^t>pxx7JU%?7>_e@F%pB(99IN}k}*=2(N&sI6W@^? zeIgUv!-&j-52sKR^AR8Qk;V`pK-QrYAqKc83>ypb>HiE57Hw|=0ge{Q(IK_T7N4pE zA@LU{(ISb_IIIg9)sZ7ji5|BE8XYY>6rc{drZiO1AXC!mW^V#xQRTYv8)I@2FDXLk zgd#uCBF|BiI+8fz@e+M<9^p~eJntSOEeb&qDfy8A1B4_QMPgb)C8Lt$qE7Zc&Gr=1 z49Bk_ACfB}vMU*ec5bpFbuuT10Ozp47nI9pLP^Q}0yQYMo#H4o~1BJwL^6FOlcF>kX~7IQa+(r&I3ays$RlF~bC?A?Y_ zJm<_6%WoSC59@N!`~VX*mvc20E`;7wlg2VSWwRK}%r>c$|MW9lxUt=Khr9`1q4ZXNdX$6rK`r zMVA!41}_$2?+{lmL6wtA#j8qhbVnytOaEVDHZioaBvV6~ut>@3AO15EN_0doL97CV zN!yem>q-J)k=dRU_g11tu@q0;Z_0d;PqUOqx0EK*@ecFTj6Bp($8>o52#$R7^b*ER zA9b%t^A%ZWDz;7VRb;4O)t4o zG^bQSGj&(9l~?=pSAEsw1XWy;Fih3a13ti9KLA-l3ND!^P0#c)UA0c3wIJ1NFYC2h zX;fQnR9_S8;4W!C_q107wp%%gH~+G18MrMijTKyc;atzv66cOlpH*I^vPC&9SFsgi zF*RTD^*IIBV_RrCvr|K_Gbl?oMobnH%e7(C6~(fILLiM*(=-6#bz*H00-+S^?3G^Y z%1$@dXT5G$|Mg>sc3^{%P!-m{;^Q`zwJ2vvVr!NVjniITlxGKyV>1?Oe>Os$b6bhl zTLbkhQH*6fRCvbLRNLc7xvL6k7Had7YT?v87ZM>WP~Zw>psrSH-!pIb)oY8^Z(+iI zkXCRr6h6|{X&tp~4N)}DkXGMxQ#CegBNuY_wrjcdYxB!Q!ii-~HdV%zLj~6u8rESA zw_THxZ4;N?WOYtUb5f}l)c>khXt|bhXSZ_u)^`6ESb5>>G;SjY*LT;EVMP~Rq2!&O zHq$&4bsJY_XVvAlwn1qZdSw@HX|#3$7I*1qHp_PZ%=UMSA#}|ZZDF=_*A{V;m&k6` z$VL;?#1B`g(kr7Eeyi65h46aoS7cmdIfk@*_t#bEBUNQge2v$9k8OEx)>0=|auXP* zrZ;|X*MY6KWR8$JIhSnPl4SYiCvmeexpP%bw|oZ}-|}kqWHD!7^d;x2f#p|)ADD&h z*M&ufqwe!)gNJ`>_;V}MOlwSlK{&$>(RB~haTz#;hggAyn21}r;m+zdxmRH~w}Pi2 z1De%cmGX3X82r*JA^#1u*>ccs`F4fB_=sK8i;?(pUl=ze_(c-dhSzu*npTd|ZHKK` z>ZBDBt2Ky?xQG$>c{MeRp_hfoxI`?NiP>0Omj=x`nDnYxj&JY=C)E_r^GOGcUej2SC2hg6LT8I+qbhn@9<7nwzIa2qSuMa?g~^4OIx8I~~_kpI|y93Yf$ zd4^zRiepxX$HJ8LiX|agZYvdbW7(Kx8JTBUnd>)f^%s|)xhIVQ#y)YENiU9nnee7= zYX9z)^|+h8`G}?0k7IY46$_fr8F(AhVO2m$SXGffQ&vwoef3S4Rjv%_HtK@vm?@c` zli8okS(C4K82@V%oeNqX;X{|H`8uvSd0A4MwNX8H@M8D*n_u~TDH@>3(gVZRpga1Y z5qg?=`JHJno&_)V;5L{`p`I!AG{qT@G5VkTHKsFqlLNXX4A!G_nnR;l6swt`VKqwc z`GiOKj={O2je3v68K!BvoBG*A(yd1(5uDTr2*Tr&l`%3z43wY(q(Du5~hgaLFV>`Jo zTeF*cvo#qUI2yNeS{XuGA4~i7fE%TUJFg#Fl^L>?nVYZ0o4l!ax&Ik9_ceQ`yRG5l zU9p;^B^s}U`mVEbi^2Ptlv}^c`@H+RlF=H#L6*JQTbjKSyT33>g?p8Ydc0pd{T#fq z8=Sw#x~VA=jR!oGJrBk>xK-gB!?atT?RaNda+MyZCr}xm#$*VdyeR9olTxJV!vF|N)aTY7_x+UqB3@2Hp_gu^?oy;%2%rSR1 zE_`>3V#X^G&1o3WkKvx!LW-q43wM0)j$12pFrtmKm?>S>BNxnP9nUXL%Y-s{nSTW zW)D2mw%DF;?;~WS{o6w^m}LmSxS2r+EX6oi(b}g zo9G`NTcq2dA?`{FJ<4l-05`sarCt=py}c5=)$LiLMV`{^yxsE_sU!T|!5+5De&{K@ z;cv&OnI3~XujS|cb9+JN@!jfg9`9`)=T*H)rxn@TAuH!xD@Wew3qR@C{_qn&(*-+H zv_0-o)n&nkJ4Tf1F%w8q{msu8??o}^*))Vx?(ZS{zR|GsfB)#yKKPZM_~re~XE-}qe`&b=^-afgMtOW| z|J0@5%}XJK>l?4Xd%>S=^ur#)y}$cqTk%I{Fg_|>U!QBHpZ3?E`rE&^abMAmUEsUA z>Pj4lJ1vk-fBb=;|A8QZ5E6&RBJ!wAGMCOK)D@jRt5l~nYrSf_Mlbj)4wJ{`GWx7e zv)Ar-+Ogr#>-IZGgU|2#eZ3#Y#voxK#GwVE#mPvcW65KrBqU`C<7H(g=B8#OXD4W= z=%?kVY3Agqsw%-iz-s`3>@00<2*K?@$Uy6^uP*PcFfQ>hZPx-)^8fM{vvV^v*Yk9A zHTAVM6!vyDx3_k;a`-p-xcL?OIe0m{y0km|yL@+BJ)AxMyneGnMc_ZcfOrTcD(I;c zs!^#HY9i$jA;g9eCqAr5@uJ0yt2A;f<-!##S+{BxYt<{juav+_4l_0jOuKzDe>9Z%$pF;E9BU;ob(xXa~GUXB>!#_ZL=tz|+&x47Voa>UGuEjbb}u zBs;e3Sxv2A#Ud$^7_eU_S<-b0lSs^6F-@S$xBW|qN@zBU{ zBvXE@#KYChehiuIinjA-(4s?=9&Nfb>5m|J&En;H*Xr1?T>q&IAn zd2L8pu1&sXsBmOn)9KD4RMhpeU`irfDm#x!PI~tp%kS zop-_VI_$2x8hb3G&%KE3qRlezo4zEc489vfMJxe+G&2 zJkRmGGXm9C3~t6lV|%ouM1w}!xapp27tFA#JpVPlRL^^L&NnCfu)zft>~Enm9~^Pk zRhw<~&uPOWw8b5}y)oQx%RSo1Bd?^3m!|5iHs5GtZFAQ5Cd@0sV+Si;J!Z{K?Z z-Sgv-D=RtWlV8EM(QPBWdFGr~Gy+FSYaZ^kO>3#8i;SP$_v%h){p!|bM=kHeGe6w2 z<3g$4I`3##{=3h>XM_3ZaT`B3^2d8vH-35d&idE0JH|Ej5I_Gn^x0D_JcQhPZ~6CC zgxC4Sn3k_J`sZujG;7NOdVP-4%T6}^y~dtkLHX;yzwLzIYU}*Q1V6tGPj9{;R=fKgb^2q;3xUC>?OOW+Aph{6^AE-YUm zU-3u>L($DJbTPc)x5l-*9b%7%>VTF9tG7YG70`!8JmLtGXhPzsu!%n*9t<ylZ$Aoh;`29~;vkKc5ji|&iF42i(%pD8Om_{|SF?drL9~5yY$MijrhE$}ZCE(+V z_l<*t_X}Vcc^F7BBC(KXG$bMq*`GEt5`n~F;1wY#$w*F8M0;!$tR%*=y);shCEOkf zJNe0 zoK7?#Ny$2@Z<)qBh$V3u%pIvRasR`-<}{s?%`y`6n8y63Hv`p5T6(UWv&17hH>XEw z_OhMW{G~hN`A&ImGn~a_V>#j|%SrOnnf}yP7P|?~dJ=S=_B<#z5vtIEE_7!E7^gnX zNzQXhl%lj;XF)S+P9flMB*c}xzR|g`B9HfO{Ozl zl}PWI)0pnmjmd=QM1u-TW>$2Dj=(5QIjYp8GIgm=MWaA@s>TA9bf5pc>Pqi;)YY0;SzH%DboaZ+W+M8mw^PD9N zVIK>+zK!Mi({xy#A+~#3JI@hGuHL`cD>;(7P*RBTkjuXOYTT|Q7 z)xP$zZ=GzOI{Vq)R&PMJz3g#|TifL(H@d%^>U2-8+1>uOaA%F}c^iAr^mZ}6*?n$C zx*Oimj&Qh%ZU63p+gsqf>F&O>&AnS5*Osz*c*0ek-+uf1-OiSGy)%AqjgwUYwlshO z4iNGK48Q;;wd9d^$RjX;4D|fx2ROOI!~Ti`6us=JWxCVT?&_Hn z9OrIVxY^?_Jd(RY$t z_MuY{M6GY@yslOFI#7rNyg0CukOz4f5~`roB$^x4y1-TWpz z=R1!%#Q!h8_t-o*2kf1=ijJt^xn`p9h**`URf|y59&! zj?T>=3VL4(zSID=;3;*W|9v1N0AK;upbH8hvZ0;=t{?;MpaRApszIO-4q>{%pbVNH z@0B14E+L=U)#TM56!KtQ*`Tf&;SpXT`hlO#-5?cmp%>Pe5BA^>_8=J&p$1kU8ioM| zUjHB)mO%!lp&2%a6=ESBra}@jAs#m2ZY>@HuAmh9q3eC2Ao|uE)}bMKnih89ASxmb zE@FvvUl@{M6sFt;R$?5Up(XlNA;#d-gHlsL(V-dU~Jc1kO{i8twWZTu+Qb^_Z z$)7^rD781Vy;k6a$i)AC0VjXDjw8B#wBJBr3T`kVIF29AmvDg zW=-a0aDZhBmYnw4qGX;TWA0yPf@EU0W@B2WYzkp6qNZftVi{^CT=u3f`v2xy0;fj8 zWm*cSP*$b{&gE+^XRO^N`{kQx?xiZ0nsh?vbf%+e+8=6?;$w1WDZavNh9?9vr~k#~ zb&4l?q8f6h=bLrsc9LaauEu(rrATF_aoXo)%B2*LXYPR}X+~$3l_W_*CCaI!PqJWb ziYI^8WhIzrf<7pF64wk1`-_k|=(5=!lA_kftMN)+m|zCxA97 z4%XvnrjlP8s0!lfj4J7v-RO>PsgZ6p?X*seCcG$pF>J2M7F7{Mk=kc zX|1a1t>)@{dL*gxpPWYHrrK$g=IF2jrKetF$z2{S{A8-)Xs>pgsWz*$qAGn(VC8M! zv8JZ7mK>qNYFy5$t%fVEitDa=>peVcCn6}OUaGJLYZnp#>TN4zTAs*-+~Z+gxlU`% zp{txlE4AwDzgi#uiKPl=UDw@f-Ql0Pb|kottHU-d#NKMelK<(Lf#`@QVCsb*^8Cey~O8L(_s`D@g^tIA^SG*aFHnq0nqtI(Qj z(2ncaO03yZthlnH5w7i5WNf=CZO2OO)rOSc>ElX9X5D6<&hG6VQmuLk+uR1O+)hs2 z%HN?v;sL%Vs~)J~`W@d2Y1wvY+0!uChD=~De#Uj@IGzc)>zi2Z*;`& zjsdRrzU@5kl|Ux^2=XaOd8xa?G#z zuI>Z-sPTF4;zlq9TQH33uGcc~&KPh9x-SR2p!zZ&ypnGI@^AmbC;t8}3b(KeyRh&M zV?_=y^OCI$|8EWFFdTaD=Duq6hA@PlobFyQ5o2%>S1^H|F!?I+3pX)C#&DaCaL#%e z56kXlf$;lg@e~K11B-9@8Zj6#-`QvLIu!D`WBn5Aq}@U*SHk6wjV0gP^*W zGPN4Ab2zdJKQb%JGBLlhF&{HBC-VveBJAZY63lQXNAo8eZvrK0E}s`E%W)~6@-TPv zBd#(yw{kH{awIErG9$4!gYS2WUFzZ{&DL_nU2}0cCoik&7MDTYk~9CJvp$=%KY#H* z2ed#hGdq7N6ic%X<0>fgsystfHrF$%mj5uargJz~G)1d(6ZbQs4m3x1bUADE*!i6T z+_O41v_qFAL|31DlYZ&ooURb5tU<@ir_`n)C}jv^M|nt~E0%TQpH$ zbWyi7Qm?QE%XCdQwNsPv-pQXY-|~L)w0og6BZr?Ye}PalH9tQ!SR1ppwzF4=Q;dY;(43&$eG%_W9mkXw!36;w|HN>IE=^mdOI!|ATotB zt$1sAhTk`Nv$&7p7&a_oG)(&>R{9Q#Ex?ixAL)J7e89 zLv^HwdXS4csh9dI(*ia(`TVXrtGBwVzdEc7b*#(!tk18k-+D9GYrS^*pnrOz|2m>C zdLB>kj)$zH>$r7Kx~VTav&V;9JbNunw6Y^RQ5Q6!PrIZayS7t1qXTrae>=DfG_m_S zw~sruTRXaYx`qq8qPKfWa_ul1yB|usxz~HS8+w=%Gq#62zw3Dr001HR1O+?*F90k6 z000001ONg62LHf-a7Zi~kI3R+nQS_r(5UpFm{=VOCQ};Sa+?N&_lp#p4OfRXQ)#o^ z@c3IEk9hYBZ`n!&)8tU~Dk6 zXt!FeYBlUro@F&?xIlq|sGruRg)v*z0o}E3zuvu;H7Yfo(AwhM>Ml*54jVd1o=h3? zUiB!34~fD~ldYy0q!js8g$6&APSg*RW&Dz6^OY2M%*2lr1|!Lx$UP>B{Z= zH~0k(!*~0oFjx2Pt7X+n2U}IG*y+%z?tY$@EO)Kkvs(|3m3p?RVV$O5$6me9w>d%O z9V!((_^eCkTbeoGQ!-Wl0k~30I#GcIJqQN4A1*BU*VQmHMF`=95>`mzg%(;E!8Q@S z)Bg}Wj(zwECl|#+TxZUmnBs~mw#cH3FTxn3ZYR=J+>JE4_}7dw%4HmiKyD}GibKBW zY{=9y@wspgt&w&~`Z zaKp!Qz#E%oyTP{HcH6Cou_iS}sN_CLYO3g_TmSB) zpPIX_lawm@?!4$CN$+o3>(~modJjGr=ovG zyr;y0QYvMkh7LOMr}nb=a~d*}rQ+*yx{~Bevbv&#K_aX&h5Jyff$62+ zVzPc1qRv!kGtpuWX;^f!(K6k%(@;MxwXoQD@=pfZX6@}}AWi`6(Jc)jbIb0wJG0rU zc6xG)jE=2iykw94^4xC6E%Lq#Lw1{Rp*8IH-+%`$_}~u{>Z8X2$6N7Si4J;rpi*Xf z9B>><&S=}jF)nY)i~gwc=bT%P`EJ5NYPZ~HyL)Q7p*z{?h+ZojEYI(?NB>{0x?Z9C zw87jgt}s+DE&S9`C;Ke%#UsCK1)%`%M%Gwwy>&dyTSyZTbf=DX_N-(6cG{>vuzC38 zb8kM6QWpI0oB;pmcw~deFaP}X*DsmHk5{bmo`9CFvBm#u9HxeIB+;!(0tG}{qZsG8 zwW&>Q=7*6+Nvy+p~&^d8@+AH4$rHI5NvP%O5 zYXN2CWTi?iaA4}IpBTqT#xkm@i-&?-{tk5|0_v%ak1Lep1ZbS#H2=|nPFxipp9sM_ z>dADfQj!;=c*MD_u8QEJ(XX(8GiOQ2W?!?FB!M!md+|-l$I69O0vX}JAAl=GYvxsnV zX)5f=U~EUi2dxlVmz)(0b=bn4sW3yM3@9fAYC}-MqiPDcnh#Z}5H<{vC`LqQA}4Ca zZ(53+mkZ|=HF{2UNwko@$xNWas3trevqm0srb<`J(u3VkQ2z|PCgR%o#l8Jeac<-$ zb4>R@b1qSwdy{4~Ft-pxw1K%((E{#DEmJ)TVE<x6gdNSb5NplpEQHRzy|IVkV|~4Tpcx6a*B_0dZneh)S0^OvC3K={O3RC zSvwMz@Pzq1kch0sAAZ`lC)EOIS7&I$-6|^snV3jId6-JHM(>nFghoW+YT85&wW83? z<1Vl0xpK}dv{MA|NJ%m)m>R_IHEfKMx9<%)Ya9dk_lZh&rfJfSq;y)H!&)t5oiZEO zI_^S^%PjVOx#(ORA&m(Qn23+F7g4x*=Pk-}^=>+`8*rX&hu`?_gUyLi3*_-}*j z4&oM<&lUJqJ9)C)a?iTZtd6;%tC(MLraQELOy8u*gunG_AVw7<>Ai zP&=k%ou;~{PY3N!drGlHM;)BpGj||yZU4n2qPBVO7-;hz%;xl5$l}GmxGQzYT2E|z z4Qc1F$3;$eXpeXpB|fpYPSx!d+av(C)_TGLeDlH^zPZN%jnK7}?|_SJq*`2j(I@}u zG0~J#hV$$m?FhQ7Z&tcpmU@(|9LSY-eI&D28R&tBmpVGOWBf81 zgH#u=Mml`cK7|E-Fo<4y6HWWme$u6Wwj^Eh05?F$zn3Zp2Xmm;fj35O*D?VyMl-t> zW(Uz)6Q^ty7l8Y8VZtVCVdjLoWqX`)r(A*7BLoyV zHt2&c2Xy9FbM&`?D=318$2lb^bXSsZDs^ceqiUM+hCc{((*$tw z_b6*fa6#2|b?AP(!F7`OB>Lxt5Y>MKIBWs9TD1p-6CzyJHVMV{Y_j89Pa|i=o$AWQ)b7W(QzDP3q7m)M; zA7>|z?D&Lh_d5o%adDtpA;*A5`EgbNAz-*?(_>adlMNP#M2m)cEh%me8HBwAWyL5` z$&>*=_FfkWWs$~Xh2xQCNk;#Mb9o5~e@`-B-NcY87?yCEBQ~{*(YI(ZDU7Kij^GF+ z93&I3w}k2Vb|FHKp(r6zm@5W&fJw7^S;&Po#CyBtic86q-C_kQcUuB!i=sz~hbDL; zd6&&`l~l=5I!BfpK#Xyjc^kurHKqZU_EJQ4mb5uPgn}r0vV+*Dd6;)NhmuigiUyH@_=tt3bd<}3d}MWU zvFMb3x0&JQjSGp8bJ<)`gq5VJfl}gE6zNR?Br)X&j1?1KzEPV1DmWlnk->Ra9T-=Y zbtHqelINz74|$1lHIDzvnO!FnDttLN!IxYTCzxypNft+ehq-py2@Q)$g;kh}_XwVk z87&Sdd`o$rnYkXy*O?WHe^q&-FBxv~*`6A-P40FXo&#TmRhP2#So9X4NZK%f6*1K$ zN5*-hN>^W*^Ef#uYpg>^YdLT{3S&RFOKac&3HMMDMrP6Je^WJkhUsvbbZmZiqRj?( zL}`!WnW9jWlz69Ymf4gO2$)r43OGukY-pFuIhqf;qlT)B?50J0vWIL`puc1zN6L6f zI;jpre$k|c)!2rohNYHMS%e33yO|?WT9XqBs1}qYLwKqfgp;->m>TzWAmWq3hkL$b zTZn04JV}dkI;a2l7;O~zqA=Qb$#;gKDyTX7rwVi@%(#&AxsaGg8U7h$tGO|OC9Q5q zDH)(X50fR7+O2QWq)t~k*T{=m+K9J=Zf~QdFZM(}Qmk8Qqq@O_kjjy4JE@Y9Q#LqyEIEDIsIG9^rLTs&nhKLz8@fB1s&tctWXGWzS8az2 zNo|Kvnb5O~yMJiMri5FtNUO9->nwMQrxECnV>q>%X#^KYvqklpy-U90%dDfzqlhRa zRsu2d=BSGDg0E|`%9u@_H%A5Oo@_I_$H^%)7MFKhrFaXfGATtd)0dpPiTL`CO$dsI zo3H<85U08oG)0Sm(#xyY%d3=Wo_eRbE&6wyxwoOqp6HvtCfrL}3S?+2OMYUvYCFH) z`cf&2f#)Z#av7*8*d#@ol959?QX-e03%Z6%Bg=}aH|sL2x}iVQi3rk@K6w{!0Thbc zrfF&{Q)t17JFqCat8`jbiA(j zE27_PxS7n!5CU;c>}*<0xfr~Pm~^??o2T7|p1;#_q+4zT47?*O%WO<9?K*Ya#904( z$W5lh0moQmd6>ru(|yYLKto1Z+vS;pj8Z8HBsbNu)EZbh+=K7xxor%@=b{`(OfN~S z9zO$N9x7YZ@`SZh$)23NYC1ev{5mLF${-hyC`y@_tFXeFT*NStYW!%Sm$x)}zBcpNxo2`7?aCw45%X|k>G*2gN?G5+~KZ#%P(H_(IZbod!29wS-0 zsnMcms^QC7+6-JCSg-vQxSjaP%(AZlm0=V|iuyFgv}Kq#tpW+MxJez&nB(E4qJW6;nZskf94m*S42$4NGK0|MiVpETq3_M zz_H|Y+GKr#TkW5uhsbKAMuupTDY!s!$4}oC3VobIslXmYg{lnp#95GwzFtHj^TvrT2igGrD4 z5yjXhd&cIxq{y!oH`qw%&VjDQWkpq!Egx?Xm1E-lGo&!61q8QQOtTe&3e;qa7{N4wo%Z_57*@garI@!b{y3tPRxWyV7P z?b)u|PaXFuKlEJ=*Tl%brISqf#<4la?sU}icYhl?ehZazr}U*`&+o_t=bxX1ZIGxa5I^%7j+$}Z>`Y_PyJEM~t-gbt9c z%=UASBM+>=u}#zUMc?od&IIy%bM4iOU-7a1EJ9}^2XJUBEe_4fF>^03CN#4hMOube_h zMY&GLKv79sU58X#OJi%HY?Z{GmMStFpbUZy28N4?2at!8k&lgqpA8fgA{?ZvsjCly ziJ_snv$dnG6}rQV#mBLy!L`kw57^(;+vy`JCamf8)1~40{{e!_H;`bzYXJ`)tl$n_ z28RzfIE*-v;>2kZH#jIK3|Rn=0dT>(HIf!dlO<<~M0s*0%Uv&E-0FGY0CKe8nB14V(W?*EiU%^(W5N5A#vZH>(+K$x*>Jfy*r$^ z+1|1^olU+Jj8vPCGP%=Kc^y7^RAv&;t7a01UT6Ex9DM;kH3(L{vw z8;CefQAGzZ(&*0tX#?pMHig^GN9@2f?#meCM3tsAV>OdL_fBQCi{>W5k`gq=S`t5< zwye(%VVBf^kQOMxgo6#Pj8@w5%qU7rJE+SE44baw&n{b;!wyMY!yQvrSW)%XO{Bs7K}!&u1)X7x-eCVWybzs5`9U;8fCsJ@!_GW>48+4M%~Zq^3tR0(Pd^P+ zz->zH@m&8lZna}}RjO6wkf85q(Cppk zDGZ)&3gwqzt#=1__y_+54jXL^*QxN)d&qvfVTS3R9&Wb3o;Yud?3w8724Ac#_1OVW zwd2FlPk+|@j7HAORl;LK5-@-&1orz@k&;WCz7q4t&jGlzu58IDVo^#ZQ1J({yoq*f zNe;@IrkwIMkb>w^A59EEwRm0aPWcjE!y+^K&OXOtg;=;U zzcAEqf9^9E>#~rE%86xV0W>4E?$DexJ`f9)xCB(#NI3R%tBbBnToUCtK{RPVkFknI zFDAe@ta*@mBWnK;zBb~H=(Vtfi%j0vmPb5nJg+j)izE)Mm&qJzvU+BN*kpXTJzsQ> zMO;$`ZBXJwDdvWieY_wPUl~hRmCue$>7pTqlAbVnU;_(0VjUr9OQq1USA?kF8IhS4 z`z6b7!aN-&*b_xHhLLGUT;(-yq#aobp&s|>69)&AHB6>Wc+VJN?j&_dLHUk`1&Nfx z3@i*4b=z5p9~0_y|on&Jr#QZK$d&m@AGl z^a1lcfbIXf1}}nD(rxT4Y9cWNkbU@cZ7H;nq6*o(7hVWdF>EI%JBd}ThUSyv(4Ht) zlo3;g&r}Z0qgi>1R<-Wbrf|GmlZu8@?)Z{N>Ty!Art^lAN-HRR9mHRu1SL5<6I^sX zT+Wge(`ZJ)n&gXxmMF#s7<%&IndA&jJG zKiK~$7aGqTEfkn&88+38R26LZG;nMkmKRD+U}8UXO)nhmLwoR6xD~eWg?BWUSJl#F zQYs2}k=Rl?*4385lG2c_i_E@4C%jlF??fGYEF}%|TNHh9cFsFN_DXY=h^%Y}FPpoh zN`{5xgd{xO3D41Xb{gu;Ebkyh-x4jAc&uHgj6r zi#jl>jsryjn&fJHDrBV*qg_;Uu{4L`6erVVGiyq?U~E2TO~<%zxgue0nPUc`m(g+s z*{p~RUth=ZMO)R=DGns&W zhoMO{j@NW0u946e2K^W?R#QyEPH1l2;j0-kbdaatZB!KE_rA+Z4Z@79&wcKjpF`&7 zYGD05Fn_t#0uOks0a*=N-}>6%cKC1Gs;$9Fn-7iV=!uQ;*1#e5#CuL=#&JeDLI=9c zC$`QTt!?Nv-*0^DJ1n~}I4WnvH^}p?VWd5Hpn9`2-svnGm5&@%(-zfVpp2oSWSjJ? zUO6-fe>K5V#9#0XIkl? zwS46}dGM)k8E*xrI@QB`b(cf@2DfwVcJ!fDOD&QsT(>so(|Dx|3Y5ZZpJq& z>2e(RgO}_a!G{uiKG#trA054No6eog8aDk)F8z0)iu-s;)`!>5x%0?psCbfG{PtBi zut%zv=+qc6siDg6*Ip0RvF+j0R~_}ue?B76TR7@RkAMB=zb;oe-N<<&jiksAGA?LD z%)9bzS)Q$onl0HBp#d480VVJdCeYc=1LL%0n1T;*8m`;CCFMwO@A&^N=F|;O04XpC zgWq^AP;4&0F2>z_uE~N<-u#V)LQ6srZ?*DGom8;jdhiD;W&VVa%TB}>K2HckudFUi z@4k)s_V48yYu7qTXljqRqC(oHjdEBnDKH9Ja!=Xbs?C68;xz7PRA)?T!V4TIx|KJ#50U?hzZ7pB1kv$G!v>Y- z`*>)FhAt9)a2tOR;3km>Jx}wTjzTh#6V34)RqxKW%@!HzDJVhqfT;>mCliiM?Ou`X zfMgIl06luI#m1*t;*9x7ZWkS`6X8(qMuJc%P0|9%58a90(ybXkjTxnp5pQlHy>F__ zZxKVy`~U|1(l5)n@gf~D2*0uOB5Vj%avI7J9bNJzNii+hu*H7SqU1nH5{M>4EZcqs z4yZx_xdN~pknO~y0wX~mr>oBFK={nUI)Jgch>Zz-(aeaCo6Kn$Eb9dK#rnkQ`u61; z=q)U7@cW{y@n(pq^bo)dkt|<`w$ySYfiN!N(vuM8{W$-x!7PcW1Po)8&>W=_CS{51 z))1!R?l1js#+t?KIP5)wtqjI+O778fv`8sksw#_bXe`b!E9@y#>CtK--3$aP2_(LZ z>M;J~EyoJ!Sc~xzhHQEVBT);=iiaaLE2(0$o(!=hMN%6rjYCdj;J|TC;_{S44={~$ z|C*4d><$+Xaxf>s_EtrV#I6jC!X6{DJfI^Mb+QR96F7JU6*KcV5yvGK#Ign>gS@FC zYt!bKifh19HDOTG*wP~{@;2*}8beYe6=Daqu|DTgE|0DufD`J@rV;H8l`^b3HIXy@ zG76n0+w86le{H;gPZLBj9;t1v>Tc|A(mJn`Jiz~B#x{@xkuN5g$+HZ%P3v(D!^1I*|^0LKJ03q&;GCE=th zLj=zSZS)W42}gbPJ_j{6$AQY86u=IpGW7qm``{BmxiJup)FhkKNw+b4F!fp+G!C1l za5@aGrXzHw>(87;I-uoDPgP9&QGd8YPPsK8VKwX6OC{*EPRFem2;$Q6G!IkLrzo!< zMoU-&RU2xP2agq5Gf6+i(oq|gH`h;LBb8GicG48qU#%5eEf$ub@C~QvUH`^_?&>It z1anlC&?doD&lM}uR0GpvP1!X$H@vx5uNsCv3)ze;c zSWAd3b(AAB(q5L;XpI#T*$>qgiH7cy2uDdSEB0csHfxzO7;923zSAi%uoi7bM1R63 z&DC7XbOD9ZDART*&z4A@MO9x6O3$>uE^d(`?U4{;~<@+@~^V{oV-brK7dQKM9AV|FS*w=y+0^(K@)yzVKE zZxRl0+XCTac?L)>P(lM!rV#Q?T{ddCR&?2e0<4i#2sdA5%U$fNX9=}E`PEUK(L;`v zXpxs`3)Wzlc3>vg`xq8UpB7rJ;bFh5YWwYKM;CPCv^jUTYw6%St3zDKOI(%3Y@y9< z*EW71kW3$e0p%Bd)iw@7@h@G@uJV>{|2K5oG<`xLw1BtVYEF2I6{yh3c$+mM3)OHR z<6ng#K%tQaG5B~<6Jg^Ma-;wCdjInzkI-ttH+*v!V@;T?G?giv&x_O}%2HsXS9*Mu zKg+4gu=j%xOfILkdz+WvibzVuw~SL*Z+B{2?gxNhrWMETebqLwX83;JHjeT4j{V9f zqx7j~0> zP=rBmQA@Z=1$em5_;hp4L1h*$X7V^@(&PxuWGQecYj_rEmwaPbCUKc=M|qdgmIBD%^{4x?Zv$(YwmQlHwAHJB&!nhGb**G~? zka>BWdpZC5z*QeDbLdEA<`_-?3JwZkS@al>-(uz_Pfa{i=iMlvm6h_%I zS3|ODlen76p<#tqdFxXzM|xnFcs7ZYi5W?07fh=;I!U9pB%Kx@wiz$Gd7nyJob6Vv z)#`+wAfWNqywLv(KOhI5yW*JUING9(DDwJ!>ll9f+I{8NuYWS5)XMl!F1g@Ze6P}P z#mTBoq(qE484Xllk@=zt(RwqOiUsC^z1oryRY-}Jk)QRWI~t!t8e-kDS@|NZMOmN& z`Fy~cr~#U`SGtGQcMIt+uc zvfTk!Hyda*SyHN)v#*$Ox!Z|DTd8U2qV=D!o$L)N!b39|0{4jw}L~+Tl z$4)yiN}m0@(DNCN4ZYCs`6sh$TzFRqye_$Afxh8H1aV}-qkFn3Hp9ya!wWZ2Ih>;z zZ?ijXaT9T@-=IJ z6UzTpD90{mKyJ7rGurOXFhg-UYk9tFSAc`z0SrKw|L#Xm)55R(saea#*DtC`yhwx9 z@v53x_id|sRGwCScdD1mPrJoII#R*dX~!7P?|k3O`MhBG>*fqS2;FSYgF3#B5(GTa z7k#h;yTE@^jc?mtJhyyfZT(R@gom2!2{%~Ah7 zAnmT!t$z4)or-ch365`AxS+26J4)tEx{h4137lH$cOPMR!|+2wbwTB-o~2zS zb0EL1g?O2Zt}C@zgVqMct6WLd{j%dh-L1Opy&vt-{2?)FWk#ES9Bd_4?vopEdvTcbWdy zcslx)zQgZ$aLCQ2(GB(?_WdR9`8o1I`n5z-K3Y~nPC9xdiAizNNu;c(e3+h|s;s1# zVNry}xt+DL(ITaw=9;doz=7HRHu+jn92h$&AS)OfG&2tv3`j>iFB@1JRA?htTR$^{ zUmr&>D?5uXCo7wqhj({-A54(Lb04dxpVuQtIP|Xr^cPUw1bhPL{d1?U-o1tf9YV~9 zO=84|5Aiu9m|-JFjvG9J{8-^4MT`dmp!8B_Sgl|#fxV0=bEZm`T5LYS33F$hSw35` zY|}CcEH)PeEfKm>PM$s=H_h?1v=fI-fv)D!Ws1?Hu3o=_4J#Hbpg{k3kb2Upl@(7~ zuP6*sy0VOtq(6mSNTqkzhZP!P>ZEz4P&$+PzCnZMP@A@L@+c~9$S&dHgXPw_dmL_H z!IGET?KAk#^61i}50XYm`S9V28A)y~oNNF|%(4f<9vE`>$KJj>+phc%WdOl}jU!L4 zd^z*xH``{BYo}HxphgE>9kfA@2PIPBPTgJ>>s7T{F5d1j$&xy5*!VKyoN5^p<>R3nQSAVmM16;chhAYRG=Xrz)w zLg!YJLKqljfkidRBrwuR(Frg1ZOO}bL`oMGQFwH)K_K(pVTT6Woyh@t9+;=gBV@HV zmYtHcS0-BV!6)DcQX066D+x+K<#A66xC)>In)26^RaytBm6AH>B4Q)n24rXGScpz& z7jX)a0WE6CscH1QSmLQCnz$-zsi_vDgfpUuBaOJ-Q!B0y?dY4Yzj3(Y1RZHAGTZDJqp0#hDJw`@?JAEhcLB9=_9fMqK)La)n^&bdUAS(}F)miqE}Gv5=)vn= zyp@zi2wYL=*KR5M5o4F3kxD8>q6SjHN`cf8>a21PQUm{?rI=bnA*Qpwc%!VMLR{;G zFzVVWs~xWz>&6t*NUO=a25V8uh3VL5Mv?&n;f@KPy1TTyEuMCg9oE=&zBfo*a^$Z zHA~5|&R4--V9SCu$)NcPr941Ut7v;8!w4&g3)=x|7yt6!Qt&`LL#YW((}UB1mbaBm z;1EwbtP^|K<2*-jCtONsTMGLEI4a=bcw7q_3)^zHD2-2AQR`q6smL>Il_4I4qn`lD zk*UVa2t>iLn5;g{qFWT3`f!C6Wlt%#XiO z9R)RL$eVHSkcRx=C~OzP+%2dLHEd*k2=xCnFjcX2SnEz*yw}MVav+3IJlIx#_`Fj> z4?b@o;`m6(FT8M|4)R&yk*c5pRS-ygq=X~}naHdLiGv!=pv{GX8M=JD4`cc$iHEKM zKzrn|nR7g*G7rbb{#h=YrWqN@2w29~ICEEBL?Fvr_A)^N$z>9p!0Hrf#ccV~o$~aL z@|xx}^7Vz6?<`?YYPcokok9>)3yO5qB}3uWZFq1(VU)zi!yLk^hYGa*iz8hk8vdGB1?rMEdT8^v#_#EF(2kxp-$7NMXiwnm)g`m_R){pXs1bbgQFtQ;){Kb>wibC}a? zelY`Dq+V9DDmo)fITv50npXd{sdZ|x3K`qi!S=zh6D#>}nV)MR$(DCz+E#r@*!~O` zxYsc*MnAxa69W;&DqgW3Hju z2%7V>=S=WY6&PB^ZbP&cbC6-`$G?eT=$Qw$T!6d0P5gfOLl4tPk;^=@Gn*OB4~zhW z1_v4d+fgG)MjdLGtl9}iRl?rgv!40vXEcyluo%9s--3sY+aH5m%I&4tFo z4KyjG#bQK==hB*Pv2V&$rWgx)O{E<*sgZ2zBr~`-7?@-cRgmge!#dWop7pCqS6`XZ zw7u|M>4zM!Yd8x8kDvcj(c*T>kN$G`*!l%v02AlH(w;in&`zhaIYJqu3Y%Bzbn21Q zT!DTJ)4_cnH@TZdVL;(BIUfe}pLV=x;%;p^e2rm|ORbGnGc-!lqBXx~jRCOga;yEe zbAn+V!G9|%;Q7J8z#IM>fom?MzmzlK-FYUvlnI9eTBDSTh-h!|5L|-0U&wbuYgc9BI zB=xX}!%K>M2Wk~9A3c({UF(lvfLek5y4uCgu2!}Z7b?(t>-&AzsMB8TRCoABEDU>6 z=Dzl+W-`T#-{cIm{g~W%IT|~g^IcE*s3Wh~$bqfwDw~}0>N^J-ukZ5jga63Kh`Anh zZYi5n{M+BIHgLSbTFq0DYfP~&{8d6^#6y1fb?&3l8MT7o3!lG%jlswm*acw2U7cWM z-2u{?*Wr@^GGN1H0IQW41V$h_8Q)V)-3Hp){j?vR$#PoNXD* z^;pvX+1Ujy*}t`$!cmI_QX#A*AOY?hgO$mzgyD9i-R&J$U+`cCXn?7$Vc}`V8>*oi zs?8kA!31~*)Da*liHUcNNgwjzA6kvm#aoi36{HkS%aF_+MHy^(Tr?FG<{UMUniym4vv=Q7z6s<9JP^Ow;4ka8X@eU)LD5)T&!YiA>kG}71^~* zUuYpMq7zox(Cq1=!F3@%eL!z)S}__UBW%C`Mx9?+mlC~Jq+!=U@sm+dpuX`~FU8_X zRRRB%?cc4@ojimfH6`EN-BI+dkqN5cn{}MAO(J5UoG?k^;9ZX4q1m+2;0-cgIYQQD zq+xGxUllX<+ECD05EdPxjwGKV!pwb9*-_D4 z_et3}XxVz@;Nzg&C1&Dd`JGaETPM<^w~d3YdDF&~11V0SDI(r8R32bdl~N?pD@xk; z7^FZZg?(9_EVrF8d5OC1aO^JZo*W?>ppj=Ioe`U0eN6=hD6 zWsYcu^_7NnW5#?SIDRHlI^y$%=26lZP0S||l~1Em0e;fv znC=>Xf@W)qA|q1bP%MG#)k)&Edrec&TBuibgRqP34(An6}t;FsO&3bN&VUh`| zwnrYU2Sf>`ks@pd3?M21)`dmdy|}@d)Mcc^WTzGiWulOkuB(=E=7t;_MqF!wu_-f| z=?VT1YO13=`qyf1AG5Zj%IckafmQ%ObtQd(GAv~l_*~3%aXR+w7g=eqUu!;X-b+TT%m_Do}pi~EnV4* zxPoMe9ZNOl&DPd#HvVeI%AMYYtjzwG2o|WcrekJ=8OyHeP&Mnz{vFM_Z{xmWf{vfH z-KpsO6t?=x82%9_qRRWlg{(?!+r|?1J{X!5EguGF=r%2bP+bHph}1r3D^Uvu)Ga*4 zsJ&`!&)lvfeSimuXQK#FRUpDun5reL*x9P2v@lJV2rvJL9;zWc7Xp8e0zXFx)lO|8 z$O0?ImKx3&G2hI(?D)DMQH~(v0Um;;ugI=%B|=5*82=xUbcW0k%Z)z2#QBp4U0rAM% z2s>s<$?2?OEvw>AqTs&oDzow_53bEtE1t?!7N=TI#a%QIhoHi07?Ve1&O+*v?mU^O zE5=0#hv)(yK?S^w9J>Sr>r%l?Ff-FA1#TxV_{{$aX%(07LVair9*Bn=ig2oyr@|hx zw4jR+bz~CB>Ms03Kc6q+09xEfi|Km@3ZBKX@aV$6dQCC zOR+fYY*o6~6w@-dou-R;anFhCn`B4Z8YQE(o| z0UR?*lM*K)?U@D)O)1cHsItW?Y=l!_jldr1L=~w-0j8?9i9rpp+JG+b9FznQuu4}| zCi`Q`4(=CK@h2;&H=${na;_*ovHDsoKX(?*Zgp6*vgFn*N93Gs0`A^E^uujL-GOy3 z>#|Z?G**Nq91to3KO;#+5;|kYN1v4-IP(8U8)r!?2n2(W>PfS@EzJsV^Pp|?h(hxU zFLo}aK?QJHpEQLyqcAGH2YI*{M~$mC?xGCpp(Babc^()fwH0D+zQbkA;^&Dch z(UP_hxN%`Mc9&H0>oN5$AaW%Sjcu86#R6REwn%v zQ=PK)0@a;A6w^e1Qm-IMb;8!JdNm_t2t+$T0hRlo1d+1*2kwIyOA zad@$Rju!gnHVj^(QYP+~TR3lTID*4DVoXDh;2*Swxci|TT%*BJSaf_0ZFoT#>kLny z+_fJ+_eht{1Y9b*ex-U(BW!`TFAuY$jZ05kI+81wen&zFSoaH^REl3_(>gW%dhBUl zD}yU&v@NoI9)q8 zT8;K0f#q9zx=_$K5zJyoMEn0i%2E+}QXpe`lA|zpR5Z4eE2OP>wKKu5opwtJ9K@To~Thjdz8x}VUs(SzGw-3{KRIvoA5Nk9_+~*Y|0O&Soi@kwiqW} zNudZhf$z+UkGe=m;;ysy{X|w|c=CfgpW*r1tCKja*KcvxI>0A=z_)fm<9agix->C# z-~sy-zU~pXIf$3(`4FD*9BRCTMF0bJyzD@CE9AAIt#=%RGq=kU1o!_0towiU{H&ky0Q@Pi`nz${`!})1nWs5!`+MZu`r#sR;!-h( zE0)1yZa4CKztelw6fIvDIqJ%jiN^DxSNm})dfEs2OTQ%9i+s2YISH41s`jOMqI~m! zf<}RQ*gx3K_hjHBj^A>TPnK_n3%q#+P`wiqs4V}!L;vHN`K_-x);s;b>%5&}e!i=9 z8bXucBxSMhxtfgScQHHW31>PFc-o)+AhCYyA5H3t3Ah-rQJ3(x7w97!l&chZP-O zJeaW{2O&$c8S6(xq_%$Q;+;fkQe_k_y1tk@m!V4@KZ_#3%0OzB&Od~DuJVbAPAXHd zCgA!b=L@A@h-msb<7reMsB@%Votj2!E1kJ+saljvEFQ9q^sJFZE9TmB8*mkEQ|Bzx zU(DF8#GC(8uRnj-j`&5=PVZpCRBn>7Rao(2#*OV#(4bF4w2~jQ2^=`Ep+?S|J$DBE zS#;=(nXM_iS2-j-$E{sA`N>6X)iq;_F6D`vc5S6u=jK+rix<*DbesD29jX&2 zV!B*9bn26yi@HP#E>P}ZzdU-CmX_Mt!eyGV)RZzmB=y$keGM(--;wmav4$UCKg^i< zv*7nnKLOt(&^;>HYfm*Iw8upoJc!Fv8C~(C0pC6x?v100#sSi=T2?tAgH8 zxkDEfajNRXn!xDIIpDaHO%_0s;;NOV6v=Uv=Ro;H#^0iP! zkw5>$Baba~%!^C6vkpn*z`H^O6Cb`b{4mV;4vVP*#sD<4&HUKRs~M7(S&#zgc~HNsJQ&4IUu8(jyg}% z+a%QGnEF&y8M%1rlqX-MNssVwsarPfz`IN*pKA+U8&Pl_&*$f9t;{dU|6H^6qXK#|p@!(^p(6GswhQOdRy zGu4Gf7x$gC(pMX;@vWJb8j{mQ`qecO1HF_MwFFHT4Bi3Jyv1diXGYm04n5+af_yN{tRT(!1ln8DekOXL46Gqp zTL+Y>z{yo;p1G^>(s1Sb_@XGQcXqDk+c^^yKPYwmRKi{ zFJ>iVSE%H8YAhW{c*g=&+py$kla6+;A+ia~*ukMblf1$ihg|S!9}8UDknHRr13xhn zy7QnvA8muvCO;ewT{IBGNfk4VT-vTmtk-JU56g*P+lQhXQ@&xIZ++oZwzDr^PNk2eim7v;_Fu9hoFVK0Es6QJoXHJq_D!eBcRN>x zRTP|H^3=8}so`W`2#gUVW)+L+P0vI}v5}N?R2Ps~PI7_rUVOy#LiO>lFwAmG40Q;O z6!g%CKC~hCNXIf4_62o6o1aD|ltd=(sRJHzkM3UKF-IL>he3o!2V>&35845MFXEc% z046{wYAQUq2wV1kr!^xDk8Pspo7sj(F%II+VqAok#`4y&EB)_!MH-n9YuLTWfe(wV zA>Vio=>#9Hz>z>)q+;fy56tw;l11~GK`?nqO?EPq^z$SqKj|RRRgV8oT{K<(P!UOC zq<{iHgk>ycIm;iy#(z`8UM>UXhaoWvgkIBEXsEPjWNj$R1+!WYAsz%q7p zHv{m>Fy-<<2nO$h$9tw6H<;2oQtS<0xFd=>3MD=6k(<`jRQA*QYY zG}SYn@=Pd2g$h*eU8a&q>?c2^`o0DRRE%SA-XfN|1cf5it6%>GYgZTg$fPRu0X6-K zR%pWp0{7AahcJK*fX8^wok1U4A%T*2oa@Co2DtG z-&Elq0!dDl0j?N>B%+zfxz%{M^QZAS>O9SwGRjeQr^KiR=c2mWsrsyG`m?GzGG)+r zwGOOEm1u5fDOy|B26kFwtBH~XqnQ?0mj(PL3J0dw^P!j%8hmSao}N>H$WU0aZv8O>8c zN4i9s!`!S|VNx)*MBHqeH;v+=@b=WkriGz-0k_lllEbs?EpSq|N?t*3>#40}ZT$WT z(AcI9N(1KKZM`Z>h*tTL3l2^#{z#j)u{y>Co}}7HVw9fPcx;Pc z9IvZNFl+a%Bt#7Ly6asVm3bH7?9z?Ni(?&Uw#V>LZ?oX3P!%*91EXE!dB+>LeAXAq zm(Fh|@5JOh&g+k*C<%L|+|>lr(!ESzFf1AtvEW9SuM@Vj*gCwt>0%gD8$>gTc^NSf ztF*){g`bDECrO=f!`(1M1x&4~Y)BV3Iu$3)x%J~vL1*OmeYyHKk0v5~!J6C#^DF@CmB4L#{<06Rd$zhm2FQ*rl7gT>4q4@*0JE4mD4s0)0z&%UNMTojki2^ z+~ec{M986y?!A*%(k7<@V$K1qNVZqV@VL2AmOhUEbHmqH8h~oMCGYd=)yrR6ch$g% z?6R?To~(^5H7pOmgdxFWfuz^!MsS)dWi!Q-)b*?soHvf` zwASI^$229IjJKfqN3nazdoBgKI*R60~tTlFy?bUlH(x~GSGY0G{=K#Qj z%fL<`K#5tB=Zc$I!$RuXKP611EdsMAxfkx5o>Gu48VRHi+@a?SL1`I5nJB?iQw$V* zC;kbPN;AJZEWe)PuRG+!%<4+W9C&fRr?=Jp*JZ zu1ggWg0~f7u1+ZxvpGNnd^izfLNMaOp13&Zsli+dEKJ87QL5g+eq$M0~p2vPOog zKQ5>qKNRBPrSwHrLC-w1cxr>@fV>LK@kqQ;anho02M_ zKy-MCKqxv{6uIr2MIFS&TTBV*Yrnz^tsd%#HNmR$fs-M?i9Qs%JhaG+?86%=x=tHJ zr9;TpIK*otNq!rx+EA1kvmGfsEL=FIa1uq^OQ{}_M^O<#;RzLSOgm2;y9(kz+#^MK zDaDtnfGsLE>{&%U8^N`BMK5~;zo|uPi^_o{NIJx#HdMiCTtsI9#uXe!^@%Tk@V+E# zB0ZGGcpDMnd#lV_$gT3cl7!1EW5k!p1#`g1U}BCLGqH0V#p38owQ@rC^UDQ{xFz9B zIa9UPWwp&;%r>w3soXXU&wy(6!&m4!)R8Ji$ z%3$=$svO9$1O#K$!|}UMW_+L7;+HN;0wPEeS23mZB#gJjO_HRi|I9a90;fXwM!n<_ zz;vl{lu5Vy5lC>FSp2`jtWXN&y-KmfQFOTLjI)7*I|@_FygCc%;g~&h8i9jJ7TQet z931p)PpDzf8$}6NNX^Orak+eP%^(%h%*e$4gah^{1pzh9rRu>0#Vx3VBt`roXPZ4p ze63EIwXZADmHJ6k5lXz=P@Nm zyI#emd8tz=lu<*SHuG$}K1Eh}2-JsN2VQ86OD)J;$VmOv*hV$KBHb5MLXgbcyqU>V zO~pov8dn5+rB5Xurx;a9l+alvQwyEPIQyW%T-ADXRmJqZcBHP6s?{3&pA?NdT^-nd zltzGzBk~kRh2>0%EfigZ+My!UA)^NJdDg7u!wrB{ok%BBMhD3%$=rhTOoDDQO+P)K zlD#rd?Zr;@RF`X6B#eu1^w3kC*>)VvH2{^S*hHP&+nt@1p*&Lwin8C%YN)KumgQS_!#Kvw7 zy_HLXa7|lSk^moS+v(^OxYbg2jV^axK)W4YG$UT$t=TqHE1KNh+pE$^Sl+DbipPB3 zPgu6;)lQ|2CZ>JdJ!L`H1>c(E-VXq`B^_bAfCUtePx8H|3&0$7LEo$;pt>PB)&&$N zSv(g1rWx2JHTg5OC*5EA16O2tE|<;Szm>Dyb4LgMT|_xx1TNKlO)lk1D_J7(9+ zW85GI_uWM$5DR3$17fk%0g>NCvI;;V#0`yV~T4HoA(QwYS zlMY%f>=&;yOqf(G3a%!9L|qmmQ&f)Ui(=dfg35&j;f9svfactWOo0r8g@5)|qF&w2 ztvSy<)W|q!)+7UGTu6oXDfi`vjtYlVh>xXsCHR9l&ryTc9RomNT$qGupUSc-OGrbdY$60INq#gcv2 zU&c0oqO5=c0FlL>hGKrIsh%%_+694tP4~0bI9Wv`Eh9mu0^M*7yTR;f-qg=SzO9C}wtS2{$ZnPJ zgRq9}u{CS7uHPS8Yi+(z3Ee{BHaOr#Sm9;xy|ge9fN&8=@VzcQ(4X1`%;xIeZBO$-XS{c8W1L1oToF24QdWOXz9;Eu8sghpKuq zXc#Fgu{heR3%~l|EFSQjt`FkV^z>c(eEC$B4UgFNS=562|#}M^W5eNYg z5c5kOfG`(xF&6<6y#zA{fe^Ul3pegKuW;XM6;;|!LVylt>u2X~8dGFt|4r=t7HZ5o zZxoLQAzYo{`e#FzfU>z183%?r4#~TCV+KQpDPK7%O=gnAEw>fT~3Fo-beh~g3DziZXXQv0y5u8 zv@i-uDAnBFO9&>>gdv2Ym8=7J`!kPwxd#Da9{|1gb!5j4=$H7Ib}I|7m^SS+<~Dih zojb)RLX}_e(LX_$7rFShO`s>}Cr_$G_vcr){5O`YL?Zt*Y8iz*I3ub7xx8K5pf9VcNmHJcPY95`I(uC^+g({P>QPh6%b$m0vjt! zLR(8BBC1&@A`6S_w<^ioD=QLWoSQ6^I@&C`teUYgU3toE{goYRP}!}Rm_Zpn9&Q>@ zj!8t+%Kqm54iE2URv1s!RBwO3udmxt%Jn|FR3Io0(vs+}v9hSzCT7A?$j3JIrn@5+FikLYs@OGG~+jq@jxmdG51E+`N4ghnbwEQczHFRkw~^ zfPDN4<|FN{kZL8XQS)<6kx1#(cxnr*i5p1@pK7u?@a<4dLNqvY8$W*hTn6Own9o+y zP^0peBpLe7dQv6H3NnM88U$M+g`2OzJn;F`+fbdSv>Z)%L@9B1l*vlHe;6YrngS1O z%IrW|x&SlI1b@}nR(^=7wOBHsz{FmJ*tBv$V8q}>i-TqHVIeGry)~H#gb4-$EgtH% zSSm9yHrZPWV74An-DMKiWd)LhqXWw*w^>x8{qvD?wmk=1krss~9*{#mCuEaGC}m>> z4+ZIDl~%svfJPBr$t9H!SYZx;tBfiCqhygOcED{G1>z+==AoB@NDG}QAu9T;A%OvB zXrRHCZ?NT~XrGyd-vBR$V^us@VYU@iXNA&`AZFROA!PzeHAe`2d6?i+GqpG4R<4L> z;)S+=kPCjB7BkjlA=E-*sU9vU%3KXHrWu`QKJ!Ue;9Lh>W-#v98KIzs)=rm9P~azR zt+58Jk^4xis5*AYwxqJ!8X<)Sy$Q=kv)hhqTy%YstLAZ2l&LGb?nc%~aZIY|fVlL4 z>A-pD$+<`sLQNoJKW${=+f^DiBAQoh5eS74*&1;Pj#Uk~mVU(oilBfDD(FXtu%rs2 zitnn@%Bv*uaw>|IvbAa~Av`Gm46KBW2#c(R-3pm9Ek1MTI3t_64I(uvB5~1SX3(Q1 z$c~0CNYOq`?Q0cH+7!0)Hl5px6Bq$bSZ!hr_L>@DU~kyRwTpJzInxE+*AwOIHXao0 zgx(7LO7IX^b=r9+1u2A?W`CjtN@1a1?N>y#jsQsUrH4w27{-?#n3l(!?iI4gp&Fnm zi6@pQ%ZDpl`U){3z-$1@gWc83={N_2bI6V%3bfjX{2FD^VG)b8qdzizb(PN!PyCX_ zM_tCwjy%kK@Ma@nXv2-+2>kQbPc#w+^EuDGvO;OTH<|5)FTPyCxbmgE-Cr$5!F8ul z%^Bfl5C(g3`>h6;fB<~|+s>)7(+OJDY8(zjjei~?F&!zRRdjfqffk0K00vD%81dZT zmSHIHy~}bJ5>e|e2*H;1!Z0m*m5EMe7nS)>MGf*_G^hbEcWn-VLkpJf3dFmlIb~_r zqsZ|3fjlOWZXNgQp+y)$p{fL+XXL1!5uf;u-K+ouW9vx~p~w_Hs4Z>g+ZCO0iCGaPD>=Pt)Glp=!lNQ=c*BY9uXp$P(fXVgMyNGUmB(x4 zvW(QN&m<0u1Cc`i*&dLvhDC37;cy}?pST8}P;mnoK#Cep`MkNIt$fU+%1gGGjXhB4 z0QOp6y?_{@_aV&zXjJ1G^G7i57*Khd0H7QNL`S3aZ!8I13;>m&9lItfB( zgKMoB=)5b7(fI+cu|QGGFk~|jsE|dlW068}nA)5X>#u>(9W@Auk&1TeqM2RgVf~9t zv$&!U02Zl}jQQDP(k3hXyX@@xYmcB(=%z>FO-;845a`z7dR^peWc1d|_8vyUua#wJ zZpvW)d(_D%9Js(IKgF4*_9KR>sBKh7d@OS!;-xvZ79uoKN2YW#Gb;)t1S2IDs%f`z z@pObDHN^=-t}qV!?Cv8Q`Q#9xSCTg~@+j~sKulUDz9O!}Uw@;wP~vx#IwG8Ckm60w#IwC3R=4Jbd~$|-n&z8|)3^%qq?(=xL#{xrg=;_H*B{Q+ zq&y!4-?7rut`h|4T2+|AYxOuZMJ zD6nhR@BiLx#Q*l0qt99;9yp6Wb)$=8obbP#1zrLv(~exU2Fw$9|ny zx*k zKJ_V$=qpj(<d$-)~RB}{7?;y?p8vO--q3Ly7)S~a!LhE%37(62lH0W#dRrL!{4uq~Jb)syLd z>DbcKyWU*LWyvfOM?)hHZp#df+~lb7ejZ9`M9W#DcIJK&bFKX`(0itFPfh$DkZ+CL z-H0T{2$$BN0gZ%yAQ&Hf!kflwhI4xNS@^9B&$Kj<_gNGoVH(#BUqA*9CqAFIT{8A+ zWKwdpL1n$5H0Go!x6oKDCo~U4C4qr*zJeoCC0^k+IXG8>xR-l;U?_@LWM?sSFUToR zK~U)xUe#72TlQU7HaZyuW|ZR$V+MVjBs6FyeQG8>Y*uXim2T?BN{@B_XHQsX_dqpP zvwdJQ93@qLLo#>h$4iG6er51cjbK#UA`SmWB?4xD$#P2ak_w7NW`KZr{3jtig+z}B zc}NjQPEioQ0bfXPNas){4Fy~eI3^V^e{6_IljIbI(=sS0M|cEMNnjy=&`$CsP_p+3 z6GVH$H&8Gb3uuvPFXkV|;)yTFNOEw4z;{o>hl9lD3$z9ewdY>l1_jUehMcp3U2`Kh z6b{JX8^3Uhul0qfgm&J?QPV_)Raj{iF@A7|VYX2YPcS>`H;gl8hR#uj1c7k}H;P2G zj9LU2lJEwN*MQr_f^*Vp5f@aF1!}uNfIVg}dlv^hqCkN|j`hR;fZBj&nj>-$IELyo zf#BtcdnQ7I;$vo(RZY-3kq~oUc4P;5NiE`oh=GDgr-IJJa4Q!8r8so1gBD!YbfcGw zr!q4sggO6FWD%)*RA+TLV+@)i0x_9*)ORDAatt^5k~V3Rt#D#XsEl;xgiu(FtC5F8 z*)(q#X>x}&(r9SC+maNuHuXTqc^(YTA zYh&;XKEog82rB1r9R%Y{nsbd{0U;Ftg-SqWGea{2SvbT{bb-V>kvIk=sa~o#U2ud? zxp+XvB~Z>#NmZ6ej1-D-X-RLzkwcdYsn}0HgSdXdnT2tY z=U6orIiUk-Y^wN*)mBih2$(yFP!Ny*5hJkfiS2Q;HnmNg$#-N>F zXp}iRBti*2Dp);E*mj=v5E#%WnnM83c>pfp0wz%Z00dBTD`5c^z?@BUcI^`Z0#E>0 z3IJQ`P*kC%Uz(+rr!G^ON}-X4SXp>LT6RG3m2Db_a@cU;L!Q4=mT@voF601x(u412 zG2GFX{urkucYt!46`7a?o`;{~k(XR~UW!N|BtlpRS1d!qmwAyxS+!@Q8GE%j6R1N@ zaKVWf3aRc%U$s|?9NJct0h-jdmv}LfsCrgcg;#mCpy{9vIH{U6dJH0f0LTzgMM#q} z>H;yzqB^OByeXVfXpF;yj4TC|!)Y8`02DDq09k6ST57EYz@*u#t=o#FFwqfd(plz6 zn;f74=E?!-su9`xrCaK*0Z_Qdv=yN zR;;;d47yqw;zpagBcr@ptj2nyErmQf>PkGiE;-hH2G%6~XRS=iqzzyKXxjj5%eHKr zwr#t%Y5TTryA$-MKu3zVNoue0%B}*C01mJKz!3pxd#!tGxbj-IWox$HN}Y!17-hN+ z{py>~+8R)(LAJPRFZ!l|I+hbBfM{_4r?~PRjJ6DH37=SC3*rfcUWu{Y8J)1kEfYXl ztH7Zm+iZUMBW!6Kfnf_DLS+3`nh6S-+xD0p!?UPDsyW*f$9rsuaf3k1iZVl2CYgjf zfs=;On#EgU5p=6fyP`Uoi&)2tNaLEsdbJ{Zf5^Iq%F4b;;BV|Zh>RL>0nwgxX9#M! zxQnZ{atpWp>%VZDzXi~^^kJ{;+O3PLxCgAj2#l@m`mPTQ!SC9weH*6C@wfxQuiNN# zl?F?!5wLE_f4(DS`^T~YKymb@0d|8+1Ew>}$FPRbr_gl-Uw{B7L;|FsUa{tDg!)m? z=?)vaEFL?K&)_*DOR{#bStm>X1{MM;7zY3$7F~@YAwbjVAP-Av$YD-?5 z2X$NyHMF?}QyiSDcrKt52!I(7Pyq}O$q|46m2AmMipdFpz}|Yc`RmD@thW6tx16l4 zPMUzpa{ybqz@JRGpuE4K{J*Wtr1ToWwfw+e%D7{yuTFz%?c2v;S9Z@NA@muRzfocU zBr1yrbtPPD+v{G!3=Mctx<{x^U*JJAvzpcx4He6}05lMG=^k7&yV-!MFC%n;*|NHu z1ixnsy19$p_$ZaMfxc${d+}K-{?iw$g1uXuyrTFF10b1pfvTOUsvz2-74o4t7?K>5 zU%Cpu0z_gkqYe%|$Kjhhp_IqP`a9~ItcYcHd;-!S9ms#&Cl^qru%$ecdsr^Fkd_BjRNCXP9!0IL(4Gs9-+$cKye%`x;! z{VjT`U{)JR*b;VtT`@>Zr*|PJ+m?;JHu)<*WnG!TqAH$~&jt*FOTwR^&pYRP$0iB} zTG}~`&_a9vZ4{_&uYI*625-`&%(ZQk77eTzUA0W!+Z?@a$|{t{?d9E4+})B@2S5@i z?c8y?)6nhGhuqUVod7v);N!s5G)>9g4c>Q-)$Gc*35?$6jkfl^=kIN}@crIfYPLj; z=eJDCN(#6>J+6Bb4{VvJ*0`N=reb7L=MBu~9_Jq<#=^&(W1e2miy0Gf&6sl?2Z@xc z+-!Y&?b1rX1`;R?ZDryXS%3)J3Z_z$EuKeEj0rl~H7|K9o|1%}1lrk!k^83>`K*&) ztb=qQZT$2LM>iNtS6;BkgE{$DK^JsbixzB+q62mSwcXg7J8m~rwY@sWS+3>AGnBy{ z$R&;c+`}yrHkT3iWoQv4wb3oLz%l3b1?O>o;5$w5458kM4(RJW!9X48oeTkQUfl;@ z-|&676JNjzf3_AM)QZl)ZO)vW+aFH~!D&KTy&Ts6-N9Y70JDs^*y;jv>fp;J>TK;} zFcG|xkv^lANqN~Jff19D%W0QZ0YnuLzflIU&T#?hiXuz0B$fua{#J-xDvHf0zU~aM zDGtP%e9Ew}pxOtZoo(|dwY7J(o~Vj6^9vmEp(V1K6|$f6U3J?|n7<~S0iCvE1ncxFUMm3|0HW~ALr&C}*>;88mN z$(9V$5|8kqKf#2qznaegc24nEt??88zzZ+>9G}$Qjrj?vX%Ywkjsd|x8h(OOuNPmq z0$^3DcpZSV0$z{iP?Lc4<3I{`@LKIRC zi$vpb5RgWTRYUVY6&?&q?>F?ERK}5s8BNYq)@?S~D-tD=VDEJL(C;ODe;@Eqa4@it z&=66Of)8-fu@Ul7a+2cWLc_B1QUlXcgJZ)jBXldYE!31O%>m7fOfXGtZ8ZV^!FJ*n zVnNrUS2rOd7okA_!1x$BSvk45`PkVcy1?0ZS~_}nf%uu4yBoSWT)euwx}X8QTz&lc zI=XlP!K{$gR>6VL!I&Y`xG54~r4*Sc9Wt1xAi_U*09bSP@R|TER;om~f(K(pDjKI; z@%V&G$Sn#+a(U2cPn!q;MSuXI#AVD7c+N=TGa*Z!gL_!{S&;&0q6ZsE6fGL_A0K%wIms(_H$ca(pks+ewEC~w|?OBo`i-chMmT8!gtIjxB%hnED zy)m^W@#_Voh@)9aUp!V=oF*Tmi?aq_8SR#yS!O#g3RLNeJXJK%fq}K6(O+A~0Lhi2>Z&3$B^w z3NC_sm;{4Lm$+2Su^}rfLNKhd;>%2}x?~DO%Z4=Vjte&$b4e}#4!8@mFRVC@E>}Q% zqk=f*NYajs2s{+g0e6Zmphi8B4bt4+K*a)&CZIHxq_X6=DcMh*q7GGH$C1(#8rc!s5b!P^!d@+8SwTL=kCPNs$xrP;ml3fcig)1fD?=^CbQ6pCdb7yBqS)_jI)OS0sX9vFGi!;P->N4TG6Jh z)kIyWGZFf>4L0lTvusGMh3f68d0FB!(_sPVM659DYX*!h|0wm186BO8uhYKU>rrK zf?uR?SBZ>=FEkPXFe;lKVkuunYRLnz%<{=Dyr`09q>7RwXOx$Wk>sVcN^H%ThcqIN z`f09tqLU$vqGyo^U7`qm;?r@4Ry3oP#WYEa+yW#2_pGNm;u2D$L6j``O zZhQj)3t-M~1+X9_3aAn^EN3_iFdG9rh&EhwE*`O35DFTA0vp~ChdI=tf_R||x+Mr5 zsrp8-_Vyhi7Hd81ftxkjp{>E?tvb6JK;jk`ipd@1anh=m)40e*)Dd6+wX)mg6qLDf zaYO;N=phTWW}HqyN`kg2!3pL#f)dzqgpPtB1!6Y|2h>guUV0ZocK4J+ZQ>DwU>=&9 z5~8?lg+j?ASQDnuyqG)>deQ^QUEGj>iD{;PkFg8SMoGWz$>a!qDy9F%7nkyx?|d?| z%wW#cviF^-OY)N$7?SiqI3>YAH=D>>v_umBUpApL0wf^O?BL7O%!`51lx7!Y=fEpc zAOah>jFF-k!d0zJ0H$+Y1RCH$q1D54T~nty(}n=fkw#T+)8RbnNkbE?(Sr*})jscL zIQabYI!O%ZtwPwuCCY~zcvD6cdY3nNI4fU+0MIn4sfg88gP%_P-0D=fPJFTvq+FW- z97$>k2-J~|DUHV{NBKa7G(`jwARfR1aF2XMC{eyU1dP~(hvr!?k}c9vge-X&OsZi5 zn(W?MsAq>z&W}i>3`r9*@yrXwE^u_c~*5EX!O&c@#$tW#HpwW+)!Bi zXX5DV6SYQ#yiEDF>w1?TI9iFd`A$ zLy#~s&es5fqMX+2AXGqWczNn2piT%XDc}US-lDLD2~B!xxerpSS2Lc8lBEHD!oRp0 z6R~6!e6lRCdtXU1w$8GwYDyC=CA?P2@DhQ1eJ{C~G}pV>RWr%-%rZ-;;b-#Bw<%_z zH4DoK!rG;^fUE87gjLP~lI<4%%}Kx-J5_*p+CqlHD(;YFNJ9oO;GP~#kE@KES|_Wv zw6CSrK=bxbCOfuwxP9nuS(8N-S2Vb{h^=ay%iMgb(agzpKyB7M!aO>H0o0WO4%kA>|yt zk6k!JC}3v#z%$o*o(SAZmq6?ybJ~c{7i~Y7kgQ~~L`#_>jMl~Al1dL2!mar#bf{K@ zLx0VpBzPZp9jiC6ezKZLB(Xa0HUoablLxK9~io`7z5Wab2N|Ub$))~@#r|i zLl1h--*Mxqr?|!c{XB8ikKP_63B(nYHu*tZq2=s#q+4k^AxA@cDPWPT@2zkr<6Alc zHkV8ID!p?~cY%1W{1UB$uPoNdYir`>W75t&HUek=!$ha zQGI}3*DIi?K5WY0e^F{jcG{B-kO-WkpXve_$mZIV%Y|$qq4KVV_AbbRNN(f_+!pJb z;O+1Lumac3${36A5|6eT>hY||HEQDi9HE$gY*b)xYf#YhN{cC6?bQE+eon`?9dr(nRV4Xb% z=B!YwrtUBJ>gU9wB*Mr2D#?3W@z^9%*_15<+`$?KkP+o!^eVudpd$gcA#WHYSa3+5 zJZo+&aRG7bpkgfD4$sNNs6k>f-F$^`xFOIsjck-b0YI>eM34|4h|Ml07UixNT~GxZ zE(VbjDbMUEjS?zUs;fsq!c5*W9VD@R~3u!0C}?HVh@Db`CH8HNcnLVc=` z+MJ3C??fj@jSJxLA2BQm46J7yph%MG`Iw}V*n}=e1}o0+2$-&Y){qV1B24Ov47ZM# zaP12LF+zB)F2ha__wefs$PWQ=>>$!JgOZ}G3C6|)?t(!UN#!=q%sUh?0vQBuF0noT zFmZ-HxS=EsHNN`NXNh(fH-S4+c&)hf1fETzQHabj3# zN*>G#(?@5HZqXvNnGRIxN`B=46^=d-flvGXiGtbVYNYsG{OM+H}u;R}+ z)+Sa#3ua-KB$>!qbh8tMNLX#qMd#|t*f;Ox^$<+9WC z)iJM%!v?_l?!^6?APz)C`wUi5#mWd4R1MScWgNpX--=DOZ)-DtCQY|jLs*(e7zQ`knx)w(r&(z~vf)bDn-3XkbL@9l*am6PE1eE_FA(sxp6+G;Tp515CulB&GHbbGaam&gH;jE5 zqD54g6R?e6lXGLVCFhMA;JBGPx`Iz+NAjyrNxwuvUyCwxmyz%kwtIg zs;h--ns{B!opU-ogF~k`t_}vbz@*fBzn4&>z)&R?F-#_vb;dAdBn$#NEACb?2^wh< zR5@okU-`%lVvT1Z=f(vB$ZlZJH}rsHg471w6UI zfU!vMGb}2Cssox({TXW&lrRFx{HQvC67+EYjgUFh*OVDonY!AGZ!Ji~8n;K*jL&+m zbM_@k_pO5)q3D{K?;5bFIk1mAq(S(k2m6k(na5yykfoct;~|B|8EYT=x+OcZDLb9* z=%g;&jx<|PxbTu^5Ma}`sN6bs1jFkm~DJZ&l7S7P%=)QTABfI2cb)^Xd? zp;3&CbK6wSxPjD8ABg*zMe$~j2t0;cSVM7wzhR`48^e`b!!vxjH{7I=Hl;&6oJ*Ly zYf!t%8O2k)oK;-Bzk9sPcLo9h>D;^8|bEvWk6tNz^A+_^oR!%2PAOC8L=T#(Hiu?@S# z;oQw*9oFNV)?M5xZ>=jURM(A~Qq#{UzLPN=_H*3CzKgD)vzlgDn+vEd#|^+QKy1lJ z>@tx`(h0oMuYJYF8W7GpI}Rk%zg@0Dox%f3)l=QvPrck(eRg56%+>tN*L*x>UEXKC z*1x;nzZ1VhAlG*t4B52TMU_zhjsn;L_t!@NXjdnGst#qU4(n1$4el2*OnIvToU6|O zuEdVYv0dYV*<8*=0x*3U1^`}Ip`(hlS9g;DI2zn7xXTZRYp&zmV;;HDUFOq$=4Wy4 zSbEjleY&Of)#u&ff}Y-oUa{|8`QT*V`F*Ym_8e*Ee+pi_;X+LeotCQ_UYGqi5So4< z*EFKN;t41Ull{=4eB;YL+xKX8vO{+h;5y#vn#t_$CVX_|-V>p79bUdXXkPDY-sbzB z=VOHEFk8$I9iqA*%qdEes2O=C)joKCJ7;_GB! z;l%6n>XrkAW}1~kJJam{Gk*1HV~-{)?qR-L+y3BxHcHjr1?c|dd6mNXX7lcTxE_!6 zs@VX0!xyTV_>+J5mtXmtpZT31`lDa^pI`coU+_`g)UiKWe_rUjU-7>`@iSY)(pl-1 zb&=_^=`|JVd;0SEVj1ng1-SC`g&K9RJ~}okB#`xfQj}$ZU=pcJh{hr^q+Biuqo+tY z2vDhyD>kd$a=YFyI4mBM%jUCLAyGKkY!?D9kJDWO06w3W6I}g2f!-b@JYAfoiYb@+3sKqQRuB>hMa&p3@b2BsabM%w-^>xXKHutu7H~6=BIQh7FIXXACI(z$i$p}-( zlZ(6)JpOzB{=17vloEjQ1_lfvf&@Z+3KuE_;qakDh9mk&BzSM#vm?(W(z6F=NQ96g zK^{4RqzJ(f7duv|A^{D|m@;S5tZ9Y<7MwcWv?)f-=d?JK3=u7Aw2~!7k(M%TdS~e* zM1+3s5;khsuUuNQZtV)pE3K7spGEBp5F+PRCbOTK#Bpwrc}kLDgY{PpsWQBG_hF#h~|Dc3t(n28Gvadw$OOBkj^ zJOlxBP(r0SI06ZoJ@nv!51vHfLZt{XQfsP}^cjE(5ztZqxtVw(iZ7{%hE6Z>#M^x4 zVTTI?q+r!vd^yGlpL0Dnx11zICD#X4x3DD^SxJJ|-I8h{*`$(LmB(avN7_Q(lv!da zL=!X_z-4IfBub$-(m#{B*F-d4TRx?5lRTcW^`6qp=hF!wkKsZuGZm( zgZ^3nLV(tsgMe&Sr1&VLk-Es!j+N4N6_V(+Whs~M0oe#3pz0XMaeFRp4i>x`&$I$n9LribIM!2&z#s%Di&=9dWU7qZF{Hu6R@JI?QBtXurV~pXmaP^~eB{L%Yh~q?0E?{ezxlFyY+@{& zpfbxVyF5v7Cyh4$ zCZp}fdo|WO*^6(nT+dabRsMF_wU%-Syhm_$NM+nnj~oX;Aj5IWij`uI++@FY->q+r z8RI&ZuMwBLcP(sfLbk~cdyTlU6yiMd1(_{Ufb3)BHEO)kRHprb|cFTtd4AlEHTNQotpq!#KqiDR|){ANho*6=H$^DtR6( zN(T|PBU3<3Z`%6`_^xz9lBvQL-g}`k!p1`JdFOfM!{7}k*tqjiE`C1j;kpb05X4Mm zAo`OW$^@vyJw5G!bShWs2&lT&HH}>gWFQuEf(6{^PKP-BA`MyP2@48phSNBesXnMi z5Do`!1Zy5FN+`A!j>U#_gdrY1R<}DsDTNmtA0W*bMjpB^eTFRLArpznMKbb-+7QS+ z#L*vt*w2xc1Y**bwzQ@(@kWGdVz|CkMG|GPHZ9tM?Nsz41sp(xu6(5zW7$P8GDCyI z`O#OWd@JNL^7iu)r)1wL@rGRa3SRs zXFgyzn|HEkMXStR1b-P*vdQwF56x$%`f1ZFLF|wGbm&hTD%6Vw6{vAs=uL}?)3Mc3 zqS(BsHa{9Cg9I(AdEq2VOF+)7a@91XWT_V8skM96RHsQ@>QK{aRI~y#paMi%st*-iW7WhHEUwbJvOj@XB*xie{{%5 z#_*kKD?oA50x(bdc;09rzh=ZM)$heCp!nIeMss&Q#-w>zILBmBIWtc8q4;*HMoVn>ts7y(kRUI z9L;U%b?=MP>PEJt$>L&mlbhKwJk+NJ6zXa7d({4}_Nh(1|7Vp}LD1c%^|wW=@NhF6 z*J!mh1?!FC*Vsl`y;ve@o%r(wv zbZh$E^tSoUbAIWAa@^uRx3{dIE%JT;yXZzYInu#-8G=*6;0b4W%Uv$@mp`29&E~Rn zYyR`C!yD&Zmw3;=F7vBHU>r^*`qAy2^pmrlu_}+c)#Dy>v3DHOnbxt+^L}ox>%H%4 z**UPgE_k1p{bgtGH{#Q-__eoPS|oBh+#@gd$xprVloun-Z#nm|Bb)PdAA8_Gzj=p4 zwD5XHR^o-s_|-Swt1XfI0P3)}yq1X>`{^(G?r#;n%;}x|-~+#oS;zjHX@75b5TC-xPd@tN_jc%0AM5-r zqxfaN+(=vh(Y?Pv{sGT>Oyl0g*Y7?6Cfo3xo?jfF0UF==u^vVxP}3O?`#s<#=-RkZ z9|hLk82O*p#h(U(UjS}kLt&u&g&^rEn0wV(0;V4d_674n;0R_A8D*dcilCX*-~M%A z4elKO-JlM(83@{-4}u-i=-Yi5Qt=(25tf(oT^Iv49}Ak8G4+m@RiG5c77hNO6>66Z z+FT5#RS$At7G~dXiPS!vU<#I@H>}?j<_e-s|KJvWq3~59_2A$R&Y>LUpWWFU2Ew2n zdg0D-m9!z?5gK6- z;UK0T!A-#;+Qk|^8YG^gcR6D1P2wK1qATv8i>ZM9q2en7&pBn4Ck7%Qk|FCA;wT1K zn*E|8dY~%Oq9xLi1xlhas#P8an=w9P>D@&bnjkMCp&2GiC^8|0mE!w3<1s?x%)ucY zu3{^WV>zBt|U)3B=!Jh z_(9|{yyPFsXRRpRx}o+UJg@1E*o9`Wl%b0>s6x}PGv;ye^41|K@3aV`OIK{e?noZUEk0+;G-m-+^Uiwxf;(C1$SV zSq`CEekOLRPh0k5PJWpL>dj~ZCv9%wWd`TH4W)UGhaOJnb~)$mr6*$ACT@-#RH7Ge zil;4d;{|afdjh6I^<{K2r`P35K$T@)T~8c(Wm`RGSXw4S^(TB9Rd3zqd|qdEUg!yI zCly#7m*ow3c4vH!=Uf6P#L#^?;!Qf? zr1cbOfsHp#9)u1lj#*!jDrk{DX1W<@9~mT)X4i{0D0EKDjLv9>Zl;Y|sFsqKhH7JI z1z5jeN>2hLh~noL-qVQI|DP$!Xe}uzURLQQG^9hOs8JQpak^=3vT1k%6PLMUmNs5C zQRIEnP~bcdl3|o66p*zs#Y8(xWcKvrYpgk z>Ah_QvLcx^Le-$=|5^%ZUFPMfZEEYeVrz@3>W^~cu%au+0-nJ->7~|brZQ+{hV00; ztY!jfC$uXkzH4Zn>W1RWcX9$X*j}$P?5pOBzaFg5lBc6xox+MN3EjxWvL>#&Xr?0P zaX#qLE^H&hte?uPMIs-wHj3g{ZF`F8nwl7EYV3N#=hD6-((=u?R+`DCtwL$4%C3dm zx-1_~Al4pMOvbA{vFfTG&d&;I&_?RerfKN$DBVtNy*etLc9+vOZh2xX)edfm!ogQ0 z%EM-DU`*`JV%_(#?9>LX&>pQvBJSbVSlj07q~2J{!Y!~C80o$)bLq|9CTpqM?2c}( ze)>-3NpAH0|6{Gz9KX6Cw8Ad&3h(L0Qh%aZWkM~U7VqSW=y+T%pfb|dQo#TKqqIV< zF@mn(!tUrk>e`;Hx;0~7rY_E{Zro03={oP4vEl5R-_e<>TN(fv87jR7uMUx|=l<>J zaqq_BZ{)@!`F^R{8m#sPFz*`L1YPO9*1;x@VYR800Yq8n?k@oIrT+fr=eBAx&Mo`4 z=>X51_#P$tvab4mFam4lhrX}73YPrB8pjOl1uJF-yDIe2A_t?e4(}>b?(kWFLTn0a zur^o<%ijzS4p{&P3%Bq%s1yvBZ4n=u0c-H$E?B~@FBZRI4g=i?V_o`g>i%YN6tCjI zp562o|597>BmFWW|8ChTH0u2xaSQtIbjh&~hu=b>Cz_7&j3{X(-mx7gFaz5#97r!F zNwBjD^7hv79n*2x-l`aTFb`L%bZYSwOLA)#Z_YNb94s;X>TMvW>IMF5$c8dOk}@Fk z@y5;DBUexdzu~_|vJtDY?rbtABg{)G88*sBB5$z%LNP7FLe@Rz|5md0$uhi!s|}-V z7cbePYGUuZ7A%qFHgHI_`AonE>5G(ZZ#&bh2RYoK4AdfRXXK6d_ z=os_tAA7W;ajH7%^gx4eN&9p&X9olbvHv|(P+M^ctE>u~@JcteFh^DssUAH?);IhI2U)m-yc$*Ob} zf3##?@+)I9T0?UfW3p32uUB;HM(~;DvUN*mwb<%)U8AB^v!SB)r;D1arUtfSzXxM) zHd{?-m%-Aw%=BZAb|VkAU}rMw9yLRv^g{oyX}>m5TQf{w!h|JuM3N>yFSb(Z|F%Q7 z_4YnMX-9Wv!!~tKcatu3+;Zw^S9eX1sji5S zG>VAzQZ;95HE*A{7|W`3m$!KWH()chaqo6=&#-5&D{yF*v+y!aTlaeNH^PSRcKfxI zF7-MRxL6uEL;E*(2UTR>a%PS9e&hChPq>7mcYOCZh7Dn0Hy=|th`X|EL&H?) zx0hx(iYs_&2Y6mL=1ISIaC?`Er}!z4_-`~Td55@#>-c2PxR19u;j(siFY}M@IN){# zUb?0A(s+^g^@2Y+i$^)e!1(emDU>rg;r65g*lvaM_-%W+k$ZTPUpP)z|Mpp*IcAIb zT7xtHGB7J~Et#u1`!+U~kNK60tdN^Ij7qtkFXS*dzyzbJoP+tGhk2jZd7-1Yow}!c z$M`v8xSa5pHKCI<%X# za0mC9w=%QuI*4C;3*P#(cYCz2xUqM-vJE>thkCDj`)MY-pr`t&x4W~yJGy^6dAPG# z1GgT>`*GKLR&zT`Ydf~%ySxW{xR>OoXLo`Ryuh1rz24LL*Pp%L z|8~)Xv34K(-}iUgvpwRs{oXHr-$#228U0PyJ>yS4<>5W$!@baFdgD9!$2$qRgK@EM zKJgy@+LwOfUw)Rghg6?D;!@MYmF!4A4F)n(!%PEn|zVcJQw%@+>OTPB6clKs}?^8F?TeiJh=GF85 zoEty#AHVviKOF6T_S15Rn?LNwK0d{NPSt+wVZYtm|N8?0A|WI$ipQZbiDWLFP3Ut8 zu~e;BtTMWla=l$}_$$?x#b)$*tX{X%Z#O#*r_b$my!xuL@cn(Pz>UDXJ43m{M8rkK zM#jgugMb0b0T-gnOw3KqPR>uyP|#7)Qqoh?RMb_~R@PTnRmWJ^T3Q4uK|-`tNZ;8) zU|})WVdLN69$|BaX;>vX{P09@U(k4u6fug0P20>m;ZZw4gB|5X{(NpBiWekecX;Y|A zr4Ge;^QE(tRjn4}y45Siu3_bfeS^}9!aqD!lKt9NtJ_2u%}u3i)NWn8QQzv#iF zs&MxTCOp`1;Wj$Ys;vhw@5#e=1M{6MS+CN$k0Wc2iJ0x@ihlp@1GQ4m=@t9h0Lky$)P`?NfB=JPnQbcjZ5$AhPMHpi& zL_*mXnve)wMaEqSz4K0HnAGhS`5Efn|;>W zA>b^QSZ=ok=~cCO8TMLou`L%azjn&CigATBw_O5-RTN%&y?u8gFwb@NUVb&zm$k)a zQ#apz|Lyk)RP9x`TZUyp)=*g&_EX||4;B#_8rN;CVv0L%RNjU|_SjhK3O4v;i%tHt z46LTP@5Yp4-cehTYaZEVPzjb*=7~A>8D%py?n&IAi^kdLl0d$BX`bVGxVEPKRXOT{ zt6o_OgrZ$&;BbMSde4M$W_s)a|DR4&Y_$0#+v_CFF4>y2BwcLfd}Eb>%Z@Qi)OnUubWl9HB!81ydO`zQlW>g+;PY0uElb>Cy(##%fS-; zbgBJLeTz^H*IHWARWBVb#zW`a_TMtr_4dvALybF+S#aN?uaKC0T6R~~c+-5$bw%2Ou)-Qi=Vbc0+M!)zOP=WC4-Kq|fIpi=cd|L1L#2( z9p`Y<;NSAtVomCX`P0 z060Gx5>bhG7~HgMBr3A>&^1yt*bc4e7!*ooh(P>}5x1zeA9C@Fbu$74HCLrj;ixui zBvc#Sn3F)kQI2t>Vi>{UL&v;Ah+-635}7bOxcTvjOC%(jbT`6W)o~0o6q;WSrAR~O zkdAYtBpnTu#m5wokGJyT7++||O$riPX1tE0Sm;PLu8|O`%t9bq36V8{l6A;B&Y+0K zi9LE!ls-G2jjRV!1(5k}^?8>}BkThDj}IQkTMMW+_7{OKeW_nm?>2 z1jThRTD~%sh+0=0|FfA2IYKd(m&oNdwb?*108*HP4CXvX#K;oPp_|E!5}~f>nSJtP zZ_s?FI|tOsf_^fg^?VCIH})Jrrcx!XTxCVa$wo_hDWE;E0DIcGP0t%2zPjOmEd16(uS7ofG=1NzLX*I9Sqbw8SShBTb z^eZt%5MeV@|5KY9mSPALEnG_*SKdh$agyWgMDc1%TM4SK+i=sS8ivnRd4%EAH7Ohf3ETb)DWQ zqZ(g((OZJ984SVgQ+0b?;R0i$kR5J(>wDiO%-5J*S?+9q`x@F3@1>D}O?9B_fX{+f zn(*CD`jEun?Lt_>!BMS1(+dkJ!e*=+h9pqRTgkn0_`&LJs+yuZVeO{O!P&%bi|xCo z3Wt?F0hwftL0sEmSkcBi{wJqkd4&TfIKnQZA&RY3yc7>PYM(<#j9Um`RG!zI@Vcy& z#X8w<{~GOv9X+v<3E58-Z08jOif)^ZN{TkQH zzA|#3z3gOP?rfv4O;AyUYGvAb_eUS~bE6@X=j5_@%#$WFjJvDn*Mt?Uuq~01y!zk4 z&~(p^gY@1;?Pyg08PI`dEu!xV>XLBQ)tr7Dqff0M^a+^KyRLMvDSPQiLphOH?zAiQ z7VJ<%)y}tO>VTR3>{KWD$YlgJtUnjvSzFt~*!C~2>tgF@Lz~y*_BC*UjqLrc$FEcD zFGpS5+(w8x-18PUy`^1mBCVR$rkpi;M~sv<_nWr$0ynTrATz>jKV^=U6Wo;i< z|FzzJhrE|1@TpFT)$om5@Jj&1z_E zc#n>ao!8{fPbd4&&3^C0o*UUUsd9+j?wlLDJ;YUV`q}MH^{M9_Js$tKe1BZYxI0_o zJSP0yI}`S@C;skZ6b~iH@%Tz$q&H}XdP1$W&T;m&v(8Sbyqy)@qT*R zDFv;BbwKMQFoD;z{s3r=yxxI!`3Q=IQ(UOKH zRSK?cpKG;uwD)efxxi0TRAUQ3uhTy-K)mxi!0I!-A)>PZSiR_@Kh<-*2Anemh`rgH zyyJ5x+_NU4b2X+jo$#A2@Kc5I%fQN0Koq>c7EwLcYrPAMKy!#Z$6G(wyFlqezkk_4 z4h#!4DY*>$K^^;+nFGRGZ>q993#L!~IOgz6(L`0OsMJSv=46H>l;=6L%ziVr) zN{qxuy0T(?HC$3QT5QHr%!XUkMQKb#jDW*ud^tyaM3L%0R!qiL{6?)?0$FrMa!f%W z`ow8W$7qDZa`Xg4D%0xyI_r zMwx4}KGa8Vti)F`f^n=u)S^gVT%IR1$Td{QTokg4{KZp@$eFOm|6?pklf=i9yhmgN zKV?jSe*8#(1V}sM$bzg%;*&|m2}zZbxBz@ehx|#co5+}iNumU;jI>FlJVcKi%HnuO zc=SGEbjXy0!!)x>pgc=k63U`n%dC_N%%Dm4 zfy=lIfc0z3BilgkGc>!(Ik{6wszl34R7ygj2%!@fese_Zv>`Y|~%exs% zvJ^ImlT6)Ii3?QB;#Lj+6VY)YMcH>hl*|I(zys(eoAq)O_v0O?!G zdSIE%zj1EmYjBuIg5 zz$i>g&&<$G8qbO9KJURw5zWp7T~XCY1+RQOG=xDJJj@s*MeGVsNFdQHh{vd_Pz?Ri zKI_Z|tx*R>O7H~IAJNZ7DzpOy2NXq8cTvzLbrb@a#tt3QBAwBKyiXR@#O<6R2~{-c z!4w}2&I?UZG%Xg5)KXl`Ql*>(E=^MnWT~!V(#-qM{{h2O|NJIqlv60xpf5Gifs8yE zHAprMMF{Ous?kyU(n0}!Q~-_CAM{Y)^iwnaQ=V(n8LiLb#MC=sR7}%LJ%v8G3QiI^DMmJqrFPCP++yxRh6bx-9->B&Kn%o zX>-a|G^QPF&b_+FU7c1OX-{FjRwT2`52aD>{MBs5R%-#(wdfrQ*-a0O0AEd3WnI@) zE7oqMNp=kquWG440yxstReYU0YmHYgZPh6~iZ$ibCU_fuB*pem8LR{HfBv_pdADQh~42#NqrA~k1f+yYCpoLkJ?T$nBSuOQf z;o?{^RoKXcw=xaZh?Sd(Wmk5wmTg*jRkNN1LwX53khN2G3Q@2%Tc_1TvYoxg z>q-wTM7cfLQ5st83R%FNBkkDQxGh}clRUfKR1TfO7zA3wJuSRUTR&{DQ?1;Etf0o- z+qu=;)=VomRKF|D+bN{KUff)&9XYF=*}SpYQZ<96; z!c|`IMH@nG!S;(iG@RY#Et^y9FQWBWw&lExBGB|@)$6U_WmOf>T|L>$+o$E*)9nJ8-7trEnx<=VYHIq8TK$%YSs%D;>Jk2 zH+s%D+E*I3-zB~f9wyLO4PD4P;T(?IcYVU#ox=Y`9i{WzicvcMQ{vc7;86NuG6spn zjnRVa$gy47H^taved0JZ<4j;*|337|7KYZ7 z9AjdZVOJgpSWe70US>At=ff0ba1}c3 z3IX=Zf-^X)Z@uw<$R7RnFeX97HO-lYOKC$ ztrpVTO=?ih%P&sNhc)IhZskkP;&U$Qq9$s#?&-E3>b1^lx!!6}&f?sqAfq)`qk3o! zo?NKDXy8QaL8a@tE^NYHTtYrC4XSvJRF}Tw|bi z>$rw%(GF_R=H$_~(zgz6)mCiQK5fp-<&oj$taaPww9dbdY85(bjb?1#Zf$Bj)VvLD ztFGeSE^aFZZsbmG|K-+Ul@;lX#vNwuerdNNYunN7fh_IS_U_d7ZPyO& zDJE~>7U*~0ZR6%>EiG^N7TrOPZ~30@8-8yrP3~BZQTjFIr4GluNZc3(>=fee!Cr6i zK5*2g@6Jx_(B;q>TyO-x;zMt>&hfyf!#Up=mDM#`sr{W>cauVP2Ccknjx9}++a#~Kx7=7%%=x?dCsn-?onwe?? zfAa`O< z;l7Y694lf4|GHKb!JEH=5B99Pjq3ob63A> zidf%BpIjGDUQ6$G8Hed_FFx+}an~kyb3b=pPqYU1#%<;Ptr1;eKcEW`B5MH*7u+_ARe>h`;z&C)A2} z^@;EE|7n+Zv`P#M&h~}nYjLRbaBm!MuLS~UcV|!Tc@OpTruKcWd3e8hPsjOo7g(7G z-Fnw~d(ZK5&-tMbdQpsddgu66TuYLQke?#tibnH$ zy9DYBiA&C5?&p5)=YH39F>cmL^kfA@EPxE}6#|8gK07zT=B0)k9hluQO9;etw|ROoOJ z$UdV4YM1K`gU7CKS$rm&*J$$=4Ufy`Z#tdb6=JjezW)yxC^$%1Xn2U2sJO`3==ca3 znYbreX?cm6skzD7>G=s7D#}NRWokN-!5FEIdqH zYUTJ$OBh-Thtd9j?ut4SjB&Uak&)y3URs4?kaCPw$MM zUul2e?=L>Doxp(v`2{>^Frhzr4Ar4p1(B65BCaaBBBn2+#*G|1N{Z)>TAIg^8bgXK zNHV3$l`LDje2KE*!-zCZhJ2(jrp}!?5%NUTGw9EuLWvTciRH>oqfC7^mANpg&ZSVR zO1)}zr>3GGpK|@`RjW#{VVy?JdiI`8o2+QIH5CK^A^8LaPyi+XEC2ui000C40ssd8 zz<_W_EEh0|B?(yE4oMxKJ_{g&R`ThL6x%?#yNbsM)eghM>0yxkhK4TCgN`zSO zRIqH=j9p}Nrem888#;vSK(eIClPFWFT*ZYXqar3J=&!&=m*sa}Qs5W2?3H`|7{`2D~GasZ?q(icwmtaDV8P*=4a@b_o?s%tCbm2j(rT@x~l? zY#pF`{uwgJeR?eHnC@tsQ^_ zuKcLFR?ls>h~=tCx1^Ig0JPqAhqH|$!2RrUJwh8y@4fg^a3rftM-DlUkMC>ak(C?i zBf+5j?K#2`1nRchA7Je8*T$wEHU&|URC;(`Lxr*Eqt|Z#`|Ts=-Y3Z-?;i5na0hr; zfaFq0siMatzqf{xE^c`8qL%7>V$Q#H;*<~nXOrQca7Chkzj$6gMUdxy# zt_mv8ddoZB%b>SB#OO>@Xn4ZAeulHrY-(}RGg<(KWIpu0&wNo!62D&fBhCo{gS&D7 z?G{)qu6YMys*~7yl+%F^XenX?%ov=|6A~Ue@rh7W6WrM4HxE8Yc~|7y1}#W8ZFJ9j zE-T!)FonYieGpaJlLHyW2*&SeBz#2UOeth_J{mIr&3!R+U*zIA#BNy+zsgo@b98}#wtROi+7_Q2kEyV19{Sq9kik- zXGSLeNXv(O7hl;dKMeI&BCweT|p$Vg5OJV`pL=vx^>7!2#>#y!q9k5Er ztR5rcyI9IbbgA@nY;7Jx+p59O#nYuP&E#iHlTV)JRH+bI)XXje&|V6aphh()Q3IyX zdb*LFfpe-@)g)1>P86E@Gh|k;xm7l8z^eqDtY}BeF_M}TrCsb5_C6_7EQ>o>()vhkXCbVS7e(ppog1}(nzwd}RjUFj)3x;C_mqnBAsSx zzh&<4rxjZSr0hZOCqPrF&t6&PhNzgz&{>tL5HmC7fL`0j`dl(Gf-_s zbGOmVJ|qwk?rhI8y2H@HcEKkeR&JxVl%DP+;8vVHMU~pc8B88a{U&ct)i|eaMx0^< zt~lSK#>aftw_*hy=spk0$f^z%PJ@#HiBsH89l-9gV>9%GKeIeOH#j0SAnEfc>*r7( zZI{=Hw{Jfv%<+x$sm(n9U)eVF$m@RI8<_A75>5oi!q#Vf^E`^+64~Z{etR!zPOE=Y z?ZN1V@71FIwu391;S1jqW|JNtk}xuCmVOSI5Y6|-ceA9N*85CZyyEH^#_M=PHJP*S zK_tt|P}Oj?Klv+5m-9jhFh44;;XZe$5xLjMT`s`Ep7!}MgS?K9lhCXC_k;(L1D^GJ z5T|{fGATLt$UhnG(raRof84##r!DREUT&CL-Q*D73>e)l&!E9_&N{EX%3)m*e6t?G zyX3dr6FU3V>Kw*~!CLZ#C4AhhPT5l(n-shAaN^sE|2;Q<0tiJWF(B#WIbzmYl!AF- z6jR~m25eSx!*+82S~q$XC~KDT55kao=NDw_H*-p|dP~xNWMpWz=PQI)ZP7+_7d9Td z*L&jueEU~_s)J322Ri{cfI7H*g8_M2ltl=bSF@EtY6fyfxO1;{V{K$0+^2RP;YH(S zf*IIT!QctRV16=(STUD=QUeP-#~{~NHS)KE%kf!_wse3Oc=~50l;%ulvQ-xL6gfzS zbO=1=lz``Qao{Fu^rK=h6MY;9LE;B;6gV_=MtZ$9YmLK!S=c_1vwHpJFLu{C(toj{3T$8zu&I%LQ-6~JMQMv75( zhC}pp3?o%lWjgx9e;USktay&E7>kajhd_vcc$iut$AG8Cd0%Ei4b@8DwsQKYa$Sd0 z2XQovn2fcuj35YtM&)35CyhIId*H-(T*FA6=!wJVdGiQ=8|PXrhKpQBee9H9mY6P{0)?{{N5I&N`*DP?T*na2K zk5)sC1L#BA*lY`Tf5R6^Qxbya${K}1=S8_1TJ`*UR-d6o5%iIk})B@udlczNA-gcuicrE-LjwUYN$BD{E(zY&u) zNoVyHj8r&pil{0z*NAkqj6i8r*Kl_Mr;xQ6Iv2TQNI6yG5s`;UoT@WGzbP!r(ox(f zJCSK!=6IRXsUOO*0h+0R8OK_I*diVmYHsD7ys~+5wMuF^Y^;`+GC7lQ#Fp$UA-+n9&tL#R-F;h<{(jWI;ETfajbXHiy%BRuR{i4T>fI zcL<&(xklO6-({EKB)Jj+tyZ3LX$gon?ZcXoyuv#Gny+q=Ql@j{-q^vzcEBBAG{z?igb^_mUgh zGoS}yx>$iWW_n5Ro^VN7>`7j}=;d=INPx#h&BmMPp}x={bdK zcd0fhVA4ZAv&oZ=Xq!OUdMS{b@sN>XLkjtUZ%6iPFhXL;Z0unU-rMR^+9ca{;6n!i}0rUGldWsI^}rr&C7$_S&g z(rc_DqYK5V9*TFbx`UOaqkre4($1wIf8k_Fvt-cnQcl3zKC9aplLSoxAlQ^UQ(MYYlazE_q zrfh4lZd*G<^g4H1uRUt3;fPIpnpxLoP63*u#41+UMzGoOw(nYSzey74nsJ=DuneoL zOPhD7DXEl7Q=k!sClpv|&BDg|}jwErN!f~_^h;@4yrAO1kDvJaw`n_`LqIq@G_o=H7RWvTO0XPLXP96OU9 zdx7+(2BIpmr8}l2i#}+oZV^msu`9b7fI4*yycc#cz*<%Rc{&|k^_co=C$sFsziY3F zC&b1Kbjez8XIH_=`lONS#8UczRa&)^3%Wf+wPv@LmI|^-Z~3d&2=f-7~$j;%yAX&U~Tg!xLO+2f^`)h`<6KQio&w>2A9rnvlCC7(4xCjiG z{mICMdnJf#UlAcc5ny?g%y~?eghW}fkD792guM~A!D~6n(s;GpoCM{2x+NT+Bn&I9 zyuuV5A+apGfLy=Esc^EB#~FoIlon#v#-2}-PZB? zlng^BJWQ|lOe_r7I*r`R-FSy=C-i`m zD5L}v{j5zqW?PNP7aDe`X{Dk2$=c%4pR2~jI9IE1(keYrEV|MS=7svS*76)(j9J$) ztdu-EnbToq0_4xdH>?3Iu>L31#@c+Aq72=$6d5{{OU%MAoi`IQlu}GzSW24h)`YRG zt*;xklno=MduzOO*5WO)C~LxagvO%G$!s0Riww7N&6tMA0l{6<0SjpdWT>-C!)QZ9 zuyfl#Juz*n(B)KN5DKsT>DX~CfY%l$$l96zT)C+wz?9d6mEcG(O*h4He13-q9wUL48qne&>cM6(F&k z-pJ+!KIaGC*o>~o?ETh=-p~#?GREz?;=%-syX2x7k4?_3_L${r3c{KT;vO90PB?2n z)x{@Xs&Q9TWR9i}{&hxbY}F|`hDVP7HXhja=hLOgyH;jhrjsVjN#Ax{kx}uhBf%ZB zJ>*%|;F+>KaiHWWC3Y?Qyq(@@8h+IqE#fd`<=JY8+da~Zp=ZcQrYXIJshjSsg5I2p z=6w0salSBSwcvSzEo_XLpEcvUZoh%f?_mOq`OPJe$sUa9cY+S&z0Dm>WY`G4&~Q$j zioNIp?&g%a;KohW5k5lE>);t}-7TrE*_Z9GnL?qCXBxX{AzkX9YVvI`N2L3$^QrD8 z9+YMt>#}`m>4hzaQ33cZe7Bv?eM`tr_n51bnEp5Hc;0O5Au(M;z|4+(5B==j$+Qn7 z?MNG~*IwbDMsiPEdWl1f9)I2cSA5A?&Z*?y#pm3jN^{OKZ?fyIE3BElT4$s0{^o^C z?{@E>de7J|ah;*L@C{=VK7JeOG0X(|Kd|_#j0QW?*5}4@@G`7f$hPo#FY$RF>6hQ= znJ?*_KY#!1?6LODn!4GgOjp?3@w@_to4l^q7)KKzCS)=@K}XX&x_bcpi4)Tv)hR#^=hyuGHIRQZa?HZNFyAJgz$aC{Ix*K_xorH?$au?eeSvpve~p2MdzYD;ou8qjrKjb#HWZAQmZZ45qZ|$U60F4}GReLQ3_i)j+10UXTUGW?8dm(WX_qmTg;T9V#GfCyy@7 zDG}m&tGo9AU%zti(e){}FU-N`x)pY?7~Q^m1q1eT`BEKD%LOq5ew@y6JBXhHc(k~J zG)a)AQKuYTnzZVQ6iv1+ZBhu!g#jmjo3*7F?_9)yBNP66cyHsttBAv9K)JklQ}_hk z@T$|Frbn?yvf7k7SnS-dcG$qd(}PZ7t6se_HM`MOUcY`~WR;fh`n&Gi*T1HCaN_@u z>rcSn_yaH*x2BOv7z7z?gBCjcIgGM)#Mx{@hH%piAPq0%hOumV>g=7{4%(^2BSbU@ z#eDE0D6`K0GR&r-7x_66MbI9C4W-n0Jnb}-M(PMiAb+$-C)>!0vBSeAD6qjN)!Lwf z3Z%?JO3Q%Cg}kuXb8joF1ex!=E}znlgAc^B&XMl25=D~@uG6YLt^%p^ys+GK63RXK z?9k3&ObyOUy3`M}+h+Neg4tuDwjO@TE-Mk&Hu4DI91V8Y5M5 zvlL~-@gdP(>(NyoPZ|xi*ob7+A_^eFa7Iua*eyT<2mDo7|7xnjSYN9Ybf^o+^l}kP znB9^UW>@i&6|brYlMwE}JGMM)t%XlbF0INWzjJ+4W50FlL#DvqmRT@Z|KP30mb2u2 z%Nd#f0!8bM4stRGQbON>4l%o4{8V8|mrM+tmGsDMLOozy(Nh^mR8rJ{b3@g$T}$s(uk#L(fkpIHMWf7$wZFO)asuH zk>m&d9$fyzTNN_#`zuE%^d7f0sCGI1=gwY z#Z8BKBZu8cbt=bAjBoF=)%=jCBU!PmRrnK$1Yo8s94Y{EyR#e!hj+XzV$m48soZp& zBD$kE5KKQS9k{d;9t%zoY7V591C1iPopmJ>y0fDd@p#8{NlqDD^ds^Bxy4`gv5z4v z|6#nihOxAr4Q%5;m<;h|p($!FVsblGh%8nkMOG|_L5#=78kfGfbZC2F6i|%mnMEnkGp#YX;t$OlSGB|_ zia;cjgY?N9SjhSgo|0`ssfpoITSUiqFL>crnG#fw2M-drZa`;N#9rGVMoP$V?|DkP7|1k~RP&K)eW{wKOXPV<9KkPE^JU7dP$JEyLWWHMLNsjD^_==hw9yc%@SLJ)C6>=0-qVxyWauXS(a*9P z^sHqKl3H7)Bi)h&p+wLXT-Q;@nOYI2{TXgee+tpf{dA2@TU|>w7dqb zXVN_ncAJUDY2|5-=+WR(RD4~%ir2C9$z6L5*^C@}n8!V)$)M6EA+I@aZBt_B+UVxle|;_OO=tUz07N|oaG{-@`x1na)n!C;T@#}R=riIjTUSn;lB60ZD#YE zmzz=Y#CT2|ye_kpQervgxv?M}>2$%R=fc)ljmLEJq2D{N<0@B=JcToQlN&kC`M3&E zR%bfwD_Sil#L{4E43rg&RoE(W)ICHq`$C1_PAnb*OQ!hIYNX6_)<>&hrT@8xQ-QW(GFd-)$^n^%^aQA{t#g#iozJ z9qwQddC1vQn>hPoWK9pT%HVXj3vCGMhQV{njob90fH-9XgLR*9%Qtwuea^Fz^Wr7v#YVX;Id$9%$gjXhT z9=Q_N2DgaQXQ|;OaoCIEPRnwu<^`cVTjZbE_@Cj(*lHuK|GOJprs_hj>_K0`?T=mb z%;RuPukW4jZ)saOgtE3BS2K<={LXR6td^S8MCT#-w5n<-gpbvn{JB9= zg!Rvm_13J?y#3OOKME)=fAtRm!lVCGpa_5E!Pr;-iBILytBdX>@6rVJ%<7@M0L@hJ6pqZB|6yCkqZ zZcpT*unKMOX*M7W#RD$vD+5dl12IYo#SrBp4f)n3zt)sCNM71e>hL?rR3= zP_yvs+$d(#M#f@-PWr6M`vT19whHM=2@t0bL*9pe-cNpdPzZ+v^bYTw_AjWU0R*E# z*n|hiiqF{YDovhn5UlHIu8je$kO~)20-KEja)L4D0US1FW2VHR(9jauf)+*a7NZ0l z;1J0WkMQt;1%DCUdhwmk&+^pg4k0fY)6E9!{}A$Y4)X|Z5eJUeDlFG10*YeI-Zbn8 z$FLibkQSIQg7^xJ8fYBlLkg!b(A2RM*AWt42+4FY-FVL%>G2!u@g9@!mS`bha*#5h zuNk%MoazD(P|K;d=Z0|bs{XM1iZ0bEWT8YUlejO`c2L%$&VF_a2wlZxs?HH50up%+ z)4~z5%n;^=2MLYKKDw@VB-)owbXA_6mcwnu!+Rd zBSA6}(JeY;uH}G|c>K@X0!a|mvK5t+6*&uX z3h+4ts3@hgSl+M%TM(TZ#TT1$-3le~+J@c&ap~AiGO4U#6kzi7&9!6@AV}u6G*TL; zt}Hv!8dCrpeTyOkQZVsxKM!pS!w?*Ill4|=IH@ol4Kz6UGC`XLFyRq7kB~4Sls_dj zLbGQ0s#66+@*ka%=LRyW^2z!V|E~KMsu0CzA&E#3#g9ZQb2E=o^bk>1#1bP_Gb39m zw>(nUh|o1(f(im~LVfgH!f}pbY(Ud46czMH2^2|<6IxbrId}3&q0|*&(yrrYv}^wJM-3H7Wokd*;($U2+Y*pS4>VGh^hg)bDH!xGb8;jjbW;zN zBu}z10SFGR!7<0PVKOaFYl#@!6cPPxBK@#LSyV(BDMnf(MME#?icu}cuRU=MK5ea5 z_7qTClT$cUQ}^gL)o>GC|Bktuutyg)CQVGwK2cJav?hg9fnXsywbe;+a?KW#goaHh zr_;mc0PrA*K_x^xFAXVgt_4?dMW<3T^MO38a-CkWo(Az^rG7IMdW{*0t^xB6np~5AiEOBSr@nacDGY zRWtqo^@(!S!N5}>8qTW3 zGmzA_|L{e4xHLhA|8i~Ha8!X&DcM2YXiy*>Cf@QDFnTsKV|Hk#>}NyfWBOL#+S6$L zm2l)!V4X-u(6SH@HdoI*7PbWNd`uuHYWwrbi&O}wT-vFb5fp>}(jBp7zPyGg1 zPeN&tHhMeK^E|SxsMkaSmm98khn3KC2kk zxTo;9Zcp}DBMMpVcBt;RN)=gO?Dxa6FbnenM9h!UpFP&R$Qq87n*|}7$>t#6=uDX@p{*AL6jkrca~2rR^?P) znd9h|cY*Z~MqBxLQ!|XEcY3qAlt5XH^XPu{*+AYFgh{V^S22$x6(_{k0q&S$B_J+d zcu5UF2#$FI4d{KNRH7$(3JSF>28oaz?OQdP1Amx#7TNMBi~aa?DNdfk7f}#9y%DBOEv>{4~I#3IVAO| zTNs5=fqdPg0`35z6=8dd01u%0p>1-Rli7w{7pZNT&74^b_j$d7*nONXB*l}Yi#oIN z|8Ok@7a=u%3BMG&O~aG?%QA z`k$M%vx=Eo30f60;GmCU0z5k%v2J9%nkLz|qWu`Pxphh#E-{IVS?zHa&k$4*B$4}M z1Kt(u5C&Zp#>h1Jx6_qPmG`a{38n>mGIZ{f=ybSm8haWym3KNd<9U^FE2z`*u7y^H zYMHX@F$0^`KjZRP__4HaFCISoj)l1hSZ|oOnjn$Xm>WS%!CINax=FU%gUgz`_1ja^ zI;T$9pcy9)50{(Q6^Ts{UbC#}f)<^hI2o1hPH7dCHrTigw{b7Bw!(Ofd)jDm|J1QZ z=}e8azeU`=9ySXV^+NSH70Q>hL%V4}yT!ZKYwt4~N_&0z7~Uv4wUhIoF>sH9(%Y`I zBz=6yvG?>ISp<`VZFU=_cXSR{8u047rF~a48E?~K8r2dOJ>&YyPX$ILIJ%*Cx;4+b z`6*WmVEr{Cml5zAYJKizbr2 za7w6Tvh$liy<5%G5Ruu1DV_X*jhDbV6mZSFiSar#IbC_z*`7;fMryj#L;1L4>*;J% zo)KHs51VNjyES8(L%Da+OV-5yQ#{`F08;!E9^w?N8paKQd`(y5TBI}^|9aO`@i>!M zTj^W10l9^CT+uZUR1ecIOV8Rh@yI*cOZfzEKdA)|0xQD-lL;6qh5LZ9c*;$k$uuNt zudk*Tn9Iq5%Pt(u-`On3oO)%s8ONH|+gwuteskag&Vv|Hc3lx5A}?Qjp-(~B4^$i) zp{@U%W5H?AEnb zugO`%9Ad1Tmk|-0%mkK3nf9<@1J-ruo}XjZ1$m>l{f!%4=`YHS??XHcC53st*AL;> zhkdFQz|O5Dd>`Q0r5>U!A<)Cx*=^W;6J61@7t)RA|JV>EKYEb0|NW4xz}uC|$&&b7 z1y~pxw6s9QuKRI!8LHlbj&2bB%DTC^DTCg7(BA7k=P3-9cU7M13C*8Agp)d3f(Z=Q zTxSX`ybTSP2_EXvY2x$J&+8b^Kl?}+KE`=@&PgAA+4t_X`QkA(kTt&JAGXam{!l@F z4%=1SE8VTxROP2?o_ID%%$Xv^v&s>?`GG$i{+dMAeR*#~0#bd|u^;Fadq*ZWdY#t8 zLo{zKo8vKmcHATj-9#?GLKJkr@;%q{*V=hrpAfR1>IuQ>S3lV6KmVs12r?7OLsC2% zlgOpA$$Tm=Ng|cvP+$*Q!CV_&8th@9+KP z-PPF%0v-wy#t}9e_BjSd7E($YQdVMW;%P3nY+Qj&q^Xf|m71QKR#c$6qNRyCa1E%Y z-qPk8)Y|T<_TmEn3KI_($Lbm^KdXBhlNxu!nSy*18ax`+idVya9U9VRd@nl5;LWeRx)2IaLLnw5Gp#K2md_< z2^FHiY&w8Rboz(YkBFF&_>PE{l?cx~ zG6mq4K5SJeOa>pJR%2*Z6~x*@8n|{uYztjP|KmioRm1^}M`F|wk{wKAL>kdShXhK; z)s!4PCBdYePSHdeT$jtCqF{Doa_60QVJd(obKzx#bWsMwyA1g-U9v zrs`t$gH5N{VUs=fjs5CrF)jn66sReYp8OTXiW(YN zrkqY3H{5R(+_&HLD9o_nk9s;;#*!fpsyd=3ZYtv;C!TSLB5!u^;G%?0AAJG2+}h5# zPN%I!NGepbML{-&8?hJ#ou+i!MlD=ENHe{(vsPXiy42Pd9XssO`DL}YNL{*JD8wIM zZt_HZ?e2P3y&DK8@xGT|_2O9~|KIiX?pIzX({~R(Ci~*Xpzvr|6R?b2v525DCU=GC z+p0O>d zHMs&Ea5#_~-8jTn*&5xFKBA-}Wx#b_S|LeLf|8`+ZYE5~PSbY63aI&RIb`}*8H(36 zt^pAi%~RQs%$B;cA<>9ROd=mT@U8fvf-MsI6Bqgh!7I*>RS%@w+xBF)Wn?iX4~$^o z`qV!6k>!j9yBr(qhN?HhF^+Pa<2O|0z+3FCLql95Q(Sf=4&W+~x{FprYLyTX>W*i< zGU4htq6MAwjwVyu1q^GL|FjBNa*z_>T@HKqI||_HQdbM&8hZG{AMVkDZ%R+}uHwCP zkxvow8pKp!Igrg!k9*NGUj+#CK7)Z}FM6_~UDAigHU^N5VdUaI(s-z6LNl8FOQ!v> zahL_tF`L@l<~r;M%r25qf=Vd}6aUZ#hoJ79i2NNA1PL2KN{s>}ETQU<)Fc+-Q+6#n zRPFe=1RB~-cQ?#uCvj3UQJRT|q&(%;8o4AsmNTLdohaCxD2BAjZ&CJG<}dz4ur)@C zqjLMANV{k^2$i%^$Q;HQrFk@Ip0SK8ooOz(Y15l()0-_ySTfnLNQ#;>ghASX&TJ)x zoq*1rpj_${PRF@n|Jn1A?40Vgu&9#Tk!6OMTv|LSz)6C7a*#d>q#*r*tc4nHTxwk_ z5Y5n$4{iWMeVI{cJdp`Iy>fit?1F(?Q8L|kH1C3olix6B1&@6aoHG@JK zDz5HAF&5)AClYU4M2Vsos7$md7npO4G#>3y&IpMW^Juw|p0qF&Q)%7$Ytm}AFPf8t z>7i0WQx5fa|Fk$=FdP%;U~uSEi!$YqY}E@8mt`O}!kq$Dsnb>AqLZFKqmh5A`y7@W ztxD~C!d97^ldqZ>tk>lcLeYAL<*~J`2@Q$^GWWc8N$|_}>cdN}>Npzh?=uU2nIvLubx4`S&h4L^c zbGC4LJI!hGju%IXm4z`5eW1r;nK`^6W~KH^Qikz2FD3o*+YHR*mlimtvp%rkuzLY+ z%1m^=|J|_YG^wz>F?m(@{a-99?CGeevu1b>>mYF~ijH(gs=tl#10t|lL`&n)ES3q; zs#Sq)KU#OnTB@3TXz6!fnxSI4pq%cyN?HQ8u$mjJUk7XN?e&$GaL7;vU^UBi@p8#` znbJoO72b{xEQTceaxiB~@rPGjvvzrrfCGCJY>^0d@n#IJtOZsh=Xe0CmhGSg|wQDm>Z?Hrf; z|9~HR>laln<-5L))|(hnqK^0@ciu3!N3ymI00!MgXF|`5z#MTym!DLHak>kdt4X{2 z3(8Bnwd1`Z<`ntJy5wUkp~q?M+F}DxNqCzb?qrN_MF>@sxUIE_b&=z~?f}Db$VtZ8 zv~-#H7Z?74|4wqe%LV1&R5_=fPQj1ZR_emaEzKuL;$Ta=@iiF`Upm!!=opx}Ep~BD zo+7# z*p=UKxXs;^k*lHIhTNUkh1|oK+TYEcmNDN5@><^PUB^A2%H7)6DVv0SoakX+|JCVM zu2{tC43s?8%28oUSIrR0N$Cuzt31&HVg5nssFe3@0& z1P^vxTQ4n0=|NVo;FVs$hrU8PUC>=iJS_B4PUr5;+{Z|`Z zL;Wn@{e@SS^q5s|D!1&Nv zs88(0*~^jFdBvReL7f5i(E%P&Ze18gXo!ZKoZqz|GgQr=3;^%Z-3mSk|D(9zvawX& z4c;Cm+qXfJGpP~X6=FJWoC-!GO!eI%h9C+~liOR$$(VBY-L2EG}KfnIky{;xx`AA@*1F@troBAo2yJ-z_3K zk|G;H;%pJ4F&3ko5!F5>K@DyIySc{u#Tdl>MO(lkO(>+Z09_@qqDe%65+Wo%Z3WjY z)}Z8xTn!9{;NMU*p)LBNDOF)b7Gw;Xj-NNy$Bv18hq z-7(?Kt39Gm;!#d^T>qS%Ix?U05v5DPW8Ar;OR?Y`bz{T8S>gR966s7LVWI{_q`Gl| zE#5;{dS&HF086N1w^5*U5uI#yk{XDGN6O;=xuq5+U=u!3HL?P58bJLW&vm2>Uy4mD z?Hd_F=Q3u+a6Y5Ltwm8foS0yZWZB^|Xs7Vmq-5TtES@K4*xF9E)M57Il}VU=CFM}= z-GITNeIjDpAY}@slrb!2Bm!t8E*>P+%2`6k1w?=ZaHWAhLBXsd`#oqWzC;FKC4)** zTAtqVxaBVHU$}fEhbrgp-Q_P1(No@IQ#R!!#-NFsXmvK@|Mt<@iekiCI-p?w((MJ_ z6N*tS%oj|eQ857q^p)IbHYWb8T{xO&sjU%Z{tad_-IjG03T|dR&fPnPAfVXV1G-() z1tVZ$#heid2SE}<8t5k-6%bk|<$a}J+-8>eQ%*nwnWkxQq`_|r1zXmos?lX}>LT*Y zR|-{4!ri5I;APgCBpbSDU)IT-6>4_+-lelBxhY6jL4}J+(|!%ITcu5SsEP6kub7K7rD-h)$HH1LUgB z6(o)FVJYdKF#V>7rV0P$UtA&&fXroN>7{kPDAXYd|B5PVqTa?s&1f(BBcs3KE-MkzEIB9&gLz8d8`YH0xB3~oi;kX&N04eJgX zl#`Ic!7>2_HVvCDT0bEgN%V!S7L5=w$erq_To#W;N~>CF;h%bBhek~31uCMhR~r-@ zN^Wb*eyQPoYy7<7gEUr4&LMaf7*Ez{rec*{Q>Px<(6&@fzU#8d=Q_F>eL5C* zmYmb#tEw_0Jr2&R2I$o$iBwvwDdK9HI;?UuWI!p+SWakyZmrQ^sL^t)hAu0|w&jS* z0_ctGqy{64O5N3h!WFD+-8yANq3kfmYMlk{|J~vS3dL30I0`hL9578|()K92uBzT_ zQQH}5s}5plf#Z@6ZT~bDz4ljqf@V8PDXa3UXX+~ix$BseBvI3w2V9AEwYyPdMNg@h4t4MO|NxEFO z@>xf~Eo80aWWwa{l`Ef6>g}fP&K_+|)}u`V-+Kuy(M}o9wkv<8pz4w;ltQh5Eu!hp zMNBcK;Oqs~0G{U* zNwsg6xlIq-U1#Cv{7x>EB47UUtCdzM^ATR`>hZgR0FwWY@XY!JQX86fYTM(Y(eZ`C|+GP(@Ome0t7tQW3ur*VXFMenf4 zaSRuTc5dVM+9Vc|ug)%Kp|0HUMwoDW64D> zt5$6l12782l>uMn7Dw=N6qBvGX;#(p#QIzj(nPSPF~SNHC6AK0UQPFA@Ns@@($OvJ z4sIoyu>TqF3GbrSwr<~QtCv32|1@JWwvO6IGSUQ+0W>CZWA>7RObpl&pIGEW4mXI9 z3h7UBG60Q{5jTqFx^I(uuH2bwXGU%Rl5XinZ4;++mOg3#l_gvSnxCmMlECrlCC3$Cciop-5u>dCv8!$^Fa4AKBJmc zrz#Zdol+W)v@-8jLxc7vuUE@%FUy~#MD!4xQKE&V<>@k6X0%2V;S5uwT3N7&F3+U$ zadNIL9V2sCpulN1qx9w?|62JJ%c`);#>sIBZ*W5LvW4p};hJk{(Yb!DAR@{ShqAj8 zHKjaXzsfJ&IDwSfXXgcN~njUvqa1?t37)Qj zigx%io4D(qIN`?Z{|h2=+BNpZZI;8%8qQj(6g{=3NU=~;bxzxvTFo(2b(`-tKcX$qiq(b>nm*;#B zxe_y0`vN%;FL6q7wxZkXV-~gPqAHK?Z;%)1q1P)1%BNU=$$t%an5&9JM{J@wxb9xk z)8NpRKR8^sF-W%;SKqWy+hE?B>}goUrSj|`3{o#tj8^mF|XVfPcO1GOw?F*ShsE2 zsUcfDEfV&1#|m8Kd9a(oc8{OHh-;wNY=f z{&@;Fg5vuZ3nZ*-mA5&WR&`1I(JqA|!y0!Uuo3NwCP@C54qF^b-a-8(#kIxoju84d-tc~=i56jhV#TQEW!8cO7^y^S3M9sghOJ{D0FNbM`rVQ zFc>luXf=AZS{+qK*Sj?gu-XVUdr+XiXLLKg26HEyPThXTcUty+zQcY?8*_V$BCGJ@ z|5L2%(_3^*WPIGCoV1LLoZ1p}8}ww0>WX5@G88kcIKv8H9S8_(J;e$uy>h59gN zSj6GQkPOwNB1ouB8J1h(sVoHZzyw?oViL0XC8x>>k}Cal1hi2jPDMo(#VJ!k21P=H zK6M%u>Qt#!sXoPn@}^2-N^>^hM9j;edg5aFvxTclGj7MWqD?7KZd_B=;Mskb|JU8# zboT5K+ZR~y;IrfAy$n`%?zMJ|@74Va*Bs(+UAHRD$Ch8>!)PVD{Tv!?Nzw|TjO65Z zRB4VECRP~YN+U^zQ$Dgrq6D{-G*dp?l%rtmx_@vi?wrt4_|fH?$k4$Q-H&DJ#v({h zH1rZA?%E|`2cKty+PQK;o+mP_9V*bfvh`Y9c?|jY`-YtoQcWHVKSZiJIv?Z(P)qyE(YZl}yYmV&Yc%P!j>7uW08pEtQ6$)$ z*eD`g%499oQzbaSxAMRw5{$?`IZ`d*$SCqMhr$CdQi9%c13F@tWwzPqzO)lcRU$fO zO&lLwtFSr6qEE995xoynamQ_}PrnEha6&=}-PVibfMd5*10QX)Pz51ulh3;VD>Ymn zqeYa@D*^+!;7)TjcC0Xg`Ly9#DbV7fNv)Xm2w8_{71b7NyO^|&|5rR|E*)`NqqAZH z=_t)y(0Di49jEl^$}RE0RnnSqf-Xy)W!^bUKj_#1T4BW$6Cqe_+zSd>INk7^HswqU zM{g_D$3cUyKG@Mbx84=4c#^BBj7!UkciVq0tgzmI`@Pg^%&P7yLO7{zyKbl_mi9?$ z4PaWbic2~Kp@~m%c*T!lWRvW;=E_cEBx17~Yuh_VTeP~3)9ifjCKqdR0Yo>P1`mbv8NORe6Yf0m<=WnPC2T!H zAJg78dVlmPmq1zw&RX|7#W;%ZG)~!R;lCxifDjFk&KryI|1n;5w!%A#afulfKQ#*a z1_H$8wF#Bup_SbQ;N*rkIzM!&OC377CCD8=+>B@!OdacBp)_pt zEqGbb7Vh4pwAw}QY}!L%rwp^G_4>mw7yR>K=&JaIRoV*=uc;Eb8HFFl`P-2B!jg~&-RSoHlV=GAq)|5#G|u<_CN0-D-a5tq7~WHyd+YJA4IGRx5_v**|F^x z<4WNX1vW_>GAf4=VoBH@DKF@)%WB3W+hNqPpG~%n|3T)WQ0ve&uy1A0m4LJbir4_A zf-vl8P%J|EqKF<7kqQb{+|77gv_)VH0Dfe0qDDRjO!lqsMk)D`pu*Qf%V~m%!)qCj zrYRgaV#0uV0FKR)B28{S(1A0n5;}QCtnoO>ofgDk8e9WR65hjuLu5r2LMV{?l@L;} z1n4VE*+~|vjfTvK;k(j@w@W@RccHXlEn`wA!=Uk8io%P!zTqt+cC?AH5F^H z376zsNGVP#0W4NzV-Mg^FIsx0RvaJzvytBw={KvI{IfYTabwd&s>YFpOh`wiWen{= zKz6!Q3sEZ49rHL<>FiNNzO0(Gn0Y35oZ@vW|D+ls`$4l_W; zOD0)T(F<)Yt6pQC3I~T*Jz1}M6yn|WTG>Rxp0$wknP@DrnvAWE?<2Q-LMK)_(;`qd zrlHt?iP+*PB8;kWP`tqXBn1<4T2NEN6P5qi7(nH4;SHigLC~bbq&uwfoeLyVb_y6l zbxx4Bbx00$_GnwDrq5U+9Me`Q$3aoW0f=pNsB!yQR>P*%x>~bn^2WA~go@2x#;Y#4 zB$}|oPOp+0&2HT0*2wUNmsH@>TM`Mo(k4(~o*Jw|V@pHX&c5_Le3Ow)D>Z=1W(+sW ztWoz0>)Ko?ICV;VBT9Ot%{EHcbXqxX|Nf{WRjN*vo8&33_O=;F)4+8p)k>RK-8Nmp zwAaPgwa}su*UFij+ z9FdR2WU?>q?`@16i;+m6d>tY%`y3}mD25Z7(bQZ#Q?`ZwWmB4FjtNsA5gia`IG)V` z&o`|^rRe-tJR=UV$ShNaNI}^MHr;4i7HilMZW((4g*1}&%20^@!iDAqA5CPqyUJjV ze;9nRSqr+ax;8H!9Hn%RJ(`+Lxy8)=pl^Nen^ICW;TSCPx(UaDly4f8s#As^Y<<2xNdTPq(>22p~T{jLN zdOYaV4A2*wm!-JQ>Qo0b<)RK5u&ceJ|ln^K(wQn(ugR?aj^?nfeBoJ z6rjKfgemHhv`A|{P}-ZSL%8U|x&hj>rBen2{Jpzkq3n9Gf{M4bx}AX&DuF9I4?(zu zdp^IZJ|ZwQWO}3~|M;)KUAeO#f&g(9@6)sgP{Nv;qa&0B^ed5DSSdmKkt#$$3(_>x zqJk`Zxi@mW{Sy-l%Y--J!rEGiG!d5TAg(&{MLAr+g%Swd2r|^FK?CSRKcqlGRK5-* zM}ASOQSv25|HP9@G%MVjx>dWp8*)KQy0j!augY;AX;_6L3z!+}Mt`_Lm@~z~N<~&F zHf(f)uJV@naVd?f4H;>8+l?-O!jNFB=`)r-LKrKYxhc#Qkn)~pBt!gJMV`Ak znt83NIx~gwy{qdWqzIH7BuLi47g@kVbKIntkU&7BKtL2nex#^Tv`4U;4BgY%;P8js!eOPAEuT61fM{ zK4=Juj+iy}(SU5R56gQ<_#iOy%SA78rXPGmYP^Zk@?E$}v=CfA z6YQ>!WGG0fn@Yo|8Z07Df*#`2s8bWlfBQ#(3`l4@yW<3iwIl+IC?X)_hKzW{TN)wr z6CGQ;HHyRyUED=vkRnR7MzidqHOiat5lrMNH4{O0Am=d1R}+!l-)0w0qpd zbh$fq0Ly|K#f1}52ZN@I1eG|z^;YEV>qjPc`A5Dn17+RV*7#UI>36R`+f$VO5? z5LQ?S?d-S&%F9}*#oBpFkjbf&s-h|r$o5>yt|)_P1C~GP8}lSd$7Ij)JklaHDJeU> z3(z*SOj8eRNo`50(FHM)KWXu!)2Q*@N!B! z?NZ{rx7nM}5p1CY4Nl~7v|s(uT}@5n5>Mn(&Yd$++<2@m@{EzK3EOXO^u6iU@S4Ax5nfhx%HJjjK50wh$Lip;EroJC%^PIhGz=j_Wz zg`# zSzOghar4B2qMFCO-1hohhs_j-jWP<@gm+BKagEz2@m5&8&W#IwqNPj#LR=nyaVF>nSg~%P%T@o=*hITt;M_7{k#+({H6MVz>1A5{&bGc zGQIRTJ_&7Eiu$y0!NwQz48};uZ`js{{RYdViSYf+mxbB*b;Q7}LES_zG3C8!l|8^M zT%HxyrpcY^tJf462qWmP%+U7qT|oQe(QcZ&k+Ro94%MM) zVYQVj9#vj5^^AzCVG`*nU5)+8a6ICzESQKAS!!BhByP>}71!76po9{w zJuPKKB&u3#jIaA5C^W7QL#1ZfA) zs^c;1PF6Hs_e;_})|*`T;0bdXGfs=2X-wl?#`$dI6_$g9{~}G&b77sZI`pg#H((8H(iu%iJuNC)^~3ThVu4H6G^$YGi7xb=l>H6pyHKEc5#ZlEj@)}<4K2HEjEu_d zSs>or7re*L?c&hI(rb;>cy7C>L1D>4CTe^JLN#VAYXu38;0zel4YCajzF->;zdz1o7Wx5LJ@Xct~Q_4AnO_god%#pgl)6C~V(~#|AH1ZcO-Y!~xTeZREW7X&{#_M5= z7(gWmSyWzye1(uW=D1~r@j#I498_;`=Iv}kTvDGX|3Z-?fr(+nLMHQIt<~w7M2?W; zg82}SZ+;d;73YGL5;uf1HlTnB{D29(!l8aVfMidReriFTXOqBNFu`ZjoMKsiQ&P4) zG(90>otW2Fj6KDWg)Y`17ByszSAm8b>!m)hOr&% zhHYY~Oa*JI0^pt>HcTDc4gJ|BC zS$DH&7>`wEt=z7uZhpS*w&rn;j%|~Yb25GzTR~F28FJOt7$aZruyTcDR`r;A=_Yq_ zsg5EjMb!p_VOtLwq0MZZCPoO8iT4~X_yn8ygvq}i;caq~Q4!S(Fg*;wfI4$+@GMfw zJXh<5QvAA2`_fyVoIpMQY6BJY);sY}|BgwkD|UOX@gKj;JayJ?tYRAX?a+&8d7*}T zxbaZ-@EzauD*78XDDbpf;%u20jzM)VREF1;T%4@nzNGK!jpPw^Gm^V?XuDJ(nbghJ zIZA!)PPbQ#)xt96_`-}~QjG9W#qCc=<1kAOHB6>4`5Yg1x+G&y~9O z_}AUhAAv@*)n1P0``Z_j{~$uBPIDh#0dGIktaDA(WMwG?0TS5!&IkJJE(L9K zW#vrqmV4QDzmyKwUwvo#uCC%BUavyG)g0&95cld3NsQ){^@{Ahf3KzYCJeJD*QQM* zT5!YFx1WWt9Cf$-VrTaht=YI{oP``h;G zI~&UsQ9AIDvclZFA^}L~kf8unxEcpX!_6JsS5-rR8X*PniY8lH)MNvY2Lk16a6tFMGu`ofVDhr(RYBHNO2@% zP0&!16`EZaoVk2vM-G^eyBNu#`EO;;niuaB2|*$#2ogg<7|qj@NKT!iHf3DtjiH02 zHxt>Snsm&}s%YF~<;mraSEOvp>;ZH(ZI)lS+I4KA2P+y#k}4sXgrGs*27LSa{R=p7 zUkHQ^y6u$76M>zI|59$FpfR2|oJxBE`*V#DIGh8Pt?&0sEt0|)^WFQs$0V% zw(+AqxBKo%sYPY-fn)U$vWrN$Vq}c5DWo`_V&k&oRoEGhYS^K?+7UVWHuZJ5_SlST zhJ5XM7V`1qH*-DH@?|r~Wc%o}`qfPL`1Ich5mA&VQKKM2s6BHNSZ9nPRw)ZzfmCJo ziFKY&^|fc#RuJ~oV1g40g4P#-@UfwWY(@86h$1>A93uo25D8se?c1;2#iYOWo9(WP}ltC`U5y%Q590rrwAlwYMoJ*oXS%?5PK*+yo zSZ0=F1x3;lfstRjrvJ)2@C-K_f)UNulr9KO=UY7qC8UpYZf*xqI>s@h0d)s>mI9n< zLYGUIhRunOL?q%<8)bMBSXrKrMtX^uUtWORXUA!pf~BquSVwy5tuU2?MVxn4cBwkr zlYj!&A%cGS@fX5O7rh!;iFcG1VyUcB_#mzY(l=Ix!Qw%O9ucOctQT9h1)W>c9yp?i zha!}~0(Ln9NsoE4$f963jw^1t;=0Hex`0(8Kmo~&xe8@by=RLA3vNkBOhd#%6B3(( z_RMLeadDb3hQUh(yb2q5=8}{yYMYvd8Dr;fPT|z%Q>@~-=W!1+v0Mrb`#B9Y6hn+C z%GT{NUn!FcEB~M&3co^L&E`31))5nvF|37Mse0;yR&f=yTnpm2AAd^!QAZToz1N#o zUd#rkBE`06wW&5un5-CnFzd6{SjF*VhaPUYEEFFa+D^69R%CTV z)Qt!9R+|~WY0#P}tNbx-FH7n`3Uc>(i(49K@Y$Vsn!I|Vue&Z>>XJSUHOzPtQ~JIo zVftD0RdzaM(_ZVW+t1qnDlE`GwG3)jAt+70QDcExxx+3is&%yItoq=wzse-Etz0K> zbk}5;_5VcK^?$(@wq0vlZL=2KlB5E>i6}{TOW@re*uV`qaBmdUi?{lQLA_`&T!Hh; zT`++FDqJ9Rv!K8T8&{~8ISmofkjB$E1V2$lZv_{~+yn~tE?VRQVO|nlkr;Nm^gSv! zz9X6Ig2N95^@D&15}*LDfT`mcLW#UkQt*T)JC#A~cvGa}P}4^oUC6B}~sVq492!Bz|DWi)x76FfrB(3aAH-~^7Tr_AL)2);)5S-h2vuUg_=rFLB#l4F4C3sn15T*)3VCu|VL{PX$p5|3LW?xy zg~I01pGDzW<)juMH@Sf5OptDEbI1f!$x2nG@{}`erB5Nqz`MnC0fmXBk18P~Zgs10 zu3=TQy114~zAB|x$xzv(v`m7QCIf9C;VKUHxgJvUpFp(e?CLR{U9_{It5Zq@9{<9< zF^mVQO+e@Rh6%ju5N~*TT`PgI`BCL@^a$5<%^AxxrJNDcFgTkds@OV6#{!F(RNY=> zkyaF?)rwV>&1e`ermx#f3!)_Tnp7>Dze8e;SyC9!%O zOrYNC)>EDC6sNstCB?K(A2FsLuQdTPx$m%Sd831ua zvq;k%>o`b9%po2#cL&iXZg%1;V~CS=3^jsXD|ZYEyk(0Sc_$D_*ZBV-!k1$Jk`2yccyLuezh6k5mnVi=^#3jt>F*q?`)wW4>`7){*w%m2W-wvKK* zDI)18NIg1EYwPWwFQA3u+-mYU$DOH&Jc+=al2W)r?kRJRjAZ6Yr6IUg*^xYp3dSK6 zZBfchhAt9b@5*8V4ncxA-pU&CX285;-KQ5#d{^3nQv?TKvzxgC5;6lM*4A!~uz3qf0#dtGSyuig?P5JjXpIR~>Gj~KKKsQck+&^pj(1Ba1`79+A-d@(hYM_+CVAMz zcIA&Bh--4{_8|+&MwK@Q?aScnv{~rxO9tu+Tx_7L(A;5I8BON%mj4N`*0pHy(6p#H z%jr&5XGh|>S9}+k&-LWUlXFtm6VUyT_iS zDG`;+)0_gk=6WkOQ18}rp9a00$i{81la2Iq04Ul_&x~)LsY6~?Hy6|XuD8E5jl+do z+-jY|5H(V7g_4^Ilxr)mZYAF$HizJ2Pxn9Ct?zyVdhQ-+O2sL)M=D}eDJ1zC6c#RZ zafGGCY|LIrUpzI)hID%)hqBNJ?*)u!ynOy~=%EZ`wo5_%d&O>Y$Bi=Z()L zp$^@CrFTFOm&Ta@KG6?{bfiyCX$z8fVPrj$H+gw9V;q%$i^3Er)^ez)VpFCsDwkcK z*J77fair&RgwQ3dS9}IEbjF5j9Y}&DXo47EWI?5ZKevKG6@ovtY)RL8n?!uKHiHcT z0T_^jO?C$7FdE9|0Gq=rO-DIb_d`ZlebrTG4pU$*rT;4eH%tl!U>fLg?!`sfw`T%I zPx~f!?pJ6_NO}rpe`;1QeU}IOR}zYc28?A=^6^m>;%Jh_R%T&<2dFfi_C@LSS#eTG zR5M=?RceG}fhMPFEBAPvH%DDFA0u@NdboicSc2SlgET0KF*k{r0iNtq_IVd-$I5(t7iAu5rPT~zjSTZ~mW=R+e*Vj%Zf_?YaY}1iMP_c!vvkr82 zEA1C(_BS$|LMkZpQe7x-apzw2$98(>c3yaJ;h;xoND?FmafY{00EluN_Kn?BhjwTb zoyKt;7mXSP9Se48ju?)h$9UfOj&nqYjd*wK1^*@3fLqzJj1w?aA9!c#5Rh0ji4G)l zEvRHHsF23SkJ?s=plEBJ=tKZ1d;d68I*5@uxPvxWKub0S*-{uCIf{G6LW8i4)E6EK zL5s{&cOUbGY&QiX@`N}sjneo&2H`#06*D#YC&+;T787^OIE_3xe^=;PH6}TP)_1Li zjb=z6Y?z3lRdRwzfl_%aWiXE9coeIGfqUp#Ug3w`w+L4wNrO0niFcNj6+eL(7XZgX z@ZpGR^Ku?|i4SRscPWZCcXOM_iPa)xH< z5O#;qaMze^Vs)F{=zmishouRefu)romVg`;o1eJ_4rpm`_?yi+AelD?j%Swek${z^ zGhx9|`8WW=mogOTk9SFsOf)ADscpbV1u=z?M;25t$bz8sPIf7htH_<3L@gHSk45!L z80nGjb8FecpN|<8beRBF5K|wCDDkF#T1QRz_ZpW;SG$3kvdM+5r zd1hDSXFZ7=|K%Hs5}FvAc~J&X_+>ahbV}ZHgPsO zoEq1ojq`68I5nFWDS0?FIx0W(qyLsYQ=Lm{DmBJ(?uZ1d7npSko>_{e6=0AD8FYL( zY<|gPVk)2W3802qk+_yO`iTLQSczmBpl5oe@adh6S)$^`psV1Zxj3PiNj$p6I#fWD z{Z^0I5@2_7cX)SE{g#?SpoR0)npNr~Wrv}30*zNBmUD)o3p zMCp>%xeAgO0{p_1xjA9;!w68hct+r=Ab^K!X+3c`a#0xt9s8nS220)FQ|#?qULV1OUXsvHWh+Gw&T zE4d7Nl^>y;H%qcI7XO@&MQBv2hO;_`3j44;`G~~|n86sf>?)A!TC|R-wlyWKFW6i^ z7lOKFbAj2qSgWO9%K+$VbaVQ(AacCHJ1E(ru3ampV>`QH8M|N$sKfWIy|Q-Hi=b)? zT24bxWwk0N192;AZj$4$ezJXZH?O%!b_k22ep@u(lW?m=TdXz!1~{<~AhBkVZGk5X znRaE=$+Q_yPr&!v`O|vy_>aw`Lw_*wVY(8)w+tQ$h9X-u3n46jro^ZJA86l zy%Pa`sT-``2mcQTWt8=nqHATThLE88rmSd7A}y2zt|_cXiJYn`x>aQal&8P6@&<@o zLW(;8wUV+bVR~%GdVI%yS5Sf}ae^p$sjcxTWJ-!@)bY_9>BX zI>Z91NwWJ=&^x`bE4{Bv!( z+jhOU8yzsU3~)irJD?>h%ug4<*IA4My}A5jNx>ilcvOc6`~_xgBtby}2@DlA3dl!5 z&i1U;fSkwv@yGEDuZg1+U5`KoF>)?}?eri)@&rljX1$EE9zZYZ5`4SLO78DE@U zELFmrth|4HgGXz-3c0(vYtfms(G3B^NQb6={m{xLdl_)GyPKEm$kNf9ys#_FvOLQ$ zegDfbo!K7UyxNo-~MtyqrOKW~-l5H^04-5yP znx{zP)I_Qi7M8|foz}^%-0ys06aCLGJ$)NSKwT_1&9)`I-peB8%mjMX+C0AVv|BeGp_NdKyj zysAOk!I1?9LevWa%*E6k4Cjs6D}Ci3C{z+1tzCZDD(I#8?X;%+kQ2<{V zah2q4DVNjsOypR3f7dyp>I$*K;#k(zdYl9yz&cl*)&Zc#rhen^{OR@_hs+)1V?EtJ zu36Kq+>;7Pum-a&OTpLuYOu9h;>}2D`C1n0n^Z}^if6$=lPYoviSv!%d;ZV?PN5SW z-#};W6kXW+&C&P0%J}Ws<13$@7~ezPXCFCPVe=O z=!stFi$R84G5{yyBS3OXPr8VYE|uE3%oJehCvFgy!s(q37RPPRw3_PFz3MkEv*zsT zSY7detOX_hqy7=<%YD#|*SP~7>4m7k?Fi|h2Amx1#c3I9#0u@{?(Oa#d|r<8Umk<& zPUdc!%IDtMS}u}w;}9ne=bas{@m{-0FT4iM*$&?AQ2)?=UY}AA@9yxmgfT-MId_Pi~}+C3=oT+YIE9fJzt*`VzkP&VyL}F%AiA1PLOa>EiQ=* zLr6=lF{08jv2ZMt6I1ih6I8U00YCtuA+-U4mDL2{H5L}t#Q&r~z%*AEz(Bz_05}*} z;1;)-*ckVqm$`ZQ8QMTPdYal`;CgyMn48<+0VF&iB)mL?JVg8Z;X#Rl9bTfGLg*ZP zV(r<&Z6g`B1j$gONSF<2y0nK<+e(=_Y4)u75~wp@DOrNdxzZ*#M2xyLrQ{T9Q#W^- z8nJry$W%O5xMGb`#H&{~R=Gg^^b~E>GiBjub^CTq+c#42AUQiqQ60Q@`NEBRX$#l2 zLLyx~o9EM5``yLmq#F%Mg7jt|##G=3bpK%Wi)IyzAr7s=$gYyV#fl3n~Z< ztH2|iD61*SNXK-?sh62TY|z{4DIN)iLv0?fo1M?{e^ z3~QXjG0BRs0F_pJY$cT|swj+5A*`_yRItGWBlNI| z7GJ#()KF8j@YW6Cy5dt;|xCUrZgp#?>+V!)fl}gH`X^&GgBIwWB4XDnLR~Gg7m0P zJ4E*7GpblhR#S;UwZXZRofSd^cW%)_pZ{l$aaU-6EySdlNwtDmWZwzk?6nn z=jzYO0G3n`}8Wv_k_WPQeyiR@f#IZnkeBQrvC<_~o^3j~sX1bl02G zJOx~EK}r2UTPM1D816D1X~5J1%m+k>Zp(aD;?nTp-2A6ag9v6b;^rdVm(S4oX;`|> z^%?!XkfljF0%#~@d3HweLvQv(T}L$Io@%!*cStpTiJMxZGXAYnH(0}JR#C+{)vaoF zmHJn+3ZrLUY5ou9W49i=zZd5+nnSvl0za`~$0q{=6;RdniT1Ug8vXj8WkLfDIMAO1 z4jjOC6|H#N0ONM8y={g7tsuYxbN^yDBbki>5R{-EtW`k-Bye!n!XO<8;HMB^aBvgY zP2y%#j@m$Mft`UznjCO1c2I6l+rZrA3dJvf{R@Z45gBv{W;%rUa383{NDdpM)#{<{`S{M{`D^cr?^%D z2ROI3(anL9oS@l=lu2Dg5R??$WGF?+$qXjIf<79-;6_jaSDxUNB}l-Ss#8LUVJ-wD zXr(JDcoGMsW^(3m*QR7>L;p1q5t$KT9T9%OB@l%tg}U8+;xjQo zDRG-gxnl2r7sYUDktf}F8DP9Pm_GbPj0?)yV7O8}d(JafY9wRx*4IuzB1UP4AxtD9 zGZ^yK@t|~UXIt=?P(?m8q4|4c|0r6~ibnF11k@t=hl)_~KcK$5dKL5a? znvD^R($nYk)R*s_Hvk&T_~ zgd9oHiCWa6ktEG6NLyOdnl_W8bm`zW_@f1+Fs89p?IwXbRNK~e1Pd671ICfTRRUv| zxUIkfAb{I>=(3fe3GONb;sPc#l}pf_(@2;a)n^tpnlmit%6cIvY_809OQd2qe@EWd zrH7hN1aDglmCfP3w!F~Q{diKmj4WRv5Rx-u3+%k#SBnFRHe(Oryj&~K(sD2iyS8|owqyIj2#x&>pO3z zRjgqh@0PcmUMq{t%dV3zsS^CFxb8)H^2~x??@B%Yt{GNxu&egM(1c9%lfLJxux0sp zEA{0WT9CEGck~?L0|$c7mn|&Wip5C%ZMegcwrB-BS!rFIlEo-4)gXtE_Zp? zUvA>EDK%yV%Qeu)8v^v8$Fu)l+s%I=C}9Rn<^ltH30Gt6gr!|>U^(qZhrY178*FIw zRUpHSl>anEJv?uVdLRRq*0-iTOXtkX~6pp%23}n#;LY#!f)zI!nK+k19-J< zW1Zw%Vi1HxrZq&8BPWd?1lP7+t9sug<*iPvt5wc2TA_GlEu#$PE0;NUkNuvv@{`;O zt$4*fZz6(0?_=(%0>je}jA7nx(8`8DS&DE*fl!PBdaAYHzRG zJ1eZ%MQcx4cGm9V)0AvHkKAamUow#u4@I#0JS=Po`;0O+K z(vE0HCb}Euep(z^MJx^+G+-R z_WxMlW;g1U&f)v^VE3Fp@8RkaU7YXjqn_?Vv%B|WgY?&lPj7r9p_F4!W!X`A3(mBi z?fi~DDBmu9PtTp<*>5<)>8^cMAJfSmpWNQ>z7M_codEin`zq6aOF5;G;~nQy$d!Xk z@e+F^Q#m>4%{y}}w>;)vuI5yZtppJBMz8dE&%lnwdR9;9TJQA|q304z!k*1o#HO;) z457*_kJ`=MwnhO7jnR@0>paELiVp?vE$r6r{8$TwU<(FKVS!L-`ryvfbcDrPVF7f| zwc_rS*3a&?ukM0S)V}6(JT3%)PzjgNl@9Jl+E4N54`K#JbN*$T(90;G;3U4xz5lT2 zV*-$g#H;^~t*f?c3_Fig_T$;iEd<+;DZ0Yi9Pk0x44$Cw3v!PQJ+K0Q1!&L>>D;i) zkOvR*a1EhO4}(t$hL8B_4F!+S2G_0zA5juf3WB1On#cZfQ-VaB_^w{s_>k^pA-&&*cEeW5DqLaB%=>ZWn#W*#HsvXr!(d42}BB zRKTLI1d#*7tqt`E4JdFaMuh1!@B@3U5r2Xl}H zd(aZ6?;O=J5>KohcTgVPaURoA6icxQ@e#)u014mk9`*4bU(APUEMi(I760`Cnmk7p zpDc&8Bfcun00`mNl&A}n%>YR*^o;EyIgh=FXPkJa^5V;7;OH7j&sLr>+W^b;3IrS5 z&3vAsKk)1%GjOh;jlf8fzGiP5O;Q<+(M9;GBo{Ck(dNUhNZ+gYl4TEDZn(*<>7ejoH*g7RM^s zcu^P0Fy?I0{{XN6dofd7W)Fk1zs>-H%z(cV5G8#Q__7Y?W)d)Gl0*6|F||-hD`%57^U*eI zQ^t6Vm4a&(u`3{2(FJmg71k0vLNX{Ma{|_bWBla2G;#pN@HsgW^gi!os4Usk=`Rmc zGlB8k{Ob@a(=fGhzald`OY#93OrR|DC%3Jhw#|@$GADP^0&g2?kM0 z9boG_q~uQDa)(qDMUU-7smROtay$?eh8# zm67N+^;1QaJ3%l~>($RPtSG1SR8w$yhO0}pluX5RKmTV{U;~y`zZ6UlmS7t;V9(T6 zc~xQ^G-81aA>oulEz}8NYq|CTVgYYAdFe#T?xZH7B105+I5NBpb@R}Wtjf#E0=4rd zf?J97+maL!)73lO5M4o)RO|I-aW*~Wl~d<+1I4qz3Q;>1&{X>sRlC4oS5-@IFjl43 zY9SE^r4nmpm0?+-V!e_<*A#5Sc1@+~hk|ujhwy8I%zTRLwmL&)rj_MxQCms07gx4! zebFx^)k1{jQ*X3f$2D*VcW4icbr^{jaXIrcIdD{ccCf&}R6)~eIc#4k z>R$&IVI3B9qqb{9SGGLYbVb)4S(8;0_HuwIhqo@~)7YA`b!i_FmhSa6iyyf%YYEL>e_VjS_ zQ=gYRPcU;C5kEVZbgx!@M>jxMcM=y?4q)})R9E0Sjeh4Be`PCo#ddZnwtwrcb`v6P zad!yafVkRpfNkrPU{E#kwl4crFN^nqC#7a>59rh_dRtU*eRL-eS4fN2Fm0sVaMoL= zmpsW$JwY{bt>$u#vV6a8bMq5mTlaHoIDOfdhD+CmSyvt5H!AZNh=Vwks+K^>bbkr> ze~~!V0vI8b_%@$-m$We4l30Nih*?oC%KxaW?huiZn4TO9atNxB`GJzfS$9taoT>PAHR${f0B(ijc<0$V zmzTXNxLfII1Kj6d=_`$SnLd4QJO7FCTU*2?iMdlTxqIi;aSIpfg4q)ESeY{yeYv@i zQTL(;nM+M>t0o5i}U&zdxc1ftctir;w_R}Pfv zc3Tabjf+tQTKN)&nz@5Is5KQ@?yx293*gKRpj zOf<_D0-x81Km>`5(RXZw8ldk`K#=sOJ@~qd8G1QZry095kCfB9ymFP)!@rzK8G51VwbZQ_&C|SJQ=HCeymf0l0eslBc^KCH zSjK0()>|B!alFv~yI6y~t{3vpgX=^xxz~ps+4GU6QA&0ioK%v#1**ISIyb|!+}eTK z4ul!pL=en1oj!ZEvg3%6$~<3Hy+~@6)APs8WUDr|m-dEn$v6qK7bJ4_MPo$P5n^4EUm-M#V!H0@>n@+V&h_1>jrEm_@k?)$y%?%wo0 z{~q}s82@ZU>06)T-3|6F7`vmMvo}nlx=zja=k^;vbJ1NKHz3^B9qh?|_|4w<&tBh; zALiBiemOtiiy!(?U;2;LSf}6muOIufzmioy03OPyjsEqgZt%&RdZGHD!M!OMpZ9sc zHQrOI?MU(SSSvnI})I%)2d8I7m8o8Q0@?FMulgQG!nGWDs;UYDp)rBa=0^=j3u zShdQ-m+v3geqzIt9VkiSo_%Nq`U(4RBiy)aAyr}p>Z7Yr2n%(30<5eQ;&c@>j(EZefN8ULP@ zWqVddv$q}ce*5R5+~B~0tARLJkz|Pgba? zl#NP=D1?V0Hlk=NhF2-2nWD&aZ36s7ooi2h03sw8jE4E)OA>Dr=~hv&X(XL#q62XHj8D8Wlr1Wwa+%w zD!Ac_JI|}OnsujJ0rEB^9&q&bYmsO$8E>%l8Y10;iAH)XzoqD_p(xdPh$@T(d&}Ll z+aB!Lrv?}7OtwZ@J8`#{je9Z189Pz#tsS!q5xWr>d1tP>Mueov@FpVZA)ZXwFU(LT z%P1)SNnG*5-G)iAmuQ}ev(6Ep=QGbhuc@(|NGH9t(nfH+pMG^VB>#1+RHyuItoA0< z5zWyV`Y*nR#w;b+0LQZIzzwUd;@WLHoHN{)zCE|wn3gx{&~bxGx4}*O{r7xN;~FyI z>jrxy$?_7Z@?2?Ly;0Z3Mn3jn`hIO82LczZx6z$%?m6g<-Q6P5rGL(OOMo-Iy6UXE znvafnMs2m$Bj+aD&&#RD+x%0X^lfx6gK6Jz8TZ-nVLytZE)tX-a{I--=RCfyBFFydY z!#&px5NU#g8Un>qyV|8|fmst7`qolBQmBu8WP=pgkVlKON&k<7b?YDqqnE$NgivlD z97gs0hrj8a5JUxhAqU91d)a@Mf~sx#Yx8Pf0R5T z6Z7W8B<>7)edOaOLCJ(a`bdjeG*=N-B9v8RQ;x(Z$UqJmfk$YEt7>y zoiJ05$~-40B^kF&CUcWNQzr|ZDN1=(0Sp6hP2x=H7K6lvcgxA97b_4Fvwf3&LNuP` z&NeAxmQ#%kZ74+HDN%!^@Q&y-=R;FP&u6A4nuMFC)voy~MrBhXz#HiC3c5(Z%b;MUJI%O@$+=dP{{~m8^U9 zEc3J>F{6AIqowWaWGWg~znT`bxiIWmz2H%|9)xk{Q=DyqL)Xjl)=83mpG$Me+TlJi z9MbA5a*deDChpXogqr@}<2-lvryr^v(9Y?1y z^OCo{dIjxQ!Ky;`y0@W1c&&6}i-c&74`kYe3wHh6&sGBWyHE}9WNVhj^U}A$&iZO{ zBmBnY3YJ^VCGLBBao_yvm%k2ds}2pJr|cr4j0rU`84=9gNHVy-E_U$S%rrU}XZXc6 zUS4|ZSYZj95TZl)u?sjXMGxyR1BvW!$^Yn8ktekj#baz~fm6Za7V{Xx*{Nfdv#i{} zGS|Y+{ql1=CmJCakjM`q7Kyp|r|m+@QQb7Kirb7!8q?Xv6h&rI?0jQA*I85X1*?{O zOl7j(H^}6X&8kqelb!0yT8eWN3m$ZziD_k*V0qJlh9^7Q`RQ}+ve0Dy zTaKJ#-3O{!Im)4Bq$&NVI6pFT9h%HFGOcSpcY4>p=5q`G3}{d1(au0VwZ>AdY9y1- z)sT)g<(h04L!n~I#QrrVNq6lqZyPdP<`lO_y=T%O8?m5N_HiDoyO&w^&Cos;tu>3e zmu4o{z{d8yiP>v>^E=;}(eaK+-T!Y?7242;ZW_AN9mbQsySJ9Dt14JLaDR8ZCEBhy z+jiZlP>b5j;l6gM@mp?a!fr}m1Y*N!8Sh%#x&joRILvo(9@4&i+?cNUX@wndaf`Wx zKo{pmMvJKLN7S)K13>o)iKF@A3FgS%jUtk8f4kc! zZg)XXo$)uV`f9HZX34v8!&^U%-@i_6z^kZG-9`$1T+|+!h zk>5Elc4FG@Q)&FNb@w|ry2)6%mPdX^OD_AA8*lTG!%bqwuYKbW{P%WmJpF@G zM`U(+_!lldFdZNIl}6^3g!9GB0ai}*E#O_rpZGms_CcToHjH|KU*y@}&%jvq?cD&@ zL+q{JH+dJ>sgJdZ9tKVz{`Fo8)=%*LpZ|r~1qvJoT^<+Yh!6pu**M>cE#OoY-a;uu z`JrG04xtb_Ob`;G30k1)0U*U7q3d^0~%W{f(YG>q6mFrFLIv{Fyh9jA1gK= z^vPmEv78Uy-YyPfFCwA}%AFT-957O&rxBAe?iVtGU?T~h=YgGqL8C58<2j~ddHJC_ zMx(wN;vr6BJ8s~yb)zGKW8k@;ILcbJ5du1{oi=jfJYM6skzov)VnEv16As`#f*_lX zqq|MtK>p)IR{tc&=L-OOIkRuxLp+>HxH3nls3Zgp- zqx{8}D8?hz38dpKPe{hrNHWs1L8KYMk3X^_QL-dO9%W3nO-9lqQyOK>!K3iKq)Zaw zLRL{v&fcUw(j_JWQhKCUR%23jr5ZtA|Ka2rN>~s+(^X0u*$v*T@gM`vn<89fQ97kq z-et%!#5^408U|sph>phEP4RFg{5mj=3~0# zZnmarO8-yr*-_A#CEO`ZY@(FART?vTCT#``+KFaULg#Ou=5$JDbXq5M`W{CPTXlY9 z5LP9)948%SC0uTvlr3g`US4N~N@$8Qrhf)#j{T(vQK-}l&FgC=!(i9kH%DL8kdmDm`#-; zht`~YD4R1HAaj;yKTbf8M(K}ADV1*GYRV{X;$2#*#)k&aY`SG!LFklD==@bFmbxf8 z-Tx#hUg?n%CFa3SVsd7aN*^T6W!~v!^_?h+nko5>DWCr7I^OA?w&xx=EIkUYOXpc{D`Ty zmMf;#E4|(;v3k(CuFbJps=fZJvRc8XA{dB<6WHCFv?i*xZtJ`@EWSSMx9%%+ivMf1 zCM&>RY@jZp6r`G?Dkp}(pTk0Ir3$Oa+8A#3E6G~ymMX!({zWVWrP6U%O7bU0POHM+ z?7}i^#OiFx_H4-VEOsIVF|3x)0&RQc>_T#ETrOMa{jAd#ZJDa<(+(nw1nH1gAj)#; z=@lBY5^UJno;iNXUP^4v8m-w*E!3v1nOZH|3ZL3)U}&nM80hMfo+o;uE#AJZ)=I5c zwyhHGZOX=!0d(!7CKTu;EY6ax;wJ9dX6uVJF68R1x&X8^^vaaj{mOTwyiDl zEykX%+^(WYAZ+jQuJ7U$89J=+Dz9eX8H}>7s=96%;41Q-(Ad&$*>i^cvb0V+>ub_^R3IlKO2j#B%9wh$;t~$OiEfD1Z6ENeV zumK3{&e%kniQ%BnQ_p0=)0e?l29@Tn3kKz5=nx4z2nYs0df5 z=c=LrbMJK)FpKhV&&qHRQ}GoyFb-RB14pnB`Y;w7vHU)741?csIsdO1_n9yxaYrt( zzZR?6HgOne7!+?Wib`<{gKr-1aTrVK-=?u^@^K$`FY0Y^4zF((5Aq3taUwTzgAFke z1F|GzgCVP?8zUAy={zlQMZRvMO_M)z)Mrqwy%G zs0Xug)?V`SYBH>GCEIbRDDXI1ef^A9On}^e~;Xa5D7U?*DN@Lv;AUGw_Zv=|;0X z>n%+_^hWb@DHpOo*WwW!=x-hXJu5UsPc%hOt^I!!>1Bc3$VJ{@V3rTQ+BRHc#+1XlwSV2DWHJ zuYAF0X{WYXpEX)Dc2R59V`sHzZ#Heqb!e+LZqIf4TL1I|WAtGIc1q`VJzwaMf*WEB z_g=Aauf2BEmQ!p`Hgrq2bhj{5&vtE7@oitXc0YAoTlZX7Hz6x_Xw%$BllF1%DEU(6 zc_Vk}iuYo>HYWckTZ_STd-rzdc6PJ(eXH|+^CfRzv^|@*emC+Mj`eyUcyl*%O*<5G zch!8)F=R(KK}WWP*SCG+b$^F)erGs`bFPJ7IEWkchgY~5jyHf0c-{$kha-55owjkK zIE*K-f?INfqyacKq=d3*nM@7g(| ztLK6X`ktS6q;vD`Uil;;^_S=PsFS*>A3CZxI<*QGIzPIrzj~^_d8z04HJCV(XQ!t3 zlBL7?rI)dy_d2kTrk`heg4g&F68fowxw0p_n8*4Q<6=#3x~lWK9@}@POL|1_da&*_#HfyIcrI)*CKKHCgc)ByYx)-{$yE-!)GPQ#{cU$+m7y7=6`~d?3HP!#BIlyZf?_ILm+bzCSQOb9}vz zd~>-^NueYI=-R3B~6%X(*1chvWM!~49gpE$mTJ=Pa~*>5`% zpZyj${f{a<+|zN~uQ}b@z2R5ssViJ)L(e;U7NeJALRwG~l20*Qfoc5!2ek zz285+=Iedtqq^i{e&x%3?B71_=l}lAxBiHCec*Go#yjQbQ~v1JzVY|D?<;;^8!FWE zzU%Ke?GwNBAAj_>x$}#;NdJE4-|p}izxPwW<}Uqv%Xt2*KF>e?^`rmm$GxG?{=>ij z^wWO&>puIx{qCzj>py=laK1AG0m3CD4vj_RQJG{eolWRd8l_IHL_y8ogGx*>86oK9|?&cY7Yc*Z287TPK|Z1cBudM{%gnWd zEiM-VBhVqy&rs9R(^Aw`)mGBiHCQ}ZDO$|iNL<}r-d^8a;9udv)L~-d<6>83W?L}F z$7pEjLe1c7|@BO`JD%=H$84XHTF%g zX5Gs5j?|xEB}S#XFycogN7b%fv=Uy|pECo%rR%lsUAuYl?$z6u@32g4)fTO$RNvCW z`4BJO*C~M6vScGCoGf{)<+HOg`}KSEFC4L8AY-mP+H`5uq*Je!959Mt*lQVo?k1P> zZQQhT@7CR$_ix{D-o_@}DER;ELAIx|X0F;f@Z->>Pmive)aB6Fn{Ri1-I`nC;R{#4 zrkp%=^yk&5XTRQkdoRO_i%0qe`F+Fqk(=+n{d9lsw4-dm>uBgwUo0$H0vbIP5<(Q#m+RH)6hvoHmvVTDYG;OQLs!Mwper>Fm_#HP0jROYvJ`a-qlXqvq&kR5metm`{mbPX?w$!*${kDArrJkPW)ik#$J%!LZ7p~PJ65{GCj0KX|H&JraI3kx)_q?j`qZh< zHoWlNHg4HzQ{i@eV#E)(xJhSS)Osr$?UoyFvuo*m^2kHqigCwFHy!G6@xJ-ZvaRKN zna$_+>}P%duKdKJB^P~n-ECUjvBpma-gL-$H@=El#d~%|Z!*_ebLdt7{Q1F;N^M9&-`n#bW>(~s9 zGaDnH>c&^0>+w&4e2|O&9OyO!R4*jw+g|o8SibbV4t*V=U* z6rQjR_Z$D7_gLsS6{heVwgVvhw$>O@iBE?Z++hI^Xg!hvF@fZ=*#nLEI39Yihdw0B z^E&uMB}Ng7`Fd0pLm0R#X0bXZlwl0L*hMEa@Q7msV;R$8MKo$rjaD3>W0;sk3C0nD znF@;;nf1akI*?O)TOzk&)fzaWZ;Dd<;KkB-$TbS^a^K+M7aeIw6ymXklANUDA_z$^ zA~KPi zlbfZxWH}vrPKXXvgAb)>L@i215&lb~>U`f#+<8!t=JP!ZO=d$wn#pxWN-NPEXE_Tx zOMv|Fq-LBaNxx^(2|YBRd<>^IUsq9}TGStD%qT}|cSWFlRHQRyYHwbMM}<9&g_K-s zRrUG5l*%R&WBO-LGuTqEO4OiqoS6{EN)fYys~8F$9^=@m%s>WJm)4x?Plb9{x!(1R z8!c*I6NwY0HdU}z#Y;V-<}zUE?~;TaEE%xM&u1>u4Fn{cPQ~g|G#ON_BWtX56dV6l zx4!i;JFII(^_td+;#IYIwd`Jx`dYrimP_XdEMslkS@z5`crj~QB@>HG+%mJ7kPVC5k-h63(r~jXTVt>^ynyVka-nN7J)l;z^p&pZL`%=N$qT&wW$t$QyWfC9 zn87L`Z~GwOS1p1gz4PTTfjM)`>oOF7+~{zL#j(%(k`NpEU9f~*>|f}*PP#9RG3#38 zTnQoArfz$%i}TuJ9|!rzLe{GxwTR*J&bBs$GqH!AJdxe*7;eRT@(?w1V?O^>j-QJB z1B;0a*3*VnWCA90m9Z=u0!Z0=sTt%+COpRox0u0lCbDZQe7g)MW5Z@{GN13DKlgfX zv3?dV5wsk%NO+lpUuHCz!%SltPuIDSp0wxG0j~omv(cMQ?N_}^XHawcmUd3^o&OqO zP&CfUgl08+)r{a)PlU>qUbTIR8`&;1S=5fMD`qKuWm@k>z(%`HP1XFzT{}3=%MNw3 zlRaTums+-`#>1ZbeC=50#K1b%AGWjo3>2rhqiQ(rv(3$GF{fLOk+yV=*FA_b|cFuETz3k0Kw>VFS%yqD!p^0x2t3RIac5W72MzQqdHl@vwVJq#>#L&Mb?fvxpIq5wH+9(sP6f5&UG!`hM$UPj*t?IO^uiH30J$s# zMQ6U^NbmYC-94^e8(#O(^)=L+-gv1#ZRdjzJMlAr`QxXY_Cx>mJj{QY=P>Ge_0u10 zu(Celn_3-N2gr@j_wMkREdC_qY0yIDz?41A)WiXFxg7jNE;)dia6?jj10Xau z!#bi$oWxbUL{}8U?b|7_ zTt;SWK}f`@L-a*XJi5aGMQelyQOu!I^urZ9J@}$0dkVBV97k~!tyXMCbWBG};zREP z4vwoMrNBi;XctVGMtiiuV9ZCb;zM=pM}PcBfMmsIyhnnh#=g5oHXOxmBrtCb$W(Ml zh>S>y%s&gvLVYAgR!FDL0~`m~#lovBM4ZBq1V;Z;5<&67$cRY6U7SdkY)O}V$uP7) z>nli_Y&kVVNNiNdg}gZjBu6{+NuUf$p)ALtBubb(N~CzloY)r>|OvnVoOf*ZY+&oWQ!?nC8 zZcM_4l*^y=Orr!%(F{$}98J^gtw7tw(u_>jl*GwgLNwdLbBId(LKdn#OUg9FUOYaP zM8($p%`g0|2$VqJEKZh;8Bxehv(!k;)XD$1G?C6s#&c}V=?qS&8P4FVPP)89O3cnu z5l-;L&hYe3?$l0hazO6;wWssGH7o?dtViWiPAg=@PE&=rEW+~q&iwRDeJWvJAPz^;*{xqW1JWUY&Py#j0 z4z78Xzg?719wo^7jK=%4OA+-?B2`8g zozWvLQaU`+BvrWKd(eW=K~Bm{UW^3_wNf(aNhRIV66MknUC zJayFVgjGFkQdzCl3;k0;o5|&5PB>Lm*<_NoM9KFE=S`Acin)>d7=M%7n@{Z=zgScd(__(GN!^F+$hN+@v4 zVKv2-Y*>vYQ|iRkd)-)&eO9P2{a04mkby1OR_)f1U0GONSeAX+qkI{ryeQZ5RbjnR z__NZ3g;}0msCk{Lk7d@6?OCD~$l&8ji)GJvI9Y;i+I!2`qK#Uqom#5($_p%p*dYSN zfkdu-L9gvvu>D%G)xNPU+ps-bnM~X5LtCh=TDNt@6&Z(iYD8j+b&3^$SW_YG8g@czk7YPK{<%afP0iUs;S& zfmDeb69)qmvx*9%LaGkB5g8l8#KU81zl6oh!_CaaxM!qF2D#79*uuEm6~WxL({G}@ z;pOICm1zgTx?z&P&lz{;^z})3a;uzI?|S_G19w53S}ko2C2@(O(-)_REZCZs+UwwHtdIlWZgx1Xn^#ZfOowRADZJO|n^TXt ztkzn~mKUeHcn#XsJC1MOm3y%~Ko!{Uo(B&!P&j}%%U~*qADlH1S#sgaehpfoz?m}T z#1ar+P|UKgX}f|YOvGH-vfRCCYlBw7$^aP90aJKYLZpP!E{7<(7}ZmX4nBF{yn1q!hRZtDB_41 zwxioh^vR~jVDVW<;)nzmctLBSg_a_4D$-bEZ!0!d+l?yvMw=xl5=010E_oLde8K@{ z8*%GZ@*iRvD99jzkc<}NfHA7zrI%NwSj3JxmT4xLXQrv9ngIrg=9+Md##xP3&iSSU zZMxZ-o^kehrk-u$H@B8ct> z)Vgc$z4+$)Zn(_$%cihe&daU7?{-_Dxm6+>>81YyT%(=KS}HM`lbTs^u-g>%oUX=2 z<}byRD$4=7f(|M#xZVnkZk=})Bm~1S$1F3<5z|@mp2g0rv$$K1JMo@tZffVwL^t~J zqc<0Q<+3Dctm~{1jp|FT(Pnq*smCM-&#N17O`)x|;!3J`qL!MU*}dZF@W)4|Y3{K! z)@*c@QNhJ7wF(dGpnzPSoFKr@svTpU+OB-826xx{0?H!`?5^UFKQ4LXigzIQ&zP%y zC8FC>Uai6D3Y+ldMfX{0=7r|G=$g-sv@x$ww|%LzB#-<5Y{CW)IH1aZ8*0kXG?)IT z@NgEVB@uM;00*Llc2ieYW1`y;OI*RTUk z4`y~64@3k=z!n&gf1-3)vOu^y(8Y*>qN5iF3+A|mI&^WxbR;dCS)koTj%2qRjYi|i z%qRwQqrr>jF`Jf7L<%s393@t!)TKgaZH{yQyyF}zT{8q}#;K#y91X&t`9hp})Tj8= zT?=!1(`M}rs49)(;RZCuF>1t{H56NO5_!gFYy=|BkjvBTC{_1Kce+bejVMA*yUXC_(_{3=~ zM+nS6xpuW6T)Mgr~h>13Gj4I#_hoT3(@%nP$7=7f=Ij(+PAstl``ko+Am*1uuKB z78YVlTX^SFH&f6SbM(3Q?ChK|8r=)ZbMl->?MGv7)xpeSXv%9f=}?-*mX6x^$QKtG zW39N7ecyL+yzr@|k>L*)TcVfkZZ<(n60Mq}h9I{s3vGbEbH-pPdFew2@y<;G>Xt&Q z{Oc^cypsB@QU+{Y$ZU;ntSKiDNb$|E%<}o&!ED-qo!e%6zxlvK=JrbocO_(&+p!O0 zrMkB`bb?g8TR+FLhGH+g4b1oDP_r6Wavhip_5L}q4yBxqFC zgK87Vd>n{x%7S5kA|u;oU;=V**q433wSBcRcQwXM?i5_0hDdtncNbs>reW zTbv;+@+Tsm15xd_Ly^~OKGbo!WN7m8IV&eQW4L){C^@Ougw@n&a&~NL=wA-gb<5XE ztpj`ckxnRRc5~K9C&oe?SWthaD81K$F4$Pq_B;Zjf&`*Am_mq!*jOu7b${Z3E*OU| zq)Jo-byf#0{^o-|XlZ>XgiYmQ7T9+U_Z9*-e&{!FPT_u!m_MVj4U2^!Qy2_YXodA+ zKo;ade%3GlTUdD(HZr=SN{~l#Pu4jDn2ZA0K|KV47S~7-Xie7hfGt>oYlcXBrb8dK zLMK>W9~6RP;)*?mX7Hy}vO`S|07T8hZH{PE$isru)PXTrMOQa{8dz9}cw*(ak9`Ph zp7@V-*FL1yePeWOM5k~`sEU}jT<-=*($x`lVhFK0CrM^A4!jdCc7N>oF3*fFmdjoa8SNtZ2s<8W*i7#Zrg zB@ly>_HKQ{5n}11r+L&Knr=C8_l4u8`Bg9if`8Q}Ks1e3zN}7*3 zDyUGHpEw$nE$J_$$!ZQ_cUY;V0B1hH#baJ-o8l*y+_#%B_MnV0p*rH7X^Nr$u5yc! z)2ENLJDL@jTY@{eCT_?$r}#A>D$1A63QC?zlT_E80NS3X)G9&Cr-xo9rOV`P4&<5#`meEhcbJxG zUrJR78KDW;WB=)!N%gMT7Fn{Cq4M_vv{ot5-)TOHEw53>kH4RE|qiBbs!io?|rZA>bl1KsnHQFX=YEp<` zP9#DL33DN{Sd1hZNw+Oy4+FihHYCL~?W(lc<%vYR=4EM;czh_S^AgWd@|PU@cC8ISQuVnCZH-_MT6amXp>o$!TUOZO19TKpH8YHpk=sJ*QBG@qpK8pH%OHB)UJ-$KnfIv z&SWkvB_Us8M>Nib~QNq;Q$JW`M zH+!Uzg0cs~Xy$38TnWeE8>upgQpz;CJbVQi^{JfOxj(GB_cl238N-%*Jf7vr#U~oZ zG`XBCQpOv%1C^8W2D@N9gtLpF;1mhOWt(8TkhNT^v0%mjdy=-i{J`Ht!8S{(b%IfE z*>NQ|hA5oOe~X3}n`k}BOqDD{h)csoc$9eBjA`qROD;U^V_3gSFUFUsY~OX$|N|1q?zJEd`z@k`7FEt3d`bgyZmetTPe`D%!&<* zpRtI+QtieO`y2%wprq@E-*T}CqRd{s%=EXy-;8emE4@53jL+bV;!0I*-8qa$V4{* zzHMSNC4%vbxpFPi-t3NbTdYX?#GVY;#T{h-6X>2RTh@$iLd>l{LX*>b-ENi#b`g|S z8}i|E^u<;i#=SMLbf;+u`=Ij#z^XRhy}ZyEoZ9;vf#)(`&8OO{ok`QeK6Ru>tqp9= zT;J@e%fdI?I^xl`T>#S1J#=i}MC7$RtuRP+Z+N!hpst&xsxUAx$;^ndYyDx6u{C(91&TQVs0ejT~ z8PaK?c!z~9-+)CyJu8V4PCPOz+B90DMjPdy_2ly@G4Ga;PO^=&or+LUS=^UcqJfB1AoQ^kH)ro=QcebQ19YXkL=sv^j2@$ z-USoi-T9lJ@Q1YVF6ZcTUxK40sqnVYm0tQfWAYzQVlPjPtNUz;L6i(0@p3Qr@#}o4 zmR<1*F$x}6snqcc!o6E3L8m7=`J1_66?$57eb?}B=LF)2f4~8Wn_emmy318mS*DaJ zcYUtV$v*xSSe}^=ivSJ!S-<}6v;jYkLcmI{M0NFB4q#)yJm42E|N9PkNF)}^ji#5vilF?_aI&JNGvEyo%oxCvVLyBDTe%#~0 z!aT5soU}NamXk zM^{x(9Ta$bdp9FUF@1>-ijR$xk?AOQdyO-rfv2UXb+4_nu_Ndl^(xPp;hvlg4$RTd z)zjJ6-P_%fD>`0U>hJCI@%Q!n`T0LI5eKeKL4yRz+{sgB;FpC5k(`pKt=cJw7cmM6 zu#sa%kKfFAOS7<{9Y7+hB=VL>5u|T}!jv2t5@A7w53d~h2r^Kgn@42M(P_wK$vZ4z zz9|YvWk{XfxIlXUa;V{$pi3sQTFRtVCsLY7;eyp+16i|WX{A-078fQ{<$h%&<_y^} zy7KHMqX^dRUccM2THI4LTw$m_r)p9LFWzFuk0I9$rte;5%a<`{wj6M2%{4)H&ZI)g z;1hwN^+@~*SQ~2BQU-9`_?k9ohBKLL8JrLdA3ZE{KNQr+vrw8Qi-W{@<#cVR#|=X) z+1UuvFvJTF{UZ=+^UsJ0>$W7Zm39hTYk4Jas})&K?$^pP@WP1)_L=)5@Wg16ve%)xgWW9OL(^@?Bve=z(f!wvD_%(l^zb4~~YLqsVJ6EjjR zzyT+y&BY=AWSi~9DI6hj4d)n1q^A%Kxvr3xlJZRu@oocZs*^x##5ahD6p^MQx00$z zjYd+|~4il)rTyb(F<-P36T@Y>?~&!Fv(C z;L!*F`Gv4wNGHgELk$TYxSkKpBQ~@peN|TBCI1wgn!1|(l?;=+1h+u_})jiS?$b79{U5(bhAHC>F8uNI!{x0 zrW6m%qD9dm6)ZLwJ4J->6>y>j?ds!--NlK8RKwi{fMGr0rAvlrv!T}h0+_f6N*jS` zo|i~N6B^=hZS68*4FwiAC2FR7Geh8%(5HvdVXSYt_?y=-ak?iytZ*_q+_6M6IkFk5 zbnPRFRYDaqr2OhMr;}pqh;ljh!2~-{jN(pqlacf2rj3UJPS7OS$Dd`%P90SLPPk%0 z2rx{okf+F9?_TJ%2?TG5M*SZR?W011HV8=XrAvUTXcv3JL(3QN}?&-Y=Y_kjuK9AppiOT zfE@=jphZZwPXqE?Q>h^7RCCR+hU>EDRgI_5O;RME$|J`qhba!VjdHA|G%GYT!BADM zwL>h;DUFl~sqae1vM>wH8(UIC4$S1Owmgt}7tnR6D(l*CdSQ7uxaWu5GFOYh(mHL13Oh|$0* zK5=^hjc#MBIs}+M`;xAVy-EU>tY=v*S;K%)#-U0?7~>-PO32=^igxv+;xQFTHb`Zn7_$|tIllm+mtht&utmqMBJi#sfw3{eVbLg;W=SWQXR5L3@;0b3cyT6QOM<(Mm{P}g>9G`bYqYpM)e;{)~ee~rsdj*)Z$a0OH->Me*M9BVHfPXV35 z^xpsEYu}aqR3Ve#>`8x$L10F1fxzP6Qz&p$s!1gph+Io+sWu<7sGx;Ton%!FdP6wZ zmXi$H=Z1Z&j=m+29UeB@6N*^8AtLvz`vHqun=3bGF1e%AwaG~J_}Uo*?kiHpn06Ir z>F;iuN?X#+B}RdjRRB*|YrNZu*c*l}D=t?xebGxnBMun<<0e;uxN?{LYaODx6u``> zkFJ`w7BW|(!BcBfvB^~*s%|y6Sd%bX2XtXQv*sKF!1H}dd>?pdLW0>Dv4{)(ZL4Jf zdni8fJwBmn7VE*q)vTp0z2<;*N8!d!iZ6UOO)WMjGX>384=Hm!d{^U=O@g2)Oa%bfJFHL=MBAK5-QVD5aUY|-2nO3pdXE$>hms*Aj%8m=ZLxgXo!O#kBjr~-|d=Q6BzdR6zp15 zfGh8#pt;VUeZ>4^7#K6J2!>&P?d4^A6J5{3oxeR{$cLNqr`$za2iAfPxF;;;ey+n= z-0oLmw0wax`w@28ykxKv1igzFZ(LU%90(a^4?Y5%`hl=ru?aXvr`Gio)mrpZQP z?e|DS?y@f61n$)kjuBuYM~Gtm=iACE zQc{T@O2KRLZ(tS;_z)0$(93Q}?~1I!^iGd()JJhd%gQ>7jMgy7;Lm=*LIZ9vJnD-m zzAq0~!5qMF4l>ZE7V9Fg;>vzZ%8o*BoKW_}uG2>34u=YYn5w(BiGv7XWSH*_jw3hp za2KKicEryCxIogvuUw?2wMv1(RA?0Kq!*+x2!rtDZ0@cmFF(|;6cX`Cpzxr24%;eE z2z@Ry{B9b^t+~=o(?)RUX5tuUqyWkPP{!Qi42?hm6%LSsZ#2BA2@%JNP-xZ8h;=G3 zcW}?(rsx{|%gVUSBjnCJF!AZ8Lb7B}TAqjjkHiiL@vyvU5!xcNw9IVMD}-=iKs?a~ zZz(>sPY1histS!=^o$i7V;2c8+89q!+Rp)!&>XTM{*omMAHh0) ztOG)DB;HE&s4MBS|01MPHG8o;g6VwsNRq%(y3}bsP2f(%t%fe z5Q+Q1kDW}8EF*v>>3|}8(Lq*DT{_axP=UExAQ%Uw|0+e1 zjPVNv&@mH;84b{oUh)B?u@M_HjpT=Mu;8B%hiI%wCrhE2pzg=G;v2!mF9+b&hmi4=T)K)`gYAY^N>5B5;{9^gP?LO3lbr# zixs%KabT&T4TEk1u`P~s@?7K| z^Goewvys9|CXElUQZUCpl$(a6$9|6ysL6ijCk*s-75oYZ{YFe+h~xed@1)Z`i)|Qo z$RNAV`(&XatqlmfQwS3(Jj2udCUPRNp%b)e{NQ24s&ER`b1ro>A37iwCB@wIL6FcC zR*osBoN+(Hu%qh#ZCI-@9B>oqpjA$x)joJ^9LoqOZ=o2RWI=(()cWfrGc-L8>?Tu6 z^>BkU%gaI!D2~##;u!EL_v=E*jsuM)*nUkCo>fReF~BAO5c*(K_in1nj8s1{t9nqL ztY|y?%t#qlJm+Qo@Uls(DAcebN-6JR1Jh#rqZQ|2|3tDPKr+!zv_Fl6LA|F;!A38a z?j_aq0e?x1Br|5G6 zRsu+|?(u$@5?HE5bw)_&D&Z$9a7Uh|YD|MtvF>Bh(g%ToV100hR?CGHY)DP@JC6ue z1FZ-l(!x0ZjQ!RjFd`+Rv`r(W6m3qGZx<3R+R17@@;&EsABvG=8zxyPX3yU3mNv>h zmvx$&1Oibs2WK{O=hQ7AK^;ytrL?4JY;RDU0mo)EMG4gqu+M_bm367)y@rzoHL!xB z}}I8{2U%Pc)r{9fzK(1I<~;~n6Z zVI8uclm`jzcG`lWZmU#PC67Xs@csa1^Yr#3gW+uwZEir1Ssm!Dj&(BMH&s@}0Ak~P zo6>;ZRAx2Tik`vR1UP?N>NB4c9Hqe>_)01dqW025DTxmrJrF=E^e8e=9*1e0Qms2i zlq|%~8q0J*8%IeFtp|RxL%f0<3o--}ZXl)-GC-ZgUt4$5Xe` z4_5)VBV|K?@z<29P;g^0+pbU_Hm?;pWMmZ=Xc>17LSqS8@-O=He(|^JRu4?6goLG4 zGFdoU%b|!(6%iti?Ens9`LJX&^GORX3HRG z0wN%Sq$R>DXuudM!z=o*^^w2Nopj?$4|c(jjX_X|Y!kTToQBQjX<@~;Va0Ql%XkTw z%_5m}U2-)c1DJ^0b}kuG+^$DUlNj$X!|(jYFzaDVycm8nLLok9C+4>q2RDm1LLC%6WASY#2bRiUy&00h2ig%vIc`>TN-t2k%PKH1K`l_xM z@Q{>1UbTCHxNjZ$018KoB{G&-*?I&Qhe!d@g0b^5<4b)R$MSe@hMCeP!k7a<9UKsf z80aA%5R()*M78^~SdFzzlhh(4_n7^9-|TTS4X9?{HBK>Cr+1o+ zPuYy|7a{BiTS<3Ol)wY<6JM9|1aT03bec5Hh8v-hx)C5RhA z{rR|!`=8GmxusKD=|lV6+F;X`osw`MMvrlv6ok_nN!oHkvtlqm|HxWLW_U zJ8RaXEDkdVvvl*wg1sAC5S_en0X)9>Gp6Z#HG6FW5p-n-%9uP`b8G2}Q&z@Do1_Bl zxbtj6V`{+>)zjSjH|JEIK$yD%)F*2SD#AKl?S%GvPr)5IK>_-=iTja(a*fkkl05a2 zqqD`~dY%@Yj_=Hs*`}_0d_CjAx^-A0yBC!MoVqkcayGc%2E780(~{rv#L+rnW1PhY`cnrss$vn6By`5HH!klQ zc`Vv{Px;eBxgGk39o(IWs?-^{bdrqx#&xKd+dIlLO&ei-#w=?OQlQ~eAdh}LuzZ&( z&4+1@Z$ZBNvvWG(V+elGS%ljH;5EI7-PmQN`c@o<66)Q>ZK358hq3C^llE8QmlhUC z;X1q;1-E>RLB88Z+~>0nh6Og!eaD{*I+Gn;iPHk|M7^(TxNRgF)3JHeeYn$k{H*#+ zv;~+G96;*@e0*p4hTqyc=~n9oq<&kSq!u2&a~P&y$>HJuKJFVHe>HjlBa8{grFH#xwn{VU_B2Q6U#r-(T75l@Nw{ zfAal=#OSQ-3+Y`XP^EJM?RSEc^;XrV_(kM?`W@as>HdtaNY|YdzX9yB>r~^(82oDq zfH|=a^0orFy@a__fIqbHg=4D;{DZuKp@FcigO{*)xmvCTm<|;PE}9GFA=GF%NF=c+ zAr7HJ&ZcU;UZYbFcR7qyGU2hfOg5v_YV~z(u8QIR8Mq??4rBA}OhjRS>+QJ(w)IV% z5mqTI{^1cW8W;>j9vDDYVrFV?a&~(9)rA_0l8)Lh|tBh(+?|3fDTU9UItLGDHOtoxd^{PE z#8b5)IL-xly+w!{4?_4ASZGOzMjehcsM>{f%@qn64Oa7Ehm2vUhkDHjh97C3pvWR> z(ZDE;j4>|Oj3Mj!lH)Ks=17?>KEe|J%ukl>NQr1F21eqHFVdJ`dr!6(9h6Yc;Xy`- z-DSs=vH^5KKVB9m3366hl-)ulH6bNAODM4ubgOB{5q1_b7mG6#sHe#x(3NN2gWSEs zivdttnoOhI0@U9+bS3!x3@vj!3Q5$*_(9=0r#y-#j*S40mudLM47&IG!aA*P4|+gp%E6R zzw^*pC!j|Hl@wHnSlFmwybzlIt~ZC$gpOJh3tXDB$)Y9{Srko`M4DAyx>TpNg4$|? zCyz>T%4v*3@`!h^+NvR9&6UPlE$8&eEG_c9tHPgHJL9#D5p4%Dj2%I7QnX=1T zB4lJs_tN6UMmo!@G#3A?uGji>`t>_gw6Gg*LmMXcI_ zzjDsGkt_THy_12<{fVnD4Uf}bR9uW4^Lfoldv#0GaOI|$G2PpFM)W@y_ zP43assOuA!gX^%`_|T^R?e(r%)IkRU_uAa{`?SsO1@96R!0+{Z%%}F;Wc%^nX3l?6 zAG^CKPk&JL_qZ7M1IZW3@tWq8U_Eirhb^RuOMMuX66RG+0gf=>RI;_X#kHw@FPT*6 zZpJqV`445JqZ2S-7du+ztV5}Bo$Lb9x>+gDMOd2_k`~yz8wM?CCn6pXhi5!Y2#G~h z+tKcB#SEe7Z*=KPS_~6GBr!$Eq3LpU%Lkue_Qm>r zup>6aRYLZ6Fc3a)YF>jFr^cv_+7WAbBk`N?2=4DLWudI}vewLb$!;V2!fnFvSd z$kL&zb|!k^TPVr@NDlMD6VTVBGpFF2CQqHAPqQRZGCS(#;=EN3 zN#@Z}ci3So4r)$e|53sU@#&c2s237K^5>9>!z0TsWS5^ll0zriXINC1Dp0)+pmd-e zKrQ*zwl4IOhMVEfpixSP;uUzJTqs=?>dC&SvQc&5Bm{lrG`aGMYk&i3NQ0Bh@o8`! zCEU(0T@_50w(B^U0ABlIBSzZ92%4gWoMSi|!arF;X}Tzgm%zhpVj_2T#HaqFx?Fs@f=LP^4u~UOit2>m_q6}yLJgDp3 zDl^(uvW0nwV_auw#JD;Sq7hYZZz<~D{9SY~LA}^Ed4bNpLQfSQWo%<*Gt$?}RvfQ_ zTwX7WS>mjq|CrHzsWat4AR{RAJ<65Bi_i!@e9{AyfU~AP=gXsl5H(@}t|wM-+l*jT zLQv0eRXJIl8J$c-k)n&I7{lRCB0v{j>D%!@A~4;2f)!=NCEe?`i^-e)Q?1_pF&_Uq zSH$vey?K4;U7cK4D8tgCiUmcM9i`%1-4~rqy4@mX$=@^U(x}@Duz)XISzuoAzyMjn z3Mk-G@r1}XB}Ma^aM>@Mj<$Xh?P-#Kn$!tZPL}t)S7z^$c65`BMZ^o9slhyL#Jztq0f~ zZ`6fFwc@FFk&eCY%iv)G4cd^#~8u!7^4MJ8wT%d86H9?iU!f?J<*T63J z|4n^;Kcq=H*n$VSVCr4??=>sRUT&q;czQfLDK_Rb7iknHe-8cJ^*7nxc|YpR+HQAb z=E)7XnGe{$g({ zP-K7*Uew>U+^;2}h7}DE4%#EqTl+zw6G|BA^%rF!V9sqGyI^6vV4(x_SW}Hh1Pq;k z{MQ%>fv53G>JcLuBHW>=7%o5{h}njvaKP=oA-+{j2IiJBt^@~aT{1*V|JL=Ps0qj0 zj35b$&fd7e2@2xdNn!A4-W7of=hfMrabY`=M*+S@-UwqX8l#&5n(L*^ zp{1c^tYKBuA1*~>;x&Uy?#S=mA=l{P2*zAD9-jdupE(BNF@i!gCSvFu;;XsfJjx?A z+?owqoO}gdDt@1DTw+vuoZ)fe2B6=DA&otPMqV{l_yNtYd7&%Poai+q{!QUW=p5(a z;)3CsbKHwWL;**V5e4`P85ScshLxq&7uGbROw9+@1p?Ro2sK*c|BqCgM6F!#aibsd z8BQMGkMX4Ppge)(E%!Ges~l@4X%Y4#vMZlwZYB`VM( zJ}$|MxtR7PU8IC%LrMp3@{(ER-?QCPMeZE>Y++|@kQcz^gE3Jt03M0hB~Oy%KEY8E z;s{^*7jmV8J;m3FWoJxkNY>C~8-OF)0i2DsLwJUxP6A?24kD0I5e$}?#wa4pJRJ2I zWb@CEiblU0kstXGBzCv^8j1@@D?66#ofl zE6~jslE*ov7~}OD)ZyUzH0NEOBYWy4qFo~<(B4Ri)gG9P|Gm{=hZg2-6=5sn3GSJN zd7@pYh2xG*S4alTg^6UIAz(VX-6a_rQ8r>G9VJR&jNV}yZR!bQEN5%}oUdS9Y~tsF z?uvl6C9HvxYo^vH+7MpZ+-yGLt~kI-KIm^gWD^b86rxRqZV^khR>8#7jRhlze%;sE z7h*6ciH>NAqFaX+)|Fa9b?(xFVCN3~R{80cOnRp&l3b5=5s3O`Ivj_kM%SY`=g}3E z6k?dUwMvNzMUy8h2oiU=Bd6wrHroD|1cz4h;GPX4yvFMDk0us*X2=P z23(7-Ane6xHCD}Gk{3a?fn)Zl0my^46oHPS9b^hB$F&a2oDHxZVLt&TrA%LZHtDJ& z)oYIFeYR?>-q4MPmY6Q*I2Gir+NzgoV)U6J7>J?~66wb(3GayCv9_t#yeXVc=(X{p z%yHhGwk1>o3yU$;UWNytGOHPmBwj`)be5HcKviVSL!x@tH@PVEIVz6o1&^4~&!%5h z!U4WoYFh>vMaV0Up=Ta65bH@K&W4cfWZu^54SHxyU@lTn&Zm+dtV?bwfWqo#fo&zM zMy7SF#iAyzniw*i6)0+=#4;PLL{loH*WB)o|FZH~gtB6so+TAF*3O;Jo$)Kkrdzf` zU~7FIqy*~6g6oJLV_p0Y-x`CEd`$|uDAr~jyV5AT{@ZQ2W*ot*SyZhbT9Lr@D1(@l zUZpI*qKA4_%pLUXNKO}sRD;2)>JY|ShdilgB5z+&X+&`=tu!6z4O+)a>DmHKuKsFw zP-cyiV&W1SPYEu|5$@1dp0b`P_reLZwF_|mgUdRTxBkk^K3#s9+qVX3*7nVMqE7Cb z(Eum_6tyH$4iC~|D*)n~HsT`?fhVVm+RK)wrP8hfJ}#U9RpvV1=-Ti9+Du!a!tiRW zyzSj*`eWO6CDFK8zij3A(kq?(CKz!0sa{gn1O3uT0Ze3fO&+fW*B^5XIFmT9kDf-ZI((zwb9Q*YfW zR7 z8ilK5nrdwwfEC-*JZk6ja%bm4MnRSc)XuB^nFB758ttZW{^{(HnV_qctwxj#QEo8W zdN7$bY%`xQL|uZ=1hNTR?mZGR|J|m#IAF4zp05+K%@cGHgMG3$$I&P=O({b%{jMI{ zyOj$nP<&EmN@qi<&Nj16dbbNZYPHt0&WK$SaSdQ229q32&1z zvq?L%ueL2p_YpPMFqdBQ|0iWLPWKcy7j`DdEc>P3#6Cmmk*7~n9?3$a6US`5;^q+; z*AYkXwHoo{7Uq?9Gh6TTbGiWvC$dHeEtL%PT(dG|2(Y~+bmG-ALz@m-liDeG1LfLw z0FaDSV{{Z(my!POD#tO|P8^7ArVEiYDr%)YLbFPHL{eRg9}lvv;wZHoc9$Jw(<&8* z5_bA=hnYe4Pg^o>4)yLTZbO#uHzYN^CT^W%*eMECVOj1NinfsM?Aw9g^?F*HYW4pj zW-FlKSG#a+N)(R5!9$}RFe%t>rw@YXg>R>F^AR&#!*SZi4)D5XtK@Y_3$_j-5!0(Jk>PN=# zeYY_K4l{u*TWLVaG+gm&pM`qFwnPa^7b|k>!hwSmM6YQ1gUgJB|299Lm4&NHabq#T zGHJrz;BtTPn7ue(-w^allk|4@md-RyBU^ffS&@G?Bx_@ik2g?nPmyn}YnpdQctd+% z^dIAh(>(QyFW_j88x*%)CP&VbFe;QcoKhK{y4B%828I{cVS;x#X?1y8t2G6mpfIO$ zA`Um1v$_c(;)ZiLo2RL;J*=;ZI9-r9V4FBJP-UOvx%YBHoDk%jL%NLPW{uBZq`&tg zoTsdu<#i@U+f*&7* zFC4e~>%0ddUBLP)Pw?t+g@!M4JwhUL!}%|7NQ=$+U{$w2278KUH_LH%Dzb5W3%e>P zJ1Yvhjyt=LAG&2tJ6jU^%_Y=R)1i30IK7V+{+{6dRN<2Y_zrckIORgD5upK31F37d zsk{4+idI{%r&-Ab;#p|Hyfx5a^H2Gb18_+$gUTBH7C4OzV@&mVc+IQvEME~ z4txYK`*=T0#edB@_hz)Kdrxnw+g7{hYOAq4HIa+WE*tdRSuB+cjA zPGMR3|3aT)ZI?Qh!25!qv60Vxeitx?6a6uF^uQ{aXac+*f2C=rwD90+dN78Emw3Z2 zykS557k97G2->luG@vW%#jDN7gRO2NdN(NgTFL6^0KAM`@HYI;@|s~UDdZfyd>QIo)1IMZQ)WDFQ3Ej2wo1uLpTZFMD`R9s}E&9mLaJ2vXvbsE;c?!Cf18>Y&5Qb!i?IYp7yN9?$q8sPyqG?{|*4K7{zEVMK}*12L3fQR7CAevr@t zr>f&fl7Gl307NQ^K{LUMaLJ}+nzWVUSl(>2%gq^`b$tE=dM79}lqEgtdXzNTEN@#T z5w)~s6GDBy+!eW_Qs;F=UJ4%zXp0{|K1t~O4NSCUT0Nbgyn2HS@HqfS|ElWEA35sC`Sb?rH=@n4OFGt2sJ#r?TFosY4^%6I}D z0C9;cKfe}a@R70{R1lh|vZ3%DYYgM23!5}dXv1+X{E!_CM}%jz!iW$~AHcq@i z^vpy9|6_^64jr%!xZiZ#@u3`h1oDA`cnr^>`TQfIJa`(88?s<(7z(*K#n?bklHTH zVfMO95xW{h6hbq?OL4>sWt8Vp74^)s(rXkGA~F(VYDuWmLa9983t6=TdIDjJ z_bx1LzZbi798EnXyyH>id-r-EUSFK#jOeXKeu>0Rd4n;w&js&U4<*o&&ztfyRK7likpMw?U&_ zt0URso$ihSi6Ql?dl6cO|F9AVhERu1U`eG)hSEte8kdBfY6A*Q?L+usu|6fN0UlQG?O2@d4iClc8Xd+fF zCBknWtISm5oHa!SZ00hW^kGg!Nu+V`$c<^F$N-V~kT4!Fn7vu%Lm2pgo2`eAB$FWP z#0SM@X3&qbqZS7_=)sMM&}j=2+xZk}Nozswhb@HD3n!_Pu%WMVO=KI3>gC50qBDg) zB&AveWy%G0qLv;r970F)H-?5NG2FCbM5|~IUCQg8zSLj(H2O@QWN(cl`%&hA(;AT) zQ<*_R<1tklq;6hwIMEEm=t^g`8FI~_s%y>!p#zs|Dw0Ul@n#@}nlz38N`yxhQzv2h zCt<2-cygGhSeO9MIaP9b8~q|ak)zc9eH4^P|LvXi018B55HuO-+2+A;$x2|_GnSgH z$*~mbF;*(EqUKVeJ22YHj3#n_!z8AG#K0QNd4pvbDJe_|;nV9q6Pk)$;CyCk&6@s9 zN~#OrPFwoXp!QLmMguJ$IMTFjC8|7Jz32s_n2@YQl}KHir&SrrFs)WLLEZ!?_w+!o zfWFPFjNHarA+^?zHPpBv@oPdQy4)*P6llSNp8p;f9VQxtthbGoN)rn}3Rv&~Cp{y0 zSBkRD_Q?p4jVTEyYukvi4v*GQmMb~i)y``0vjPPzQ8m&nr1qCvqQoltpz7MGMs-Oh z+@f*Ws|vR*j-Nw1=hFZ@R*#mIpijhQ|61*+VLi4rtDAJ|K?x^fR^C;xRAJmNyU0^s z0X7ea2#|O$H(rnIBV{tCajh2IEA$4Zjd2v#1AE%om`o{r8v<&Q#rVx|-dCK4-3SN# z%TB;Z__VnA8CwQCPkBaFw!;`QeZRS#B=_?fcI&M^f9N*tEfP`gAhB@CS2c*L^Q~E~ zEf_J*4Voa-y5X#syen8bCB_G?0Q7tU$`xv4tk=!ng|HSVs-PbVUBa5KEJU z(=+bzVsqA#PKqjJmW!+8rkfHo1!_Jcd z1547tUuLr>Roy2S0~Nx^2qM6`|HVU3Hrvh8&L)S`npq7S+S?v}UR~+xDDs)uk)25~ z{!qo~6tvsj?{;^#MrUoeeS)yFvS4i}(CLlvdzp+?b`eVbfN54!%n?l8vIr(c+R|Fd zOm4E1iw25*H{;j7i*Pci%-=AvkibiT?Ie+1?=U|+OAdapfi%7-(pnkI*JH+Y*Sr*Q z*WBjD0e66ptB^kTT)F1P^Xoc=Rd^R2-txXR|KvuAB-|T!uF3RB6%*s_elVpu=G@t^ z+0}u|6xB2>`PHB#6SQi!ybg!B7$rXOAynL`7l*sA5&T~R|A#&X78ZifwCu3shuT;I zCA6Db$ZL<#Tc2ZjBP~1({|wh$+r<5Lt%MyBdp5 za2dpMy#nMkEQG1w3%JMXG3g*fgHyF^o4|yzAQnrzHx#~z8@_>gDVpgu5%jO+YpvKa zHa@H;3v@Ugtc3!}KO4ls)0mnXG_7Pyr|X-rEb>7kJTf7qL=dwj%fm!JQ@ zuw`tyGXt!3|KbxS+%`!}qDq8Ex5~C6R4ZIEF$xR~PwYI>Aw_sAMbTR_FjxyKWW`Tm`&~8Y@Ib=^HjCo0vjGvuoSC1Bo`}OTKi(9z>$YNRh^*l3O0a>0^Lrf2kv~%W#}pVV zzbri%TSYL$7T!>q?a~0V^nijCFSS&l^715zlq`!(L%G7Nv_dkzgu_6Y!zT+&ibtruMxRM7@FYf|`XbkXifOeI*%#)Qcgw2lV*#=tAjQz9#v z|CB?NyfQQi&6^Upg2|uJtVH!0Qjej>IB3F@iqE$?O2)8H*7O+tL{i7`>U z*!0JdX-2G^G)6d3+*HsqYKX)Og9?>^0Yo(J7*jI+Lg7pql|wvZ*-&6aPB?|Lv4bzX z9MOwp%jY~%+IdFf`#{2s))$q#2#C>_I8>;y(U^3wy&D=~yPHY1GWyXnKuJ8#|IC{n zRM&MqO&JN*c*LsIwA4^oO>&v2do>jac+Jpjx7$jg-r?0+bi}Z{P)rj*g?vc=<1}GC zQ{ohm4HdXy!FY>Hb{+0W>t%nv&mC()P1VH z4FZffWmkE1S9tX{wRy*-)LFd3hxe1xp);R8s7ilqv|l{efQp-0IKTv*vF%77<* zoG6VPQjJ2Q1wGIkJ)Qj5X9!qW6|6Zp%Uw;h)a631qyVI22pqvQk&Gr3X%d$povuZK zuf<#c5K4$?IOoLJwB6JDQc;@m*p3y~n|y33cp3iyPeJJ zZIr~7+?tig#|1+5nF9K>kxtD-rK|@~{n?=HPxqxt`s-Xq``lDrI${vcs*6?civvYN z-9%$WsvW?ly;Uo`fU9xK-ylxn4Oh#-F$`PfMAow1-RKQowbg@XjR^ww;P|sW=M(@SpNgPe*|JcvG!GsR!uy&o? zB6Y?~j3|CJy82DqXGmBi`N=W;!qgRDstu{$Y%r!fw*=0W99oP9J_p@BVI>V>H^h<* z{!7sMNDlUq^%*``3^qEq6CQ&l%>Vps*k>V<>T-Bk!pGz0Zt zeOXm3b|l>F)n(O9+>IHDeZWn9lK9%F%9^!glwRT0K>aMB7KK}424QxJV>xyOL&ZCH zF}*NGVLcA4J{G+CapvsR!;ORC?nvZrhKF}OfbxykEMR1D|BjLGon&q7Sx>cO1K4Cy z1W4!%m=OwP+(1z3@!gjixRgS(mxpGW=p&wk=C;5fQCZ;Qi$#&e#-H zhy5z%HZHv6O@=rY*}Apk+2Y0wnn~HwA-`ZhQie7Pv%!{4hb0|i-_ulkrL89dhH%~~ zBW7pFK!kR_WS||@DxKd_1nNd>vmHuWS>Qh`WZJLPm)>OReBMolcr+kw<$-okE%pr; zG%qjaS~s1`PdZX9wjFqCC5Xzd4(Ga<@H9L8b5!=IE&MGshHb+lB;@ zHtCh~rH}Aq)c|Z6EKiJM3UsFKNioev9!7H(QkoKtNyc1CHn+>Q+)>qW^Im}-cZ)SI z>RFH({oQiE%K)d|H!vOTs4maI8r?0oo7ioct-TYIDJB;qY}zc?dKYcf6+kxbZC}e?1qkj57;ck=W}#Vgm6b*7y(nbA z+gO|PPQ2*B#WrcrbMThDd>GN4+3pp`bM<}SLiYl)ts+Dx_wvR7@l5xXdR2kd($JRm zvYhW{CrD1KYJvxn1uh$iS z=Sr3#`r|PA24oBgUi!^btteOZj;ug{BOTOFCY7OdAjoWl#9`5RL@JZarIVO(C=OGq z!ZC?tx08zE(#;wS1B3Rg4XD@Vr@INQaM5#m{cg|8GZi6T&INdP6_zw~hy-{=xCfUg zb%5qbB({hJ=T~KUbmT?$|5+9~HcDD*dWxDl25BY5YWWH)3t6_x`f0X!P`ca8`|BGF zJS@Dc6%}kXK^37n<*K=AHBBwGa*ds>opSBb_s#bg)@OuuJS6d;zRuq6{to{z&Zluv zpN_9iaj|c&@%Cg6KYSGO#XE>F;X6ix2IbMBFrvhXf)1=>W9*?uaU3^lk#(%$xRD}D zl00cLWkZ!KW6d%*=ZnFKG~4}y0&r5!ojRFtS`q;ap8-VGh>Wl%DLF)rm?C28v16b! zEa&mrfHG@KhEKaDisCAyp(o8?F7-+YRu;5K%5rV8Y>cQ{s*2UMi`Ul6z0mr~D%DqR zt6RjR8P1EiSTP6T|Eif;rfaZUXM2LcCLB_b||9Lzi&uOMohfH?EJ zXRzOE4IFT~d4vYG>YBVGGEC{Qw{V)_BIGMN%A<0PFqdcXD>bCn=~b&!7wKy*u_>#E zE8M4{knQn2%l9-t-ht2xNdx35sucaC;?ASn4}Cr5rm<-0oAv6H9BM6v$#aC9gUKrh z2Ixs$4wgdKgWE+$A#{9ED3~R?)B;Un9v1eYT@f@!4HaHgSXnZTIio;5;XR{3I4oV2 zA8t?#7ZREXSV0iuHh@?BenM6mENj}-mH_cq6MsoGpcV&|YQk7qq z)_vKKj4M^N{~;)5iZTk6HGxN5VtNo~u7M0&G+iKM9PZ-uu6#qzSL=FC+%r^C@3l9;1oE-v-?rdh zspv*p!7?wPD4j>?qCOS(%oYWL#Y`aqWj9KMn4Y5Xr6{`UQi>gZ>WW~w5Ef#lDd%-e z$T2bl|8g{^t{9mu&o*?4m|A^B>k!GR`v4fo^7@WF4)YY&vP2gR?9$9SdnM3Fj%$%R z7(F&{)_?RRtGMBQ?X@Cyk)oZ{65ZLZyZ_F)cDt|bu|z#W?WrffXEBO*Ia=R0X2Ib$ z>?bD+=S|^$rnv-et_{dI6*4M8I#b1rRXGF8&HP3Zo_$Y8P;!xvcr`3` zSr30_2o?vMmOC|>g-OQCRrG3thN?Z_YU=CS!0flY=H*alRe=?&g2BNTa_=TNsTFzH zhQtgEgadZk2NN#kB0qhYgVHCuyQa?ul$Z`H# zpoa!XxxFLdcwr@A&N@@L;!mur5-k7Uh; zGqqEgua2av4@U1Z%||qbA}>4YAQIX7L8 z3~NW|3Q8^!oRO+BIv8bMg_yd~gg_BA)GQO4BnL=6(DWN}6N+wNqmQ(y z^hNx`12AgQrzBPcuz?a2QL^?-!i8Z!T?m}mf*R2H^{hag#019Lh}(1uFr0GCP-M!P zIjJgDM162cA>H>m2quTB&FRZ%4GC6`*wYAQHLE`LIV=~cHEpr&ZVWLiy%$2)A$dq_ zub$&jzG98Er{r2Ey;xY1W;BMC!)x>w`>+P)_JD!RLyusmkDsMYmjwwAxeRa=Ve+e5 zgacR<2D?0>#3*cKSrFcUsy|7s%5aIgDQPqurp7__f5TO390zI6|C5clj)Hur9`V@X zJx=keoM4eL|L6->?jtOLc`hOG!@~@l)CdIT-AeZXBz=0o1LngGkK`6a*JRSZ@UW{? z%!?2~n8s@AH6?w$oTdII7sBs_?|b1$Wg~~DdJYhbxy}UVH$Qd^20<1(c%@kd^NE-X zuF?_9<6|(BRuDy5W@@2HOYyiioAcoUY^hmWH{2E%sp#f@AsfNbh0cJ$b!>`DEZjMM zV`l0!z?h*b$X#YyK$!EIF>L?Ml_EKFe@X_o`Qy{~ciE4DUGw95yZZZ))UYx_akh7~=Zdeb&yv&CPe zL7MMNKW{?2-BhMY*Xb4SJJbxsu`4ybT*~k&W0YR4sUcoliNE@Ye87>}Rtia9PPXH< z`nbl6R^*X;2Ws^}`9k!%a^js^P!pv&v&SXolCOJP|IoR)LDA-2{-Nm>mM+Fz6zlh; zQIa!A=QGmVZSKiinN}4MADHBQHwWjM!$w`987YQFYaSI&Y$UP1h1>7sf2^q^BlwGB z%-n=M^7~9>NCc3(JVt;@=j}XmY+3}$i z&^pk?`TRfq<<~m3%<(W{E!R_&;}(8LfBp0i=tnNdVQPUiPIN~F(ts{&qI#~eC*nf~ zdd4r0!E_U|Vfx2^f>JJQ5GaS{T30u3ag|Njv?bu+b?*aq4_JCU2O=|g7t^O{IoNzD zVtj0Oe4_?q&=-Wyc5w>FYEbhf>X!kDXEwgl{~2yIM1x0o$CY_1#We0m5J|##BfEHV1vU zU@=l|^d^QONWi!-e z>xgVE*b&SYfDqVM2t|z`2#i>ECn_j8N2nd4G7V?~j^a27682f927AIMbMiD(ji6_d zFmsFuM(D$PUJ;2DSr+T#V8gdRJ?NFkr&|N5d}4`W6Y_jhH8xjKB|MmLcSnRs78yWt zc5f_V98Ghq;HW+z}-E(V0HY?UxR_+&)%91o7rw+r&5hy7EoJ6451Q<8s02aKtbVOTtv>0V>V zKJCM4$VnHGz@HlBnx7eR-Vg_wX(fiyXYB)n z$mEEUxQo7*b;J3BN0T07mz*k7c4XKWiXjrN}FBkLzo#8(fC-} zahb@5RR$W2A0;Kk<#PjfU=Ny_b;T{grAUBApOk56>gJ9cXoy@Vo4*O7`ICYP8l3rc z2OT(LHmIm8+Lq1Pq68VE!(|kFkQ8qDb~BTj)`@+?1_-c3jT2crQDBi27$kPKW*cdw zhKHE%c~B?0p4+)N8K;qFICZ1SJ2|tG)9Iyn%7w*~Y|j{g-BeDT*=%Ua8id-D3>ua} zi3>txlkxN^#lkQdItNsQh^;prxYvlqDXk;QeUSJO!PjB&<2``lmHqRYVYhsbYIZOR zouN3R?4hdw301Ly{|}qGqksS&5h-7YWS)m(r6TDM+vQTH8k-~JWb8STS6F3WMTEwP zLECndoMfv_SRBKnrDK_ST-uqRQ=50{5y2{hrTMXOWURr6NNg&mZiYck zk}TIR8+&4W%dyQ?F<=^8G&oND*|X?*vhRni!Egysr7;59e4+5Nw5p&oc1*Kbh%mLM zftVHaxvlru|E-gVkFXX-Sa+g>iVTdZqGoxi>-wVoHd7<`mG4SzYDt1w8-ZKPwgKj( zUfVoH^q68hbZYf05`a>uYD7vfKD)HG@98vMQ#GGj0m$Mziz&QNLTp*ufc~bjW0^c9 z8(1i^v|Cb}d@DnV8#Lote4|N+0LQFEF}YNc312p|my5YJdnI(k1?A|6bGmmYIFH~8 zx;3jt#92%GIKE1ooVeS%B6VU@do{GHiQfPpsODl?tDdpZ8ZPp+q?@4EJ7vF8uxE>^ z{t^yYp{lH?yA9hAumqEx)FX6zS11XFE4Ky>jDrN&rOvpkfMvdptBfURjUa4f6tK9_ zs$2ud|CXRIxu}D}%7kj6)nS&TpLs$R{Hv!d#0Z~jqEj3&v3a`lMyTIfC#pNUN{qt+ z>AeuFUXiLPo)du=Y$|f8db+#8#~UtWq<}=MyCPh$XzNwVo5H*Mym9~mB!B?XE0KZe zchy^^G~7@&+{SD7v@eQc`0KZTE3z#oXq{rWIH$hEML=`biOEQ3a!MK|DQbU)s342U zuABx|oSRBF0A&i17N$7oiL{n9wE0-Tur|PeDYZfOlm56d*LH&oiK%gy#-hWK$4@0we$eARq#X>7VW( z|HHhQ1)r=gus|&lOp^+{R1TcO752)RY?;CUo9Xkz_#8_V5Xz!f3Cl`)WYEM^@x=5s zvum?^;Oh|=$e~KNLSbqST>N99ONC@iz^F^EQXCF;XvQA1kQo;ZbCJUrGNTnN!7CGP ze2|QBe0NH`4j7E1BH6pdLZx=OVGzT|w02|$+lJ6J1hi5zy^ziJ6#<9r&L9v1@|;LA zVu12c)Ux89^jDvftX@Euhi0vrXDr7v-6@)EGJ*TIe2cyzM}@`E1##R>b;QJ|d=C8w zRmN$jam}mdpl4pSvv7U5fPJ*=hLyWqo=&~krkk`QO9~GVc4k~6%)4=I`zio`aCeAxT+!|rr$B5i8zTC*U zqW%-<0{4IbWY1fg=sS*I%DSKi1X;Ye*L>Yt$n)J8o#_{T+tFpXf@;E9MCFipo|6^E zbsCx|3KYzJ3SHjPmYUzPTHiO$-x^ZWg_h=MyoRz}c5eyOaW0Y+-nHy9;#pxLxA|3t zIpM8|ihVvvgBKU-Ea)_m0KOg1_nLQ7f*Oi>;$GP4$hctV0`KuI|L^f`%IE^)y56)X z6X^lE5}t)Vvb}I@Ya>=N@s4A0RBc*wSB!nE~=RFn559bohy_y?dPlj zdux;4>1%P^o;v(YQBc=+C%hjH+#&v}n=TKFE-;LF+!SzuGQQ1M?)2|;%|}Q)xS^xl8QdF3W$Jwe-r>!RVv^Dx$~cV}i-;c zf>J`>I=b5uAp#%%&gIVS-}@2pFrUA}?p(c^@2>U9IPbmh`@awTz+dli!Ql?djWi*{$MOLML7kkHQ6=95ma?XAizc{p4?f-u@b@L3Xf@jpgO}_fl96^)BgY zk`BH=@??K0k9+bUmn_xX%;LLrWL~vvC-XpXPzVo_heUIsD49k{lX8?gr&a3p;h0n+ zP7QZR)JC`yviZEtpxNtkB#E`o>Gi5uIgaSbe1}4-LXa%OZSTOqKyJ}a1mqDiQbc6v z&$9AzQSXhdlMl@}0PYJ$w`s#&eLU7dT~3+D#F85+#u( zE1`H^K#!$3pV``KI$#Gi74zLaSSo{~BP?qm5|eX#mi+AtyP9kwuzRYBLk1 z2}MmPW5u+Cr1h>`%wO<;8wT<_Xql^E!&PtFn+wr@#|IRbgMxXp^i_ZrR&kBZ$E#tw?69nN)iZ&a9}iu+l#3fQ!SpJT$Hg zC^x;;6ZAwaM=-)v#0*$i7#nujUF1A=&R~&!%+nv+xz8uiq#e{kM0UM)j!f@6l{K3{ z8US0942sG(Q>$c7Fwdf)mR)I|!RJfQ$f|4H=*+zL&3yG86J2|)+qYj~lmT~CC+p0U zPF)vK*iS|Seb2qs09x(PiY-P|0Ym;gqECiB9>rr&OB=AA;Q9cFxX5m!%_~Y9EbLd9 zSB#0mtf~#|<5y#qYt|AONpQq3|HCz>S0hvmASGgzo^eKmo!;?FJQHp7m}lBmWayzO zSOZUEy8Wz4f`=9nRC${@7tI5wU3<`YX&t29tEqOIuTSy4w_dx??00W^CXh4OBka5b zYx913ny5WnU0eb{A)c4L2rJ5IBZX;zJ zyVAhr*xhQyuv}PiQFB97RZBm{Gu)YHdJR&*j@fYCvevAvN#_*WS7A}IrKMLE@jP?v zI@-A16$fYKcfG(Uio;or`pa=qUQBXpGZ$@(R%IK0$REapwg4|c(7%YampuLBx$Nxi zyzi#)c>AhZLbd_8WQa{{cPLo(#{ZMEk0D}ykF%e|!soP$B?@D1kif>y^EnJLtO}FJ znaGxKI#YdO9Vl~v${FH7EAp2$$4A`CuD zMlE7mi+qteCsyowH=E!6%GEm|UJG^4I?`#>!##3jjfWHT4S_&^3W(1Vj|*c9dGtv);vHDXBQdN60PqHs=Z{rQ-y z=0E~VmLfn%NgW-{&?y!s(ikuNUK6AD$#Kwd5WCxC2L*`*9m?vB+W=t}{3k~<$WD0% zae~YQcarJ-@{bmb-_4-b9{(y5>Wbmonou0aJ7g_VN3Q{69cjr1G@=We9udkm2^qOz z#dE?tfW*3UOAT|>G7as8>4TMDT=-v;b=yX znk|;77#ImFBo~w^wAiseFAhTl1+^pzdUnh|h_a-kxqt?`=FFg^?tlivr8&ZAwzEj5 zjB4|yPs53ye9S@>RjYlJt6lx-SOsAL?|PT46!XqYG@DTxj? z3A2ccNc}3Qs{i63FeEHV)f7iq#pY0G*lwd1$|4nT$t}ne!ZfA!xoJsj%rPg^o7y?1 zuZ$Y`EEerhFCj1&i`GR|HtVPROYxA0YxcbmNv1sOjXcqo9oG(})g3G;Ag;Fqu*KdAcir zhB>QY1t{TkNeFRAJz5(Ri}Xrc3bGY+=~~0U_B4xeuN8>J-y2KUejDns{HAId$wsNv7u#(8uk{(Z1*?&P`;uI$Q2?L@-e9GYdxCYO2SZzDnlr zg15@%rT?>6AKD8SAOPR_F0TVt+}*MM8v+X$?lVy%?>h4v3kT5m!4W>_`<{?83%7Gm z79ME0s`W`boy#2HT!!(LcCM}cW7F# zhBa1^J%w7gcopTDk1eWU!1nrDuRbQij(JQ~;sLqmLlWaDb>cN9g7VG@#1;w>EJo%& z^~u7^r~Ut?l*=s2~G!XSzWCV z@cvXF-CN${w%k4Du|E9@S}-_wA^fIEiT85f$-0pJ)BWro!p!0>|IBQxyao{5pv?X*d9Hu~5|7Ik&vhWsw8SMH-mL<~ zVDJELg66IC4nP6_&hti(^gg0wNUwk{PGfR!G<53`3d_+5ClSg(eE5nbRBfo1KoFin zGERvVuIO*>tln-9W6we6V4I&`GF5V#%EuaR+YXQB?6&K|v3Qeh0ZpL2GKjDU0|E(ykF!p#RX}VJobU-{XZilH`J&DWU+W6u${I?EF|v=g zwvSMbuhAZkNkA(N$ImiqO!UmraV{hMrYTphtGU3*b z(!9D015dJ(GI0YK;H?;sa#&C_2#d!cp{*|E*tlR0902s3r+dK2myQPaSdQ?J&?z;` z1R$>(w2>090i1ZQ3JJmqu5cWWk`xqyS(dE}Z7kaqzy*cP40{9~<8c&DXTA8b*b?Q~ zc%>ln;OkJMZ_Wi?0#Xk{jymR#`OI$Zs)Qypu@Kkfj=D?{EubO$aNGb8wOH{XMKTm) z@!tyXBGio{8}T8*Yjr-6GE>oSZiOV#f+po|QwlTSWYXPi5f^h2_5ThHWZug>2<#V0 z;0gMKguvn$tsoik&9PQtE$5OBuH7P1l3j%o!1fHs*h3LL>UeX#3bYXftTA>D0601{+bQ`pMT z&!mzq|D#k|qtB#;t-?j5?2EF9kBS`fxLE1+$z(A^5Hp19TzF4FK;&F%gqx z?Cufs1~gGCB^NR@1+X&@FfvE9?Jklx+HN6*!$C8XL(_pXHvlz5G($PnHDyX5ib(ZB zLAP{@^;l~Hl+;qXF7o891ZtH5Qk693qxU{dIgt=tuu*ozG*(1_DMFze?^ITMjt^`t z=%{qYisd^44Sps;Scmn#u;xrhu5GYwH2jQGXHl6F!MFd`k*5t^c__6G*MCy zFcd{i)G%4J8ZON!8V5!j1XeVzz=jgBcr#hgfk$g~1V+G2m4|wERc3HP9gNf|i4m8o zbwgs18>{n6hEZ8zk6EQJNC3w>jWb${X>F*LN#mlMur*r&Lj%-R9lh0fq7(rk4Q=2x zUGMd3gGO%76UVR$0>2FIAQQc~%rLo(5k(Uo9Cd8Vmdr47LkBjG&~{W|4P8YuQ$Ldu zA9YjTmj4hDVMB2=H`Fzs^i)&9L{^n23(95{#!7>7WRKuC!|DV~_HmICA6qtt45L>S z0auNHX5}ITmI`Ol&RH8|S)*@bM`NN=PAm~++5dbEG$vsTkG37tC<~WXG$zP)y-;bl zZa*Q!U;I%I!PI!Pa0wR8iTMVdM7i;2~5$b8jzVQLXoHMM_||ms0^W zVJnl%&MRRhGkS@`Z~K)iK9n!Y?8^ks8Zef4d=@swb8)etadWhCiPF4GwgeKObjuMu zRE~g))F}ZBEVMCnaY<^WbpFs#{aTAF!QxMTA!ut1#z?74zmO&fYo}&6IwJ^owRLM> z^E4t@Yuj%^@GoHY%Hrx{PXX=p8WZvI4m->(?=WB%VkUA1!kqCD_J zJ|h~BxXovdKCaXC__Fs5*AoVNB0i{m^4mTJ1H2LS~n=pFX=4fEI~Mg z=Vyd-cczkx#`Mu?XHQ)5K?sA%KAbnu^wvz|4wT~ohZA&e@wRMzSW_j|UMaFPClq_P zm^BkmLyZTDomf*pcNx@NyC`iWCjGvT^5kL@l6q@tx zWNo#iOt6B5^!A7pZd`x>kdQL|!sa?^fv=%~Q7n|eQ7d~^j5B0eg7$*B@Q``IkTXYe zCZQ*3*P|i`Rv(!d+2*ab&~V45j{ok5U+DBEk5^IZ1w4OvFe`$GoA@w=c!tYA-?ZXq;oYuSn))O!))UqjTRLr{HHI;FqYCa*ytf-@|N zZIL~=JZ)-t@-3QETjDB_y#Gev--g3n^!?E7Vt7#DThXEwhv4a`=^*^ zZ5`Cv9A0V*HTZCnn=IK>k9CMlALqCEd8n*94Ec%~df}~umz*wHn=*K4|I8p3nubHU zZbLbgsrRqb7Vav#q&0f5jdI4FL)v)srLskOiW!@4YZ4Vv8ZpTLi}MCU zQMpkYOZiCA5K`gb%qyrTT*5U@1QtN2Iz+Xd*@h5c4( zIyMG4n^j~Y+U*Q*@V9ApoUPKjKl;a%6(5ZIxTOb1!kYJ>RaztFxuF}HaFL(ELai(L zHJ5?t+!QPNw*R^tlmG1+VMY;Q^jgQQ+=h7=QBQi7)1ek8bIgsWum#*iDfD{3?QdO~ z&F#BhAoR>VF(zTHzxP|0%>arc^b+Ll&N0g(Yq7IKfdGn9AbKTIy4NwremXYTy)X=xd=y+A39Kx>mK#tq zUJa+*%ZI)~q&-jYTKhcNhL_{o*PEgx^~@V|iWOYl^Zc;M7H4Zx&eIpMR|h03m9J2- zA{ogA91jDv$m}J0&&5iMvEK0{HsFdo*sd~XhdaoM8+HF8*OOalU*0i_d>Rlp!x_0|9}ein zuhk1j8~={|`%>aLt28s`TK0wap#KOqsh!XmTJP>*P(@iG@D9E=+T;QI57Acmt5Y;B z(_asK`AfOk-QYhru&I}K-(^zl$<_m}zb3Wn-6;_QY0^&t-U0}o{KehJ>f$f z!wbsZzsliBfbgL|9xQ<3K^^h`{o+ONyMQ1>lSdbHoArh@lvqhjcJT4L|*3YG*tDo#A5kKIcpx<4g;UQukpU^`U7UCgQ)gAj>Oowq>+4%c=`1It?JaKZ164?Gh}mGEAe}0$r|AKj zF8^dOU&b?ZMf9|EHT89MMCmbixA*tALLVx)spfc~HMBGKGsd$;{KiHHJp_bseqg-3 zeFj-GMH3cOxYsXUz;y8D&BG^547-E+6vE?&Zc(&|4CCGF$e^G>gAF143u&^SNsuH( z#v4F@05FytXC9ns(;fmBId$&j*)wJrDqCFoELtYPh@?uFjvzq-1c?zRN=&U9heXw> zeL!YCVKF7xlVTArE4u`fmb6-eQEjW0k)0-V%p`CDi)m5^O%8?Gst;l_vw{o#>B7j- zmBCCrRiWbelPF`yAUBR|&VYyznMI5O<9U@ESfX3WLRRc_T*ayt`_ime1MEQ%KmVVW zJZklwX6<(K#ujE&x!|@h3p9LLa&d(bdw0R`Zf3oD7W`7LIIlxJ>bm<$cQ{ew$cuVv zYF|Xq`i76@Bk->2{-(|M{o>Qy=pygOn)LVc|LLD@Pv)fsAYNUaz`}tEB3Ob`SSg4g z1W!4|5<-1^#nFXhX_%p2xaA_0S|J8tz#uLxIMqEAT)+lkM-^DYT_rHpp)C|7x88Tb zDC417#Swv+6+AMA7Lkq}HzX*TSjI_fnf20HF`QB=MW@Qg~6=zj34OnH>!N3);FC}IdLiw5EiY zuxXI!nD^e6nqti~q$b4~MWM#Eb~!&mX5r z;X*-|(lbw{=ct69Hl?U%b9#b)mY=A`0#dRx+`(t+((^SqmHYX3p6y<%APu2~yc z)jeM61QXc0Vy(5*_|&!OhG6xPUfLP94Hm=&7ZGiUBo1>e#x^dnwOf6+6=UBt#?4}R zIqtEe+>q?i?jz1hatY-3Dj7;+AB%8#X)CkhFO?r(nK!dwW%trs!yTdgRNF35$azo090wlELl663gf&HqbskNMz|?sb66T~2xZ z;+!i07O>h_u!TfuO$bPr8Z>Z>6fjIi0B<3&^u&&0uqj`LF|0PR*&Olb${1-)!3{HR|3IR;;QU>bb@s4n`iUZ*Q!F0tXOeK2m+{oFhEvS5Eq&t)6?lCm{X!M|_^KpAG?2wLV#}0b1cD^K9f& z9I4Q5aqAV6%mtb#SixB4#xNNSQkFRQi6Pp{h*I!H2~)8dR>DGGD}<#$DJmFR_QDWr zs09r>rhv_8hJ>{o%0}6n+YOO8AuB2u$`iZtq3lmiog!2>^3o`d2?ds^V#EkidFg`&7Vi!Fao0565?bhTS~t%@wn1_``L0>wF6m?+hzV&`Si zsj*dUtYhEixSLcx?pM`06XrHIeQvoflK-u%Wci(Sz6TwzcfTuTEP{7!Q}(BMFA&8t ztO&0Tpze~HjNK-8>w)YQoJ)~l*!v0t!bRXDv;rWQP?}P*9UM+LP^ONePHXV(qmmrC>tP(}QNZfMv5d~s&EzTt#U;MRA#-3m1gMdKR-7pkT1 zF>=|c+~kUJx$ku^tieoHF=yAD+a+(5udM4_ThXV5DG9H~7%RMK89?bJ7;|Gy>%Ovg zfo9%Tes#EJHWOmd?CP%u%Yq~U7Z^F4_2_0uXj5JKITj3V@W75e=t4-CrHD515Hh?P zWfrEx(p-}tK#aECCOTWLS3cyw=?)7mIJR&TRa9jW=NWli%jhg`e29`DF^ zT`ZKZydxotGL&USJt2(24AE9;0sv2VoEQGtgN3MJ@!O)9Cl-U)#^`@5^BC20@aV4B zWxJg|Sz4?j#7*#Q)r)ZTL;t_q2v|S6n>ai>gd?hTW|B1UD>`r~L?YoL^L2}5JiLbE zc;Yjjd)-rwad5iL$eI_t`)2*|!k;9`| zd4^KmzPJs{9hhAU1)YB_1JMcKiq)CxAzdfHUhMgqFA&@pm`3vDo)YZdE)9mf&0A_E ziU&H_CRpI;sof#8o%5X_Z&`p-Ro{;ZS8x>)CfDE8X>dDjmf z8G`5=gxOydGTRhx%l?_u+8y2$B3PP)or!J`qXe!7?xhqKHVfG;!4y1G;V4z8MPjL4ocTc45JEr= z2HyAK;2Uvb$noIeePR+8;V9xGJ|3YxHjh%s4_r9m6ULf6z9LgNg?#1WEN)up+2Rfg zq_kCA84exmG5_DB-Ps!2#)P4gWwF#UR--ctNi-^&EnpxAHUqlRo}kg8OYIvk_JO*M zqf9blBR*maHVq7>pdQH@CeoZHZsHVrVh`RU5awej9_7jDBO2|&`H7dLLqDv1Gc3n&=N9U9SU7tNP5Tt zz>uLqBh=yLAtYmHon$b0)aQ&PC2eB~mWwgno*K-gVE{vUw3^c7SWdQDI_M=jUrcsI}%Dtaes-}KoO_zO^25p;FM_S?n$REJRM8r zC0=e_2#g3gDvAS$0EuSUUv5}BOn`Q7mSMV3FA3C3d1B(7CYD75hIA9F9nZ$C&tJjg zW)kR7`ot2-Cs7t9uT>&x&Sz+54>YCbe(Gn9W(fswz<>THfL0G|5~ysB>1_VY2|k2w z{QqMDJScYl!El;IMK)t>xn+?=i-vBfM{>?FN|=*?sKHPtpg~GP0IHl`p+0CQ2-aSA zdPtba=q%_JBI%W+7LZE}z=>X}rCKT?Z7Kn9D)S(Z#TAQYz9*?(si|&aRy5_3erA*w zAvW13tQzGBm}%0Ss;xpu))^na0jQS(sIU4e>>%iX@)en)6`}m!MEt{9u_;7e8)MBG zC=}|ZZ3Ep@>lJJyp7xo)z$Kpcsh=J~yS3Dadwde^ywRhAhbP>aV6OJh)=98faD`D-`WRZ89q#GQvc{ z3l-SHaagOh))b)Ti=K|8WqoUkfRZVUF5)I`fDWt6wyc`MY_j5_b7@1zE-D_--?O@qHxLxhYJt-I zEF5;I&<^dK6@j!u0&RF;pa?={h-+xMZfj^^m@I9(P2)wjNKj0w0f6D8djD*_39qJJs-j8HAKYQ_=IhvgZSM~3i=?gfCK=~lrLFxS+jeiOy6yL--{IwF z)g0lFCWy;qz^{6M%6h^UIDpB1?BSYh$=0v0+VA}Ks{Sf2fNnsTIxYY^8RU*0gU(Z3 zgn{K2f}Gf_&{@#!bnYQRC!hu`b?VY*eBcGAE&;o#q25=cHZSZ_Cw>J2?Y^jDf`+Pm zW?JP#20&^7W$p0B@bCIA4Fl}(*6`Mv>kX4_qzdfx_HMIXuMl4;%jTc=?x?DQuiP&2 zXl_^Ct`DfOr`5y*`}%4XtE>ZX?iK_t|0?eMb}<+aE^0Dv0PCQFg8z@?(vz($B%=!F z>7AtmZ!QFfNC{(@wvGnqf|Ts;@#>bV7(@^s2XbF1Z;9Sv0L+0PM?f8C;3jn<*y={V z7BVdrZ)iR5zE1Kb|1KuW@bk_qVEiyBe_;?m@x~TuDL3wu@o2*~=@Eyo62G$eb`SZ& zk$Q{=n3X`+lu;F5vEXJv7F+8CgfZd%s^1Ro{t9j}6Em=;aTz=A0LLsdE94({=K{ZJ zuIlX0&aoCC*wyJ0pM|by@%Nm%hN z^Y6k?0?%@>Fo(2A^Y3dma~bE>0QaVBdLkVN#@)QJCVhi8d#*Tt?zS58h?cWX2MQpQ z^9S$c>tdkS2}ATi#JLi3?aIY^y|9Ujs7vSuUQ+VEB6THaa%))fRd=n}zC=!N6`S-0 zUlAzdwwFPpa(01cE2nj7HZ)sLsS0cF^H_1lF2Y6!=ueF_2cW`CTLJufu`vttG2?ZB zE`Uh`@H69XG|Q$>tR(8XapuM`rS0{#va^KUv^e|pPU}NYo3rdvc4&Fm~y)Of&;N@9;jib|^bnBP|rYTIE>3W8R{wSu?C!xBs;(QyE16_CrT>vc*qaBW}nD zw&3cu7I!XkH#c%8c3~Uw8Aonz7v9-?Ms8qk1BVW!#hEtO^c{z@I0ryxW43H;ws}VY z0#NlxLfuZUw;xW1_8!Dfun`X8(AI8}l!BxLTY!o0D{kuk>_7^P5}u zP%c2vY&UmDc8t&XWz~40nW&+qx1bM0eGU3lyC_f-@?Hu$?2)LWht^#xI;J)?QunJT z2g7<>vJaDO**cd_G;V;$CiaAN_DbU>gSUEvcaOyS)0QjotaG`;4kc}K zi$R!qoBJxRsCl%2ILZ!t$_o2m`!&jbae&Twd5 zdD|lGptCv;f{)*jw!hQQ#|2=KH1~BEN#>w9MX7) zmfVZ{xb9_M*1ez?x=shRh^{E?F8^?t7{W*f^gMr|7ZxZKSv=`$W#VgZ85@Oyv-+*O zy3?zb^BX;YK)96!L;6T7i@Kd=`6fr35WR{;8FSJ_Lb zGY*(zxIM_jLH*l51_pr14|;k7!9zGC7K@GvGLl>}o1#!uIt_%HQ>l=0*&4kPArw1~ zj!I`UI<026*CS^#T2t0hQ`Nj?d~;(ivNl5aq&3w@(KvCCz@0ZX zOCV~3S<~hpJvkTc{+zR9%vZGcI;cYz%~S>d$Wxua(OjItWekQ-!$-+fw@@NFi8W66SrL0 z7+^wz10g#WvDb3Oa}Ew`!OVBUzW`p;emUuYD5Z5R!K!YD(~h|e5E%L4qto_m+_!W3 z?Ae>QA)qh4>cX0}s{irg$dw!C+I%^4=vSSdUf>|^bzd6NZs-2&0(S7>)!7AHGw*rN zPR5mI@6AQL&GF?IQSHLz2m+2l2gFhZGKi3(m~5c{F6+#|CFX<32?_{n@Bx29lShFA zPFhWp_QH|0!)lbQ(&bjO!F#BDRt8Zw((rcv$hcf?JYMt0Q(U~9d}HW7Db<%s!>N5 zWz;#YR78}z4F8@SE1x5`TZ;p>*eXjIO-mKBPdwQ}ugdh$N%hab;DdA4dtA69zcR@@ zlZp@qB%_Kp+4Sm5loGfQtVeLY?^R}lgCmY?n!Up|-UubJQ1(zfG{sAc>-Jkn!x&J^FG1hD$d1n@c)58Bjh%blKWw&y_h{N~N0A z=9()V7w1Q1)|tmhWmMN)OaU%NJG4es(%)$WHu`B_Hb8Y{zz}AY @F^1ZSo_F5hh z?%)sO0RO8%XflD!Omhl$V+;(#x?F%$Oq1(&*6wMo)%$KkhxVIWz-Jbm=A9cIe9^;K zOx#<)5jfUy$d831gG`BO(rLI3pjVoFna1~CVJ`nXU(!JjSgg{=ngj1{u1BsC`33D3OXUPj{vxvm3vkMWM}<(Kd=2sdcK4DdvU_iCHLpT zyU!kb?-?IXTk?@j1A+}UI9=23l#WMJ(L?|INzARu+Ii|Jw>sf)EXe!Xx^az5wNTu<;0xdPOt`a9kg$Z; z3;&$>T8OBYI(QfT zxsHGLdmx|u~niMYVU+O+aAw$ zggy)8F^?&<)EsrBLOPlI#B)N;jJbr@ zLQNJ-h=!0Z)`VstlX+1a;jx*Wz-YRt=}e0XvYI3G3m)U50nO11h^S+!rr7xwcBT)M z@MKv;1F^|lda_lk%cM=+NXoA1^CAXtV1+6qf5#x z{8FN>B<4sf`qhvk4y2P{=0?H#(H9O9t!owBH4E8Q8UjX-nd)2;FEEk^q(zqfqUu$_ zLnN67Hn4dF#5?UW8{2?me{q1FYAYjv#`YT_r(W+5aVj($aH& zC2c$H*$$r;cB>*hYgk>|JwDc`tb6PzGFy9FkdCx%>}roDWQrY5Ipb5MrG_OrAXwze zbhfiiEo7a0Q-rF}W-r*T-CmQe`^9a4*}B}ZdX>N96L5hCY~aRW^R0&LEd$a=IrtXi0?p7u zLa&-!PVCXXnADp*_t%;J;ZI7G#bSd@lDzJ&W3wRO>>J^zm2ePO#;SWT`c4bX=XTev z2M(}*Yy0CK`*yY>^)Zi&+*WDkSIMsuvLc7^Qlsqb#r`WEdS@h0@0nPdT?pw zEMIq26Edm;4`WnbruZc+UMiL<5o7wb_oNY)bEfQkTs*Tyf4|l6Jy*-Rp)44#2sUz| zk8Efd1NhJ~+cBbH_|XIl`f;voGQcV~WoE8plp@}Ad8p^%E@LKkk^NwmuMCf%IuNPm zvlt=eq`?gbrHf6Ck|a23;Tb0gH#%-be4xBIZ=oQC?05fvk+kTC#~H5UP+Vzru6JZoC_tsDwPXcCcF z@8c$tCb3Sjg8e<{Mi*P)&JOs%edY>-JG$Trr>K4xE^UDOyZ^g$(ZCFlc9OJV4 zFk>G=UCBbg1?E0?$(Pk_y{~L7H0W9@66=v~6ABJl)$;uML-_n;yp!Y7!(bdDoHQ5}&R+xH*3}c+Jk{ zcGw!uG!N%{->YXl?;3sl7Us_Xtz@JJy_mL;bl!RH%(LoU;eu~C;R)>bxB^@8ohJ3- zsXld-YkM2vew`9R%rcRW-0RNhI+Bvnyeq4m?Ar8ZpWV+J{Pe)cC#7nd!y^#EBQsy(oGHd$0r~8%m3Gt)x}-UK9t+@kL-N)k&ys$ zk81R!FMU8!w)Se^JHF;zZ!zdPj61X5^>kPB?A_mYyubhUv`@JFk8b<43V;6kKYab^ z4}W8Eyp2n(0hB!Ddp(-!xV=~pW1+syIXBP?CDGfy4hk=`vjhl>Gq*FpUr@hZV?8wT zE%wVf*#kiP<3ALE)iW=;wE4WL)xT~=m3xyFBHb6T;6SO-k+`lWV!W4|b!TY~0^g>7rLoAHHEc_9T zQ!N9;!NvPQ$S7(hjzrsjD$z6(+7_PNu!KLXJnnoIUasw%46BbRb0rDoJx}6t%7dMY1j=`Gft0huq&!OYL&~^(2}0_}ucXSZ z^v1#?$Zrfjm&D4x zN~iQpywuCQ9L>EH%)l(ot2E6Ngh{}RCbBHY#eB%vv?|ivN^sP3QzG=A2H#?8aIo&cw{l*$h2qFo4CHqR8CM zjEuZy6wcSo%r#`WrtHkmT+if;%FtX+`D9J`q`!fr&aE6c`g}>$+|T^1&!Kotu@unl zOvB_$7S3yx%5wto{KS*j_WvG%(OpgYAM}0s!>>2=jl+6iconb7_R%=h>BvK=N&kmhW zCH+tUUAPlX(f*9m5Y5FU?W&?^w;+Yl7uC|qiiVG?Q5(I{wrsrL8~|TL%nB98sKd}7 z%@dr289Z4qBxTYnrPC=5%>U>dQ5dw(JJnA<-39FA(k(4e+F^#CTt{>eQyk3=t~s~4 zjHou1ma!~O&OA~}WzP-GRHv-eBt1?$ty56#Q(BZ#Qf*S8LCK+H;0A5}~?Mbj|b zhL&)^8QnpTiwH#>)4Yk$EOk6ggwqR+R90mzPSw=-{8VHG(NZnZ>C96S6;)_`Qa@GB zJ6Qub9aL;B$93xt8b#C|%2m$;yPDcjyNOlv)U;kLuVQ6ScD+<%?bKsUR(Vxc|D;!D zy}xCRR(-WsJN+MY&DLH8SUowCFFi(F1y|1dpEAAH#}kxR9Z(8=*kO%1c&*q}omW%M z*L!8oQH9cdl}?S#SpV`NP;^{VL2biAJ&QxdO>v;r2HnVU?K)*EBE^*hx#i&`pHpQ3%z( zlMU2}ZNoQB*FcOopcPuQEn2m$*E_vRJvCamW!j__G!|`9fxTM;Oxb8CiEbqa{?Xc8 zg;_Bxx#x1&;mkOw4a8)Dt9ET%W1Z8qjohYPTgr`F{}WlcrP9l-T)GW3l1<#NjaqkT zpOsCI&H!A5HIc5%jhDe&iGAIhtVt|OJ8G0%&YfJ`&D@RUOK0s}%aztYWl7zI$$uP< zJ8<354Uu03s{gbw03P&KaX{U`C4j0-{D=~i?vpn#9p#JLx%k+Zhc;?bpq)PTn6jNd>C8UMcx6H-JAuALi{(# zEnoJ%UHJXo)>5{zT3Xj~=9Dd-UTQ0MD-yH5!2R`2#`rPb%T0va{?kLy=jf1UiOD$R5 z56;QfrC*&h+Z(>(AkO0dbl}^kLLQ(cfK20VIBTqGL~fX z1ymv)U?UFOS7p->HdOsJ-MzTuwwwSC*kg?}(>KP!nw?#WH9`bsp$&hFjky=N`6Tg%;y`Rn+D}?q-zc=9Cs_mL_N|K2ck~W%P|?m&RP;E$FocW@&!94xZ-OK?{0@ z!zJ*<3ZsT1SSex9=j)wlnx*D{#+H|CX{p9(sy63GjA=_*Xsxztns&d3hUhg8Ykoch zm$hg_HDQedWQ}$_i?wjsCTa}*cZLaH;o@%_VX@Z7nTP{I$-eSG3YQdiA0QFU| z-sHEQUm5jegT)Mveuf8t8l^_%Z1ra<_GQctY|S2QF8*r_{wl0y=v>}v(k@L^4KT0{ zMu{Hlh*DWPCSjlK6kH1rwnpIzZR*ty&Q7)KyY6ke_U*jp?9C?ZkwR$N^6cSG?*Fb< z?)XePoK0*$HUrl#-FklPAM&fo&Q8O9YJd)I?`CfBrsOOAY(z6|nJ#Vg=IWcy=|jn6 z)^=evm~NKkU+$2A(yHw~X4Ogzj&H_h;O6b$_H7#eZs8Vh_~m6v=xg;3?*eCV@+B!r zHCMN;iL>C_#?IR9P-}(C?ug!Q?A5sj4{-+%Z)H75LlZJECT|r-@C8rr;Z9@3e(%Fh z&$B@0WR8IFgpV1+Z?5e{K*{YuF6sa7Z5a-5BL{E-@9!iRaalC)%)84GXYwd#9}rmM}PE2Kk{iea`=SwSC@57|Kb#fxLE)8 zI@k7RM`QeoaA1#a2yG~~KwVNdkWZ;A@*^$|KXmS<-E0SUd>8jx*Ig2I^7D>!UEgj;TcaYEbEzSg7 z=XKK-`EVEcdWD{PuV%TboYNsE4Y<`{iTD|7$EAkYiBfrKxeUnuSPyS`5E^Z$NhF3!-)(#_M) z)zQ?L+G}jsR^Hp(;p5`vW}E`>+kL9@$+)>_W$(s^YS$J^Z`jR7Ge*92rfSzmuhE2V?^rak-gHsWT4w&coBAU$4G> zarer>S0YXSU(#GluRG1Jzq)MgVdW;^ZUpvbV1Wo8sGxic=5yYI?0LtagcDN8+kl?g z^;{_v^ylG+uw_R~g(O;7;)y0w1W#Xsu;^lW5XKlGd^6Tal&N4(jQrwuorxrKDn7>Zy>r@&96rH>sNEtFRJ>YOJ79Isi;Sv%f1DC2ja4 zsJx06I>&_Ghou0$Jv4%`&TPwrpy)W2Cf(D{g4x z(yC}Z(2cS|wd{KMD@D0&%P6q)%FC_2_};7Ug#j*U1zz_xHCVfZ4LopRqVB73zQM+q zDvop_TdrlrN}MdT3uBzzKmil%F=6l4%kan&C3KLm`Xo^wUsBE%neiS55WR^D?P*yk8K6^wMAl zlQ7m~TfHpLXa}9AZ)yKrYuRZ4Y(WM|hi(7&ElraQ_uhQF-S^*SGuLvQU z>$Ly|?l|D&il+ADZBx!R(3Sg*a>j-$es|vBO|7@+q(5#tK<91x$bh}E_CL%YnYc`v3m^q=)NZ(yz-ha@4RDMMI*g#T!W$O;l+D8d+p3W z@BQxJ|IYhnk&hpK_eyLCOZJP`T>AU)$1nf)^rK&YIIJf*{r=RW0nPQWF&O)Nu|4}4 z@P5rZU+*razymt4Y`TKo0Ecuu=*|Y40P`&3Y0DHek+yoO+ zKn}W4gXU`>0~ff#4zA6HQCols>9+qu5LNJk$OEDfUx>rd5s_5++nNZMI6@^gkyEcr zAua^C3l;W)io6>=xQH)!RBOK?rvLrU~ zj!f)VG`w^Sq%Z`Fp0eUY+($<_8Zw66LSzlECc{M@vI~4{1@YL(LPEZgi`vv5-~*Ly(05ad#W+v@eN>WBAFIk=0_#>w%{D?ohMbNN?W?q3o`VfNQEd;Wja$y;*%*@#OOt@<58j(RYDUp zDM8zr(u~N{s$fN(GdMvMFBQb5I8BBy@3&SjytJ)agUnFJ@zbEbbFN{9D_iv%%7!ZS zuSvxtL>X|G9-Xy{Q;q6Ir`pZMO0|u(%XP);O~qmT`jB z@Yd*l3BtW<+Jp~$;qF>k!Ke#wLEnI3#|^p3O}_AmMNH)>@6X3asi{R(tX~xG7r3)T zER1tZX25}0$;@Y>cJXg-F=kF1yE(5OaB`V!>1GE}ueSEI zs<9F0!wEY?m45aRqdo0a<7U?ON;ZM@acXDLTH9ENwytx{?uUl@*YFOuq!GK2VmtfZ z@)Ea=HGS-UcUsz61L3vK=Wpc=lb!_SdwKPUX+3qLQnyWQ@N zuY29^_O<`WBQ5X7+S}hK??}XJ%B>#mGlqa^IdD-&nWQ%2&?pmwVlGGG989u$<*N6CCHW zFgVg%?Q@Qc9YI51yBhkk>aAEIr-hsaMy?^-WfN%OWnM8KKOda07zWUb}F9sTC z`nAX=ch)D5VZUFTlw>IUZt)xMQIA^f|F*o_M;~;6%iTdm=lXYWylw6hjvH}5z32l^ zd%>5o=Tj)W%@NQ0-YTB*z)#WRTizJfGoGFlpZ(r-^6hA8-XMo}^i6f{;f(A3?aDVj z!?pj<_R@>)r#HEYT@5UIi-RThxA#8o`QH8R{~hzSkH770yMC>V_4(0HtOk5f{own* zJNkA~f_8ias31I-eNIGwsmJfEIXu6?lOWm4qOeT^pA};Wve_CW7FWQ%gvJebs|c$bC%cb}5L0VkmelScYcU zU=4Uyhc|;dxPxi1b>oM8ZFq25ScmU7hAt>wUsH91VT85jgID(Q8et{-_IQWXR=t;LYh=Ns%#3*f+c#M>YjF`BLX4s6F zn2M@c5E;OK*y4%DcV?lui=?=XmDhHUNQ}z3L{OlLo0xW;=!@VejOqA`j<}9js3N+^ zi{ciE-FSWIxP(b%kKb5cKNpVvn2hl!Jmh#*Y-o*}c7r!&dN`5qhSGNyr2S zFcn(4m5uXP85xxA*oaE#l4L2A%6NwTql!gYjX24ZbI^fXH-K$fCbUzRc3BK}sg-Uy zl6(o4O?i^HmxpWFl~n^{`17HA{nU`D{5e9IA ze_4{3XO^N_mLbH9)?k`3nVMw?jXpM*M#7D4DHk+`kn9ATN|2SrfSCuN0K1u)nrWGu z*^x)*jZC?iU}%QJ zsen<~kQ_&vqnVz%=8WvA4AG^WvALIbK%J`BVcKZ`3g7?{(4F78nOT{a&S0DM*qoMU zf-PuI@Clutsgg~ng5#;6&MBQzfDQC1m617?k650>Ig5t&ZFk9?2Y{QKIig%CleYJj z5L%$>xuWP9lc)Kjs@a}0dVi6WpaLp~a6zAONQ&E|oBc_mK-!r->606}phlX0Em|AS z=Os3pqE33G1p1^*SrLw@qv3~(1Nxx*Q<-*InLzrbTnVL3D4gq{vSR;8vQdh#9!s;8=$;NsnCE&` z6Wfcn`Lh2iq#C=i2Pm?II*AMhV#uJW3|g}R3y~%Jg9E#KmXNaa8LlR~uN^x|7Ynqn z>a(S4yRY+AxU!3(h&j3e3A*Jfq^mo#liRzSny_VCnwQ%KecQHd zdm_Zjxm|a>&w{b!TCR0#w}IQKy9>3FP`tJax!M1_z3|EeZ);G@#=QyRW+V2wxO=D7 z+q$hAu<7f&X*#OITLqcxru1tCoC|RZ$zjinspAT(eF=QxtG(*_v{Qh(lH0xroSyL- zxdI$?vOBw57@fR(Yx~Q;bDO|{h{5hlw!jOc!TZ6{Fu%+@!tjHy6idQ~(yaZPv1Cf5 z(b~Yas=@1ul{FK z02_?N2VBQyY^EUW!A{)7Xj!~bd~9n>#U}rCpl?#(Y`x-$ zy&N0KnEb|GJir@wuira>U#!V#tQ%jN$QY};Purk(oX3;Q%6%-8LPyHmio###$vO#n zC|ALYOt+0}yrKNdo*B0+Xu#~-%ErvfDfz$`-!@wNG_pGGKT)0S( z&x?6$SA5QfvbxQ@%Bg(J2yLJ!N*><4&fr`rIZ2(Z8puK?&?Cx-Sm)36tiX~%%nAL` z@a%N7=q7^u1)a_ZZOZ2h(#`)&%`P3%?v>4aT+=jd(-hiu4gJ47c8|G=06(p{ zA9=r1SCU1Y)8eQPmQ1gi?9{b8&Fw6}R$YO%{JmHmV)*>kC>YSV z`>83N#xf1nX^qWFveOi8yj>lq`RdCljMRkL!n3;3qg%b%V8Cl_sA_%CTm7KNRnjIc z*kTRSDt*>XXVZ_yeXon%#Nh}Y!m;i*ROrBeqG$e{nucf)B9|-I&9XIoYt@^-M$$BsQlgD z9e7RL%9m~2^ljODoY_gc(LDXo^y0}9jlYLG+U1jR?p08ZrOl;p) zPUTaM-w+DarCpdR*y0WHy}LNqIi96q5uVHXpBDU|;VHbMyx?&D&yO4BRVLfuo#au= z<2;V%Q(ENYjo29O-A@0G=BVx1rA+8f{^WEX)5jZTJ$zRQ_D>$zK(o&M_TIp3CD23a$9ht$YF6a^-()O&~qyFI) z?q;lh*nkf1Rz~e)E$!;Q?sJUpbgbvuj_Z2-=&TJ;U~VCkZk-2Il+OO_?C$UX9`MkN z?i1MBWohok+@hy0@9_@RL_Y3Onea{*>*+4=7Ju;=53AI^=)mslQ~vQDAM#f|@*;nl zD(-#Rd$y4d>0STMm-+sv_8^?jUhx@U^EQ9;xI6PrJK+tl#0%f^Us%=Oj_~mA&VHV< z{hXe$4)8eN^iKcuhK}g49p4P^P?T-j$-VMg|Jlq9^YxPGG#~8*j>TkO_G54Mn$GlT zZ}ZRI_Gl0FK}zViA?Gg1=O_O2RS)z^Owva`g-PFWgbw$DKlp@S_(V)Qo<7+fzxa$# z@+IH+B%hiM&BJepop2sHvg&;h|L3oG>4u-_XTSELulA(>`KBN38=v=8kN2pa#4jE0 zuy6N7k0hIq^z0n?r=R<}zx%uov{JA4@B8?>?!olF^~RqstUl&mKcSxg_M4u?&>#JU zF8$A6uA~3&{GyNj*RTC$52U4k=-eOv+n@d7AO5{>{izT9KF_qOul{xX@O|GZWrFE% zPyO}}{nU>56d(PY4g>+=kXSSxkxAu}*>pakQR$RgwO+AV?bc}+e7ywm*hMy%(Pwp< zy>`FbaQU1buix?SFOl6B`vC(5{|3BRaqet5EL9@4G#8fZ*g;Phjx2;d3AMvcdhn@kByU&mzkBDpP{3rxu~71 zud%bWx4pvV0>8t-$HmLZ&&|`(z4*}G-QMHj8P(_G@9XjH_w_CN{r?$ir7D7>S+u$d zv$p?@VK!e28%8_=QDQ}l7cpimwkKb}jvnpg3Hh-e0b@cqky5#mWlNWyUdEI;6DCcY zHE~9IHLF%ag%76%g$Oj-P@&Dh3`n?CX;Y#{jarim^%z20MyY0vs&y+fjbFiHOW2Vl z$g*e6l4CanZQHbN;l7m{m+oA(P1imRyZ0JUT%&g&%uNdCsw?;3CDaQM?#h? za&lxnMF3_7NG~R5&zfO=-h9(0C`g=1+r)cP3TrK0qrjG3^l5FRxd7Ydx*OV0tr2H4 z7C!td-nz%_BKN#ji1Osm`TcV)8hZ8TOo?Z=csPk}?ND5I|3034d4_uoOqXYwy?XzC ztW&0zAD{kw`}gZpGM#gOe*B&AVlJcLek+i`1A9x*wgnYz5J3kSjF2}2Ck)R*^Ds=z zIO{lc@2++{46(xzMJ(|h>@rMIMZPXv(Zv^GB#)!^Dl@}I8Y#<>J@`f=g}NVA08z76^hV0F#y!`&48;F5=`YYE?GuzxBoEaujh>vclQ=oW(=tvhC#{sy zOD)Y*(xV*hl(=rtbWgxg(W}NMWghk_0jVy# z>YoeVIqlY}RT%B4k1<P<=%kS@)#kbR#x`Aa;U3pn!Fyg?LQa1BPFyUjuWmeL z)|iTXHOVQT-15pXzg+XoIp5s#&H+|j^s}oh-SpE@PhE8(2xndO+=$|N*}$V6-FAfm zgZ(dGZtvZ9zgG`l_~BiLHC-vYYTco?5&zx!=b?|@%cKm$PB3vZS19+>bthh-*pixZ zqS6yqo_0&6PaldP001HR1O-q4FaRt7000001ONg62LHf;a5xMUkH#c&sXQ{DO#?GY zty-_xtai)odcWXsxG3f-91-;S+!S_^WiYgR$k^}riruo=_y2x*3I~LNfq;mIiiwPi zj*XCyl97~?mX(;7ixUn8Qgc;OaigTArl+WKgJ*hooN1nLsk5}TwqtF%pifIqy1Ko= zyr05H#J9-F%E_{(M67CQZULfKPRGo}bgzVjo}LPRe~9FEtKs9B>UQPi?eFaH?(y{V z_V@Mp`uqI(@PpX4Oj|~v7G6o0wYNVDdUV>WZ;0qKmYv7Oq|AciR7fghB}nP!>Ur6L6@s#sczEVg)> zj4IA(UwbLe#w3T7tg_Zh>#ertitDbr_KK^C9GG^bmc){2?6EikiR`k>l1UnhnrYDFXHG(CqG(hO zE8};~>PY9D;C?IaDwcISzn&!O zUZLv=xSRoy(g%>7`MGJGR0Q+nE}hzSdS;`qZiwIpufW$rgdl%q$b)GL)yjw{r>yeI zEVs-cu)?;eZLu>KYp$_9-mLS^GxCVpi5DYe27df;S6cyMN5evKJ)oIeH(DUCocro|=b(Qsdg!5xHsy}V z`e^gjthesEkEAI{8k0mnY5)3^NyjMomF7}_Rk&3LKlSiZUtKlVmJ{D@*}`8A?(pNm zJ=IG3qAfk$*yBy8-Psr7Xqagn*Xe!pJ$F5rl!`k&-_9f7b@J}V|9+zix@`ab_~-v& z3r+eQGtD)t+3Sjwv&+nEfXMn^mZHTO9u19XqG=$sl4h^eWl00X%i#Rb)V}v^5Dh## zgZxG~Cyz7%PRvW8*%p>5>wN@bb`sod{Inexp09gy(_VXA@w*(p=X>^1oWsCFDNC7z zHW6bEryg}MBnr`a8jK>OyL(}`u+}^&+W)bRRBPI%KFB6Ltq&y=5gPMjzRER=C zrtoPveBmGS^_sUS(}&^oAtk@TJ$05xZ%mA*M-bOXKb|u@%Q0LDA@)vIBw_-IRHZXJ zcsw)^g>hZvqCykO(1k9v19V}gE`KSc{~eH`XABx>kftOEnvSCuB}5!cB2m@(PN2|C z=|EX2O_u7BrT@>&<_XCLygS;oUnt?HPR$8PQLZzc*lTC@3^u+c#c0t6sb6AEUB^^P zSAw>5nmS|YTnWTieX4J!;|i=dM;R`j)(~Qk+zt&B7s>1W)TiHh(BvFjDIjXT5M)J|cE{m$G#(9W|Y$ zN*8)+%m0l71eVn1-4(g4>=-P$O5gh8?*psq05O3kv;M}_xMm!UU0@4f+KO>Svoq#K zk?B^?c4vV-i<+cxS~r-cX|E~GYr-(xSHG_CYt#6a`3BqBvVCf#-Era!4@=qXYBz4( zb*JVqTht(0Ota=aZ=Ftjr{x zwf`)NIx~JbNT2zDfCro+EE8DEuTCI~NTcA~y7kl&h?^%aTBH_Vc5ECDww(Jq2n`Ec z*m+LZ*Q&86!3ws-;(N4X*+!U}{8`Pu&bFX;m>zd}WWDpzZe?-&Fn=W*y}!BCkAtj1 zI+bh!Nq)hlJuIqtE7{(h>%gZ^BpO=>jWc7_vH^`oMq%>$M#?&{lBA5J%t*_PxMnq# ztQW(N95=g&ws7S%9zT)3^w`8cH@Z49nP;QB-A&CNc4P=i6bJg{+x`uq0Vbabix+%) zOW!0#p279plwz4QEqpSavdPg~VnpAx(l@lE+kNKCCkl9ho5k>_r**(Dq8-3_iT^-a ztNP$&ZZ$DY#NgbqD2G!w>Df)bZVe6_n9^oGHMw^e>*H|;#q3xw<`&qHp1vJbD zA6bE|6Jds?ad?@C%^XXYY9q0Vx6gb%m?Ctc8!)PPCxFmUDied z3@(Eu0fSS3jT|^VMaQ0Hoy|_`Whd6M7lrz6@|@jtfBRg4@^Rr~xb2$NCT>iA@^h1z zdg*1c^cgF7!dqYFc?fs*zG0o>G204K zQZt;a{RMT@rF7d8#Tuid26o00BmZVQxu<(cLU$ZuT#5GXCXME*1cmKv$cbFqT z-ryRQF+SecfrsN0Z3BWo5qN$_H`7LW*&$haaDs^!Xl~|sj~870aej&iY^DWESR^rV zh6nQndOrAhFT;QK_kRM1I=3Z+nt@vh2Rl3xMnuRgnbBnt=PiEp4V>}-m@yPj7-0@* zckUH|$F?xBP<#>idv~XN*S2?f=3_j!Y}5CKhDBY~7jxJ5PT7Y;+}C~1W^;36hQWg{ zfT&!WSA!l1HR+dhAwnREAP$Hae~w5Y*RoMHV}ELAdQ?V){?jC6XKQR^B5{RFw8d-x zi~lT?6Gs7z#8GY4 z(?Z&IjC80Hr!-lV)q<5oi01ceIyhBO^c}nNj5pVT7S?%O6jhAK6{-S+KRAvP;$)_1 ziL$j(SokBWR$CfHI}Vg!)>3;q0#-$ca0E0vR`-M`woT+Gi$mdR@L^V2h>O4{HDEYw zd-Dypc#F=Jd>RNUoyCU1B`-V%hZvcL$e4_EcuA;)ht61tgBX6Fgo}VEhSD}%B8fs5 zHe@aXj*l3VE;B9hXmEh>W^+ zc+0qg&8U4HnKs~qNaa>T35b>=6knC5P%tTugh`H|15r+*lmb^}nGtov;xj!GOr!>C zMR}BSButwjif4BdRrp-N2P29kWF}cq2{~tCSZu~Ni)j))fKwG|=!+ywRXi64Nu6))L{GTEFT(g9D>O8^vLqd|HGB$)v; ziJ!rGmg$6;shHc+TNTzYU0I%{$#GdpS7KR}=DC_zjj-rc;)py$HeMhku z#2AbvH;n(8Sd8Uz#|W3(HJ2H{f_4dw49c70N0GlXlK;|XjC|;qEy<0j1yRgdn1;!r ztKx4r$vULhj&(6c7gU`g8g?lvTcg39#S%$*@|VV;nxa)&qDfqrL!-lXIU%H@q{&iv z=R}0HW5Q{M6<3?IIXDK&kxXhIcgUL|X`H{=heg_RFbJE6SW6Iv0?yf`ETeBxR*Crc zZzL)Kw-jp@Rhc?NkH18KQ`ler;-2Qoo>OU~TKS4|N~i7Fl6qs5Wzu1KDwV(`m9fAW zyeL@BmO>{vV!F|v0P3GOA&haDW^Y-bC@3fhs-Qksp>Y|URXUAjS)45=rBm7^an^%c zWMn8Xm>$}y9XczHiH;)rYF4)+OQ>Z{*qy7hS^tYlIbisBz~wcgRhkyaR|1$NGU||o zwQ)iEn!vVaa`>sxiljQ_r~*oYONyH?r<>n3d5D*wn@UQ;)tAEATsuPh%vT$gi+4^#sdZ~!Apwh^x-a4E! zIISePt{MBTp<`qlFs~@k0ITY%K>Mn-(rIC;tEJX7xf)vr+d4LxYiMRo?dqeR39%{# z4aC|pVA-Pw48k~uNgqE?{}|+iy`sqqGGzQRHsX%A$xA>r$&pDkDI2ZFOoPdv4(o6f%Rx-sX`w+poQWSYFnu&YpKepf+4vq3Ag(kme!vjIe?rYwpjJ|nruIwMOP zxksCw#XDZ`WVN%HNO!80=6kixVI8cqzF`QV%!-C!%RGD{q(wTjJGNr`DV}WGw%eM0 z3rTbERI?*Vr80{ip=o3GagBTXeE-VZGUG6>KFg&VL%of&p@q9uE$~%Edl^`!xB<(6 zyS2Dj_`U1cy=bMc-piDn6mk;Fo_6ZFoU5KOJh5l#7m7_xv?BOrPunpoVkbk z3nq`6#7g?VEK9ot>LtPZPBpe>S6X5S<1rZ;S{dN8AydYXzg%VS*$fowH@@wpgX|jqO}!hNFK&l zUkgbgs~i+Mww+{=2n-M>%ftZewgL>L;JQcTO1I&fa_FIb_-TaoZ^~b#_e3aPCAQXC%Ydc1k5Nw1Q@XDm5Jhdk_nP34Lv6~M;Po3l(NG-?EUQMV%yC&E<5rYxGT zymU0X48B@oCz++OQu~mhB3yk$JnTDX7N^66oIE%OPxR}^Myi&M>a2>)sMr%7=@pHb z9J4v?q^M-b{Zey;(0D3Y%0DfxD*+$vOVlF$yDId8e(XLK30j#a$nx>7vP@r*cuWml ziv6W^v6`K$xT0j`lmC$^fG6U5!aU4)T+GNUsNZ+PVvAHqr^q=6wea=K(2N{RYcj7a zr~mlFK>SGCJcuA@XUm0`Htm~%3FCZx`&4)Bqr#WeRsN^vJHtZv zJ&!TsWKJ&87t@j_zjVIcY7(yh3*H?Wv83g0qLi2Hlg?d{-}6iV>i-D2aO zrEPhx+{|vS2O6y3`NKa>2A)j#&@^*pw{&1C+>;6CrgluQkvnVpXgY@k4$F<<)EwN> zMT?*+#s5b&&9O+$ye_)0!&iZ-aj)Ezxu9M~OvD;t=!c$w^^_e1+TxE5?K~aX;%r$l zO3(Rh*_CW_fRZOj-Q^*z?VhcT|M;U+(``g<&7*?QOa3{u{B?|5lLbr1M;SZHk{SAF z$FmJgAPjpRZ~-``xm&Bui~Kb0tHAe+L?FKKI*P*s^yc!(r^6oSKn}r>jNPHV>^?Vq zzWd#N9!}z|XoWT~jJL4c%-M5G)QTQHlU3J=r_?AP)J+{eF*vMMUA0JV)tatV8QdgJ z9_6K%It#b&P{`{2opt?YI|4Z6bNsjtG-|5}>jeMc=t_;_JLp&66o(}787{GF?%aLd z;s5f9&E%}Q;%c(W&YBU^>}sE8&o1p9@5JDMyLH(^Q{m{^9qsC!=c1Yr`C({z2)BCZ zPD{Pd!Yi$N#CaSuNa=3SvE1(Lek))1rZ%F>8Qn7;pl|w4+aQH?EqY8%_V0O|~_dy=Y`y4gw*Ybw;^0I$7o6@^xkACGI)yW1Qm44~NDwX-Y^I63|xfMZYYJ}8s z`KJE?Zp`%jdh|}->J7A=qF>;thp?m}x(yBl4uxV;G**sDq>yF4e1@$dEX z`9C~30Dkz;p~VI^Z7yg~sKJ9j4;n5YypWKh27>=6WYk!PV@Ho1Cqjgn5l+RBBROdJ zxRPZ{mvlC|k*V=hmLyKhtO>KTk2o%efNBBdrHhzPKF8?HnPjOEk}shqHUH9dCsnHx zqzEEfX%)9iLBV#BmhIfNV!DhDrABP+Ft=ynzLm>%lY&}zk;FYaAzZ97UH@1E+-a^? z!*T60Q{0!Vxxw4^c58&Roa4%7G8oi$xe4XJM+v$UJ(_fB)2C6VR=t{aX@?)+b(jz_ z;Xi@wI&4^|u;GRc95SeT7*Zxw<8vZP9EqIvM2*9bN0;uA!gA*`LoU^G^Nv#QdHe+9 z`q^yVGG&;ONvJ_59-YOfcf0__ef{=mq6wN5R_OhTKA5M~0x8Y~RL$4()oL!oXVf-z?N!-)?M85As$$N!8fP_h}t@NKin z%Hk}?2}0v%pC5e;638Ki^s&cwbgPab*ka4=pN7=wKuUIoTQ0sUTe^-Rm9TuO%a^Xp z$htTvsEI@ruR6*B`_j9SKOEia>AW#C%C4&Kn)32XIWB1C&NQbAG^;?i5=O>U!g}*i zZ}ejE!bcl*RMJQ#T@##VD4m5dz_4=B(@9}0Fc)9M@Z=dwqmgXIP(kdE#yNA;u^e#7 zYb;hKY9%kJ)_CpJ*I$7h@*#jcNJpQ6=wZhnEB9=6j)ee94k|slbheI|N>YxuF;U$P zzwYXrlP^4B^-D#|un||P3C3KH&HIXg&)WImeb>DRoY1cjO8*R{ah3ff37D%zK`l%~ zND=-p;e}JN^il^Nj0MHIDrohwpfY9EV)0x(F|UTxwNPV3#Tr?gkkMV$99mVDHAh7@ z>ygMJXQtWansLSrpd)j32O~Y@wRSoRh~+@qX+;8BXqYx~ZrhZsOUpv;0A1I+Zm~)c zWOCq4YbRzUu(Qppm2URWK*^n0-$GsWH{U`PbzAF<$C?Y_g7I!tZ%`e+l;^g% z@Dqm6bhe|^izm54-tl0lE~P2yd1-S>>r^`k3X=SeA#SEP*@9S_DEf zIyugfju3PW1uHlb=l$$Ub<*Hd^vDnjdT?4GB>$7_<`t4b>alkfLqy-aSUvhsV7t6>t8HfC`Gxp#P&f-qfqQ9-zc<8B?j&Q-^vNhs`yNi)l!)R z6P%$M2S!Xl6J@C~ONVg8Pgr?IN0~!W<^QhfvYi;Kjzu-3!g5DhyDTrQTefD36`Wbr=hb%Uj2VdBR6cfU<^$Pd*F zX8&ZWSnx$Nnr+0~sk8>fp5m%>-}L5dT^n1m!tP1<3~NdjdYpwi535caDTszVB3m0)xi>^8X?&iZ^lnx-U>)b;a=5R1wac%z%SRK=fqBcLYwb zG_#3R2~QZdqX{Qd^JrVL<~F<7kxxi^>r|-W^Q&CFojZS9R_|(IeUS_vC7pZSv|bH8 z)D;t5I~*0d?)ZAx3vX)0$V$Ne7^Q?$Z;KL}tXabc}LMG*w&s4|;KGS_o4U$eRoBG;37h*vgsC5p)O#b*D#Zu`b7ljHjMM z3@zc7ID{A6(r{>J)_uN_cn!X5TVKpww_djoHcp77>15aKHoCiN^C8<(W|$6CisVP&{oB3xZO>tvnY+S9hQs}8Z2Wnw~0CwEZGV{dQ5 zVv|O?DKXx0bBX6QcDk!xeQwF5?_J3r4|$?z2FjANXT|WnY1ClhE8JQcpzu1G;tq;0 zFHK41D)WUbl|9#%-<#P_`8U|SHinz!_v8GQR>2`NV8zNj*vhor(b}Bz&Fh@=A9diu z58;i^uiEXXT6H%+_jzwCIPMDs^X!=6_>b+h3mh$#ut$owWm8&#Cl=Ahj zHT3nzp&Y?QE)23CJk4f@5rL7{m15#}ZOv~!S=x3t39p(vr@2lfw0(_3NLbrI*Ore? z_cwe<+%%+@8^jS2`y*SdYk_Ay;9kc%*k{Ore!reNyq_^s2LABAa*^N(U%sfEk7j}K zOFYUK^7QFmaqSnT#Uh1N3Tw@Alz$5HuRb$eWJORWyCoqTX)d>Q1mIkAUpX*pBS(knEy7j@9xm2jef+X3b+R zMey)tq8#n<_$#yyO#K4tcnGglJfSQe??#{y|JLZu@Nb2NirP3Y3$Ja0{x9@?E|7R@ z=)7cu&`lsNthmrHX^O4^)$IYP$^hR;0b_69dg2(Y?&`YEYdY``lTe_j2v*<&p|mbfoJ9{Y&FfMy zm0r-`j%)?nv5KEtX$@g!D=xUJHxCD2n)4vBPFZ_@CfLj#b^|;w*t}9mgFO!1oZ$70GF{1Niu}S z&BMqI=;|;dVh#-|C-*!d(zp>EDPswV2*A|H0YFGegwI~ukR=R})4UE4z>CzjsK6p| zDj#vY*ro=}$FL&qd)zJ)y)ud(i`95xO#m;W{0kJ5tA4ypAUP!=c}nrBvdA>6i9)UY z9RJcp4yYn6k{fA37yWVlBKhL?WOp+-Q9O zvq=B4XSgsKjfMf|Z~!Udwls_kv#Mx>aU}K3C~=Dbr>c*ZuIC!UALJn?{v>vsQC}A%omO$$oRJ)+-8@-W-p7NCrukAn$J8u)r46J;x(hES8H+$2) zDvk$73Qc_Kr^bQ&c2Z3ZPi|Pz=p?AG0XtWNFJHL=bq)eb8ARJ0y0Lz`(s8IH-ihbs@&21RsH!}9xRArX$? z6UUGJCWgyU=1?*f6?IA#+fpt^G5%Ji2*&|4&~h*Da#`b5RgH9Ek2D@W@=2XEt5ol{ zQYZ8@lUHj4!|v%-Uq{0%^TTq?Y4)v4k@3z5FdD;T&*UvYQ_;RaQ#qlP1{!RwSxkZ%?f=O=-*Zdk)=GB= zK3(Tmrz&G9@G%4840E;^=N4jlXHwrZC!2M$))hgv%S+bKJ+xFh8tP?Hu)!p>Tlv%o zbM*T5#R_cmXvr0H(bW>m^>Haq4G@Mv9gByU@GJ%L)tb|Ib{2*LvL^*cB0qPEb`*bl z^l6vyVDA#faIs;L*N!{@7vsTIM~s|kFAX#H42Q8MS8{hsFT>(fR|~gxh_NIqGu#wO z_5NsXFK8kbH&Uxk1DR{12DMOQFn$@N$SetFoI)Yi1zWQKRLHVc^~ZF><)>;XK9_5W(M4h6R%s>A|q_*OFvhZ&O$x2kfzcZMqPc|~?U>r+EQUMe}Wvt9pw{7k;P!{4|2(b{QOJ&)j2~A@i=6gG_1G>6 zQZ9-RmWFgwd0BV2wwlc)+7f3wbl-MHAK2})s5}8$%G$0(-4AlXW5si{_$0(Cg zdUx1M*|w>e>Z*)bh`$Y#$JDr*l%DmMoo01Q25Jj%uW_BV@Jdvft%>)dR|D9`+?oX) z%Q174ClT*aTyrg!eF5>xm~^)?u#j0=*Z7U`17VWWM46yP>6k^ltScH%r4$m6Gc|%w z(VERmrid+bQuv>CO)m!)ZIGbC8i)&(nw#W#aN`+n?@Td&wVn}LSnpYoFq!EF)95%> zBq8>bZI&|UxvCHI^|XZ!xfu$uk$uq)mbVY1Hvf%TZuUxbIZZhj9DVkqV^mb26>Nbi znFaN{I;2C{^gvxrX(5=HCgxdB&UT?GgY^;~?|3OUP$A(5v!%9n`_(w9O!90kw2K;? zr>Tw4w%VTLv{gg3eWhf57~N9R-SRW(TJmDIH>){z8sT=6pXN)c+PHSZS7lN;2Dey8 z*4tiki0v~PqvfrixQUr^c6G3(s<;T~dO_{_t|648lM9s;xI+C}`Z_mpN}9Y(_jDyu zuvdCv+W1A2$dnuyMt+KRf$a~{Oq>8WK*+zJ6L>yQj+;n2b<~f)*_wppV)dZJp`7&MO?xf_UkCJDheEajn`t@u;bMT(@mw#Apk)m-I5- zNk3=D!(;;KBHb{NTX$q!CT$pVf2Xr}lC#wQYp8XyBB(M9d1B)RF$ev zX#J(U+w`QVazob}eb&cyzYo4OCN2WUf= zj4tQEI!K_ ze|&`codS`@V#T-9hg%xiN#F&(hSdLWeSL7$%Xw^#ljg%^&jN5Ega)r&9csP1usQzx zT95^Cot6*IL(!PzWx3ZMrM+ER*hfx;9fqdrxW4J~8FifmAz_nqRN`UbT7ed|b{I%gZ+eSQayR?SIRNjv} zV&gsPmVdeB9Wq^~B-Yb#$I2BlCVx`fm#gNZkdV!JRyco?k{_Urqv8dsV64L$oB_bjYHL9&GMr6ED zR$5wqcxZlba(H@Zczm?9d}y4ao?dunYHpHtl!9tp(4KOnhK`a}mR?eJw2E?e662!c z9(&T(UN}EPLsLvsS64qXOFv(0M@xKDt6EE-K0lnJr>Bd9KbVh$zr)AV%gr7S9ukZgA=`y^5}awUsBR<2j_0ZjgxE088zliwq8y-d zCCh##Uve=pKqk$ZHeHg;Q!^&do+fD?xoK0SOqN82?nHVssL!TOpF)i)wacD9M5g)_ zl7S>nm|e9*eJS?N%&-4Qk2oy&upyYVBNaB`S*Hskv~s>eYV&U^+<_7w<;j~cZ^4{^ zc=bvQOH(LU%Wk=%h^?{XVUESsO4UpkCtjI$Ax2zGm?&JEKS{o)x}Lsn_b{)tTW^57kUc}F0K(QUDia~CbN9C(3+_zN(hz0e72vmj$yYPB$y zjAgmlqFFYsfad>-irrWee#gWYc2@tF>VZqk93O6Jvc6i^MCb*O&o zCAh?*kCvyDp+iBsl%n?em#L`(y!P86|#L0gNqlFiB!b_K1;>PG)lovzz^vS+tOm(h6vbwUDEd znql-9v{PP0?vv*}8K#$dYImD1W|m7PI<|>f@0hINd4W{QIq_GWbUGKXued1ejI=5AA=&BimI9fqMGu53Lt10qZOY@ zvC9NIh_NCD14@`g1!{=U%v1!ckgmjzm}1b8oGI{HGz$?LDH%)<3ns^$@{zU1joaff z)jE5NkBxPr35uiP`V#%}n9U!l@Pu>r%I0LLlVYSyfBAR3A!AHgz9u1eCO!o^Ot+6I+)f^j$Cf-@t zz>WWQ)?1eYnT}&4#-%w}kCQIGcJr1uH;k)&kKAHQ z>M8K?7*f3RFN5jnecU;~0ok%T)N$x?C)tmO43{zr9xnDR(fSMMZmm$0u&4S6xRUdqXyuvvk97y~f z^k^uB3UokfSK$dc(x#2Axu!?gsu41dMI;(^Ep3m14cbyOuP|v3O6)5i1?ZK(>?lls z*vZTr<8`k);*T{C*caetsJLN~PK18^BOwWrKm~dPa)ZlZ?vk=A37#%R~&Q}?meu}+Yp#0u;_CA-6+kc2B7SyOJ}ogdB|l9ObU_^gb`F{04q)@Yu%s))jT1ZVT{5Qq0kZA)|MksXoDO_ z!c+d*XSXK31Ceojj5^qe$9Jx4HymsVzi`+lGa%t}1C+`>Ly0)v-BXEN|UJ~ zmlBSVwgik&X=*dS!V-wZDq>KD+1jA`Mvidlp>tdqL&3H|C0yNYYq0h%W5}pBbhDP( zB!kW2s_zq0>MB{`na8_&DV}G&-*oNTolq{No;V|6<#KCKhko~<|7)P+eg)lj8qi=o z;~z~>D-)Dtvb(kg9((0`()>&+u=*unejydv$!7MWmNkG0p(bFEdUXGl3RdNOFZf@| z&Jv(*iRoxzr6APi@`WtY8DEZQ#6`45r!Nhrb`+aF*+?^*&@v9#!o~?Dv4Td+C8K3J zvD7ZLk(%J_ruo)aKQq?jywXiy9i98!?aanzHSA*=*g9gTxEHS7eK1Q!Lf1eEd0}}4 zAcIj0NrmBoo)G>Cyrc?8n4clK&J{Z7qre{Y(Wy&Gkl%%thVP+RKD!PCc zU~MVwE!XP23^VN@#4B7*J?v;Hm^Eo_xCSbIT0w3+z;fqcE*p8=J{l{Rn;baJaD7XQ zc+_SY@ykdV#k$pKUW6S+wl1yjTHPWaIoH=k3qF&u*FX<0L)rhWPeCnJz%A6by@?>N z;tEu2bzUI72GgKFF?>1BT5_a?wG?70>g~rGJI-^~Y=C)EXEk6pq=+`@g4yk%h*l1y zDw8F!mqrE(8&Fr4j`C+;i^N7hTEwq((m^C+i?`^3U`a%Aj+IKz-9j^(JklaEyhg1w zUTu3?ERLt?sA4S6I@e=;U&u>N9J6Mc*S%JEp_|-nVQ+AdmW#Q4iA`)-I=PS;$}(Pa zu9jr>1c&cUs9@dg;5M&W&xL9%)0@6naDPy)5$O2h{+K@d6s3+Uw0%#Q`%X8{(R>={|G=kZ_zkdqH02SEI2Kw6++g(z5v)=WC4Z5^N8{s*rRj2@8>q^0n#7HdP?Is#H zv{oA8haD|w87Lc-$cJ)R&#}iU_GYv){x&>L=30HP+W5b3uhGk`xn*s9Y%Nmtm2bJi zKBukC-Cz*xD%)+P%CcQq&S?z%DH0+56g@%C&`pq0=vSDHjDC>WD!G~JIo;D;;C?~f z?Rj8Yoy^sVl-7|TySYc#sZ6-lO2bsyOZ@{a^-=%w%oM>anZGGq^hw_^sZX?7A2-lc zupA8;o!{YAk=}(<;aQQV-5TFH3*Oa3-r?A$-5UJG5#u%26e8Kk-Cr*K+WtM6D{0Ia zJVChG1VVWp6RZi&v=v0bU$m88>uuoyBot>oRO{)}>otn(>0x-CpzGM)A8wt>oEr(M zoA2FM3mKj1aZH|potqS2EJ<5Tr2)RBVW9zG4IRP}X$1~`*z_rtkNJVzMTo4-!INl} z#0eo1MjR6!q5R2O_pu8rw&M9&VZAus=+U3a<;@c0m7zU?%Nd||Y1riX4H#B|ee4b$ z(AEC^kOADH!T_V-IMF{9!L==21ty{jCmK4VA5O%!gSFOJte3E)gLBZSQ0=240n&eP{Ho4*L4 zGxD4pGTSuT%uLB4e4z?ABGzVYV9jaU9=e`?fsi=@;!%!D?h#^w*^ciur3(q)fzji? z(3T7C(4k%BFJf3PR%LuR+8~%6K%SIAb`?QRNp!f4GjLz0Ra{yYo^#Eb`E}YX8X-hR zq(`1tE)p3Qo)|UoB0*G`{z>EWWFG%4^&&jJqeQD)1~=JDlD z9Rib6E=WewWgGEehgd;Ht`R)s0@nbiUh<_`jS)DVkNWJTL^kI}wo`RF-dU{+ND?F2 z4blxYQvhCp7^>uRnhJQrppeP`B~nrGS@z+@WM4 zWq_{ZAf_AIttOu-9U=x_XfmQQ*-%X)9}hJdC9)u81{H6z)}lQh0T!Gr_E(E#P)y@t`lQOWhe;;m zAo(1R8pQlTTQu5`o%I2Pqpkfu@o7dn!RBsAlc zZe5xt=9*z1V(N~1zF{Oz7;bi@Jf_yIAVF_>rC1i{{F%#%DNl{^O)A2upi1UATn3PU zfY(6CnO4yE0B34#t^6?{1 zW$1=}*e)@bTjmK(h9dtfzSA^F6``skbB4($FvgC^Q7WP+Z@u3}YUH|3Bo^Xm@7W`h zwv{jrz>gYekh)i=w#S@4W+T|7x|Yi+AZe;VDQ2Dolu@Z7BAsC&C`=HgsYKSd-5=a1S}-n&`&}_YQ30A zvxuMj(SsCHs-sS48U3A<=k>iha@-9cGNm5~j4mVQKl$}NIEzzW#y3J5G%3TU2s zf$2RdPZDB-X;lAXYRH3XZRjN+8h~Y;v|RFy*jKWapsC?NJY=Mam2^%6c-&>tx@h+V zXNyu7jdE+ah>P2VZlj`F%|_~sZf-+@t9HI?fy8TNPNNJ0X2w{WY9;P6vS-(-tFv&x zV4kFB`HwX!C`q+xnz>sJ@}sy3A4N^93Vvza9>Cq+t#zPX$l&BpKnTon?0{t*$d+x^ zZda)$u5H>7+Jzk{>Xg3G$>fGa<<8}F^=;209^QGY=pOA})<&6l%^GDP7GmynDy^+0 zZ2>ax* z+j6O{QZN7b9;NkOFV$j}7zCZLg+o-o8$89$-XRj1hFJm2=C_H!<8%O3g^ABG8tc_ z_Vi_Mscv<85B>`6bG^AL4>ZW&jFa+;Y8Fp|=$*~0o@!^`Ddopes z#N_{zGLSlECLSK*@p3Z?ORi1Xo+A%#IRj$d!f*;matYompt%HNHmHIME_WeEZng^I zX5|?yZYt|log|$s&aB(kG$>}M-uPLK4U@}`FFsl+1l5xZf&tfEu znyW?2C=oK;0W0aCj+&1#G)8+hMW*X78?Y}go&s}p^WZZWC}PSeGblB4n+8oaJ0kxZ z&SW)!C)e2pUA*c9h49ZbXkU%xUzhSM?Q~F6wnn8uI!p2;YbI4-uoCp`JJa-2&a=b9 zl_2)84_l>L>9Y`9nLab{KO^x#=ia#igo@Us<&LOpD0DeR^g(}h`>=I!M8>LxwV^^Z zcFOfw`)o!_CNZm=fu%7%vqw#c^iVmovVFE2%NJ}vZ4{(xHDq>chv7-LjPVX@_=YxQ zN4D{~EoZTCWlPotKyqelZzb1IQVVjbl;hqCD8jyHoF=X!NzW1ca{~VNgo=n(FJmaU zGH;U$-$(;qBD9O@FLHB6E3_hIRLK}4cXGpZaC`Uy5#s`btuINE$&7S-iR%A#&ye|| zwl$01M%IEfYc@8=^y)o!j&DvO8_or@Hz30IW<%^B%6BAJnx_b8QCHnbTi_07qu^5G zYa{7xCip(LNrBnhGFtg;D-LZFG{}v~LC^A9FQJmW@~efFasTYx5JRa^8Y_H27Y{IV z566eoxpMogb7wQv0x@L51$6_JXK^p4UV#Ufoouh6*M|4j;_+itcjsr6=TBB3=J*h5q?as7L=(uee`drjJ_~ z(jE{akmRGEM-KD&llP;xC#2W4_ojj^ze! zPw~UqsQjcgI^A#H6d9R^c*leIOhUV+!ly!TGJ`C0sP^@1uAu)AD3PnH#qUCO)k_4p zi*VwRZJGBo*4X_52DanKS$p!!mLCF97o$+(WqjY`mJYE2IW{$2&Wo7ymTG?h}Z7 ziN}!C-YFlsyIvOoZeM`m6VmE_B+Sq2())n~(R8F6q772I@$GyEzAaE|I^0qn<*V?R zB8a(F<9*xoNlH4>>-6KQ`e>~_5LAT2O2jxhB8y2Ob5c||l*tIO8QEmC(y8_N^@hC{ zGDTuOUo@BV!~Jf@<&-i-bDzK2Xn5bx2TUWot5e(i+pGV}lk4MKJLFhfoD?)WWW+?I z{NwEO`~(dZ9VL}BIHfu@U^-~+qB4qdh%!)U9WcOEZN0_yRn^TE9*{l#)wLDo4UA)Z zeTFq+F7=d!zUIx=-Zc*J9thADAGa+hKSwuLPgj?nedoGPlA^ukzPHEM-{z!*_^my|8$PO!#g-y&{Hhf1wt z#)uoMt(dVdJ;DLq8f%MLvt`bmJ%830hArr__@G3brOdI#e~?{ML``|{WRKWFz9b2A zid`p2oJ5i5CCiq0GL7GTf`_Tru4#PYe1w!}*5+BwNWZF=dNb{ImN%{LRs3G^<->zz zZeD__-M0>wljmSpFo1FC_p{GWs~wJgrzlO2D+>Cf?IYPfYULZ&B3p++2N{G1tpa_M ztsu}Y#4tk*DT;>(`C#j3A&H)F=zv)8kw_g5AAF6SBM7W5MhZI|A~)Q2JBv3Zay-tZ zkv0GhIU#X7YPnF9GK8l=mVzqD^Q;R*$x8pGENVQQuIvsgE1}GCJM{v5GCMEP3-b-L zN;B~+6Ln0bHOJxt5wrUqJ0LqXdaS7~Brf?9D5_c#O-3}q31_vn=sXl1Lt7MQw%RJR z49>QKnR7$@F0BtAN-v`*rd+(akSxae-bfxY144Fsc8+3=B4iEh=o-`6 zD*cpF*-4JoSxjcN1vpW5G$UfVmU`&JH=(irek{U%YkLUB8JG4r z@mGmEna7ja+_hJfQ4W^ROt!uf=c<&{vgQCWpVIT69}&HD(}UhisG*5A8ue`rTEac2 zU7<~4zyVejHmQeIvfYj>M;a7)*CsweaL{&5U*#(j9AiMC1}U|;r>8!{UYbVfQ@Z_L zSeu3wlOEBCZJ*krr!n!DaO9P@Kqnswar*DdO@l}x0i6zLLw{Krp0liJ zmC0v4v(wEGc)H-ZP7gj~UD1jLJO+N^Oia_!)1VPPvfa*Xee23o?t~l-RIn3+pb7on zW4uCr@KheWSMnU?sa(SkQ)$<3TR^bxuZSRNN5S3{rz?OHcNF7IX zVTs6>4jO9AHuON>;oy@p54nqohJ#ws4ANcS&NoBiVT_f|bDzgN62C4sP9|a`|v3sXQoZV#qJ& zU6Y|ELZ4u0rMTXV(Qz@tWiGi`1e;)o6UO@G|BR`}XpZhk;X&yfNoq4^;c;~@g{DfA z2D?6HFkH_thalOxQ*k&85nfazd9De)R@SS9JXKUA)8@)nO|eCrDHjy+$;lf|PoJOk zr&MWIN^)eNL=XSdTVyyCxTHdKL8gpa`qWp+=(R7FAfqL6$_a*yYLttL=o|rmsRn2X zGj%^prg+dm2mmTGuqFMgUM2X_XAZ4^Te+b0Y|17ru5w~$J?nOSH$pav(^_-2YwQl7 z&YiB)wC7t@RChSfPf5yc$`F`V_faX?&i1MIqY4A1sy3hqNz9t^m5;Jy zL$~OoiGEECUOZ?0EE-yu7|F2466rV-8y3XQ;f#+p;7Y}72A49Hu`zX>KnY6IC2kc# z**tB06UwfhKu^1+xM?dh_DEI!ZJ{14E@lPk)YZ;MT~kFcRkvp_-Oj11TSXBT!N)|g zMp2cQWZwUA_1n{=mF1@39 z8~fsFv`QoFmCy$Jt5eg$7czM@gv_1`I<9~gosA3|k&!CjML9L8K{T+s^0V67A{dAZ z#;tC5+o|3TxEKy@XK?>(WVBWixeVP1$0l0K_c$3>%=PMYQx+3mPN2m>4HH#lra!`V zN2c2aZvzd{UGdr&D6G@5j&poyKa2)aer%WqVUg1*clyD**y~d6@MLKOLOnxmq)oNA zO7jKSvM+4)Vra{$Ex+y4p{ialZtDdx|5H^WYK(*nCuOrLZLF*|sD`1NK90zF&LLY# z7oPw7JerhvqrWQBp8>tbC_yD=y{vJ#8Mwgyjg-+gj$owE3+ZhmEk$fAj(qaF=~(B> z!ff`Q%#tL)BpavQ)UflAY(v@(4%ySHL~}gtS?hG2%i#}aIH$avNl;??3ccnnSE~xv z+evn`e)BKdR@>(Fy}84d)U;cl8*h3{yOGiz@t#pqXk@B5fHV&;*Aksl0oAzCbX0fH zUrK2+!5Sj*c5i#4SMbu77KyV+G0Z0OZ)(hY!|Yo%ve#VY3eQ%{2WG7fTf1<3PTcGR z=_jZdtXobdu~8a-2%`uJQIKC4dB|w0vae1d-z2@g_OnsUT~0r1pBM})bJ_o{)@}b0 zE@{SgKqHUEz4Oyi{^#a#rqCNpb!j;W=?*nGu@4?$s5b*CqmIVS%oFg*fo-&79zAg> zOxEieIN}U%xP2f#b}oPY?Dqt77cs6o49=b7XMPJk5=!N}@9^DKG&x(|-$t;06~RF&5fI|HOOvMaFPOC^fiH4hm+Op<~;OA}~=yJF)eQaPpaskdUAqV?f0 zwGzQp=oN_)u@DP9jbfZ@8xmeh67Q>-oB_YbLo6B$G$ebhnmaf2YXqWMfgJyfw8d(j zYN@c#10(^fKNx}s$AcL}^0T^VKT*Iw6-o_)i#_I(GQ3m3ExbM6gSg#uE#G6nw+X)Z z6ElLqAP{S+g0jD{7_N01z`O%NSIHvjs|+Kwx{HFE!IKe~b21@`fIHY9#Ir8$5+OJW z9rGhPKUu^byw{4n-@K#E`96A z^f;!Q^pcf2Kg+uubWBICD6e(H7IA!lRFMpgdy#pBw_N-^yqg6!+Qn6>jKgaXEms{+O`vydo;dn+r!X~|@S zzLI1%5@g9FAdZuSt7s%V-(bPxh#4CcMc&E3#B989%sCa~t{DFtN6O^MpY$diOShp6 zns-D-Yf1t;f;XxW7^WNx9TWr&_zT55H9c!Rn2a)el%#^JJ-DFAa2NuIygl6n!!BGv zi3}LJvB<1Sh_`%2HuNp(t2>eGz*e(E1Cc&vWJwP9_mF zONcu9+P9n#KLAMsozyw*d>Qa0&&W)Jbju^~e8tF;H}?M|8cv+Ow9F8C5=*5S7H;#O zLhKV%1JU@1O{O!=D6Ky&1keDj0JhuBuIx>+ge?Rmn0^}0inE(O>NIMS&_e2i|2&&! zJUQsh#9JxB`{cXhARg;Hu@WUj{8NJ?y*Y2J%aoD^x{CjJc5_H zq)ai;NaezXGyuICR6^Ajs>IKX6v(^O%40;r2K#~r?b2B^(1)8eP-3i^nK}jD(MG9ZP)mLM@G1e5Kh1^b^WJSIrSr zO1e#A_(uRt$k6l89aXBZVY^Ya*8|ko-t$s?8UR=Qj0p@MgB!pzU4~7#fQX^HA~lsc z%ut4X#%Z*oJNrvr5z*}nM85hiGup;RSW)h{04l(%k{wx*Jt=OaTc6Y;E*M#Yyr~JZ zBUBOB!R68nG=dbhTWW~gi7g@{`dN9iLfQX(*L-MP;#`P}G*#YQ+EQ&=f(cV!>x>|o zT++NX;i*S)SU~5)P&<5Bm^=uur8k*$uCtxVY;(a~ii+{u*tnGtniM~it=q|z*5FOo zyr@>UM3a^s8V%seh3ie3eO~AV!?Ifu<1L*fMY^=pUR)?1{cKLH6WXewS5MM{*CW*| z6IWEV9@3Ss&+wNbf}~LbU#6>7TW|wm*`(Bs4R!5Rsw7O8befCZ$PF;e?ISd8JiWFB zN1CkAcvzXVok6`VRA{x)p9BxS)uXpSjpTiY)yZ0sL*LCM+&7d635YuUqoeF4IN=ml z$lWcWHO9&9%3i}wA*fsPXoNV5OSq0Yr2%Xr`I30>oq+;vJC*UCNWrOs4zC6s#y z*VV&brP%*{xz#Mu!s~G*tLz}ukQ`tt^1pEq2U@Xr86WK!c|}P6;ee`(Af%NAr49T4LSOi zQd=d3da}JZ-q46WuCTSG$1U3sC13$2(c1-7J6X!!MX~GX#xh2_N;fN&eg?>soAv0gemE1vI&_ce*caoFo?Lr^!*GJCdfF0dM z4#@^8)oCuAv9+N&d{b$JQ;Gj25KykzQmW#K<-YOM*js)^A&`VRmJ&rwTv)DPS9YLM zqoYj8Cg3-1|1v^HBaQV1O0W%Fv<|0;AuC!>7 z_9`0A--JbEkIpBM_JSMc9x~15f{o9f;tF2SLRnm0rh2EDcATr56(*%!nx251IjJSF zA<3Al=dfU`HfnwTNq+wh)OvzQPRoW^9qHdgj9_}cDah8(!sA!HO@(!0TG++^nrdx2 zZM7KdvM_6|J!o6xT(uq;Z*WzDBQwjUreov@4Fq_WTT|;lcragx z6VncC+_(m7qa9=~zTvj?%_N3xf|l*hplz(WW{i|K`9{k$y`jC>IAg0s`P)?um9Qt~ zP>AJU&@@|hUbdJEM3q3+GYCYR&;z|jY&;96<;Yp{xNh^%&S+g~$L@pQ>f5;3o34Gw z2fY`Gr0esph~NMHl6x5IfOzj$gHk`fvhf8a);8-mhEE^XW~TgaB5F%DOI}b7Uy4ZR zoWf`CsRcdA>;5e#*`=4C{mSHiP0Hyg3FweQTLY6E$-ESCjw10;B)SU?r3Um z@`G^e+9F~>?*~j!@E2nb`;7xQdrswQT~hmBbY2ta3T)zz4{4j*(T?sV!(u)N*%g;_ zJ7MwS<#VUrYGL1B-6rqyKJX4)aZnic`#xXS)?P;ka>Q6|8y;lmmEPWi?f!0L+}ZT( z8PG11T512s2x>LmuuAA~2IrRsU@9)|6~dJ}Scw%Z)_MnnJrx8HPZ7WKg67V1M|9C~ zB<$_B=La}~V8=kN4*Tg|rh5bt*-C3wpe*EPNPV462Vwe$W@ZhxYas^YubU%7BOW$poHXS6LN&stnY{qviu5e+}_#6o&k3Lt}6`|e&A@^wgjX`1+cB66`}`;;HuYM1n~uUTn^W2OyxOiBGj295hr zdm|5U0SA4qC0?QUV7+^h5UeI&32t?L^=8vIJ#qJSZqjfn1h8T6bbucnANtHs`sX)e z>t*`8E|tU30LMZHtJmb!Uww+shErG6uBZ4cwD?t=>fAQ%T@d{%J@R|CbhIBBYwpxV ziSoA}B%)UF4{xi9^=lwdL>!e#@n}RE9m8UCIebo}6NPY!9cs3kOIK6Lh@IK;a;r?H z*Xs6hd$~L=Po~{rJ>>OyYNaa?C@2Ux_+WULsJO`3i1_FzAx8ljS!roV`FM%R+4=to z8rr$!Df(%Ex_6Ljg{Q!3rLmHda-qt%imJQIi<>*p8=PwZ3@kkCdz-ARTxyz_93VX~ zU2Pq(oV;p%jLh2Y4IbY8&BuFf)Q9SsL{LLoMY(9E%$<#U{J3d@kqH!$PDoY^iN%XbG-1Rjr1E47%#0t| zkm)jsLP%H4aG9KvlM+rZg(ez(Rw$09P)jd0UG&u1(Wz8FVW}#Lp09GuX3?^xtrtXH z!20ClN6y|^Sg*qF;p*1h0j!O@u}fF3A2)_Q<2Horu&=?p)z%TlW-swQ#ftyeqD2V# zVPuDsb=4eP*#OC0$T)ij+?g^jQ&FpGIyo~nO)Pzg)O^iGb;*}hlyXVpQuY*Xw9GIX$!1N~(B|Po45sNPJ@smv;{dC7jWgOt57S;JS25mEWQAZcC z0k_*wOA%+>iu?KKz>XQDl-(iH9W|v?DqUA4cGPXTiIpgRN!L|IT|oa!nOQzSUub=) zmx5+-#dlU&#=W!QJuq(b7+Zdc_1Bl@G&oH|3Pt>w}SEH8%JisC3s$bqFqo|LY1s1XM(w`R8+Y(QJLDBNuGXY;gX)V=iP$GKRC`= zPknOUvDRB|Ru*$J9y-?F1^pD|EF(Tf4L(B1yDu)U`yS)m!|!#U@z&YSznU5+`O+x%Q}8 zZlZ|pqCg^pEBc;S#ax`xKWB%ye~ zXiFeaisRXf#N*QtflBU&oTU0mMz=F

AbL({MeK0?z+G#QW4axPJg}VkQ!ZLX?0p zBmC!U1M$y4h({VGxo(0c%be;|hQXatPyzN59R+x&8lrr`Ih(T;>$LMayLm(-MS-BS zq9VQ4{3;~E1D*?SD2k^Y4|&>>4)ca_HD*NbZthw|e(VFU_}tGJt-uo#e^|D<=_!S+ zSjm$1Fsc2;X@7y!B4Y?eH_&)XiqvZ!_6&d~z7^(Cf9szcDHTA&$#ISfh#3+RSOJb1 zusqml4-T!uhmw8navIcJ2}}1Wmz4o%bf6VcR22c?oKAJT+CnD2=R8!op;&70g6(ei z!s_5F5LDC+mYBsh+$rmiPBES+*t9J+IRc6a$)x`khG-uBWI%h9aT((R zV)i1_m~)V2ARJOA7~_YiyveITDOiCuAD~7DCE+e_>}D628L>Hv6Cor)pr$g$r(_Ca zdHrA|CdxK43bN>Al!RpB;u!^{$nJwO8A5e9H$Gh<@^Wn>Whr}R22N(7pq)@2D78c{ zI>4)#u29u!=3>J@I4gIgVwyWa=}%Me(3(B03*N?v%YH4&WJEw_e zv$(K2`EP*gB#Yd<1ylvaOPXagqcx9;&2-eyi|M=`9LEXOsA}e7dsI^qkkQOtjOB5@ za>z$BdNM`Yvw|_I=LT#PyELTgX{3wUj)wn&2GB`TpoWxDQmCUrDhdlX3LO>BXDGZD~0F$4qcaH6b86 zZa~l}T-vT8Dqj66wDc)Yd;-gYhV0zARyGp}`c9OTXk-}3x7a2H?KT5NWMILf*XgbB zN2gR6V^$oa(R`nANr~UOKP;R zNn2aZYF1!D95z-@eodM^OFr^$DQDBFVUgcC1k55hmergEqd+|2(1=z65S|_|Z-H9l zP8D6OMb1J-EL^9~mVlE^Uvas}K(U~Bw=JfrZu!(_^=N>>0dP$Q^k-w+rnm2+p2h2$h zlKHy4q$Lwn&Xqj`2xCLhRnh;mcPK-emLxf6Xw$jQjgWS$m4V)wT>bsRw&y9$Z@4Gk&xBp}7P>F-WkRCwr6csD}c zp`bTu7v}FfL0k`o7G|53%oAbhyETi!^uj?d)u+}ro9+Ab$a8xllGZ!pB>%BpC@tx~ z|2@Yx4mlnXcvQ`XItBkn2=z6)y6bO@rDNSz`?(8A?Cti~%?|ii5e_oQ95sop`W!>L zm!oANRePY%!49^eZW3y*M!kjo1eezei(M|GcLz+FkFaZo+mk)m&?{5cN`J#E?2z7+ zU>~6O9-6wJR=#1a!cB?s5z^_gP`vdSk8EKu zck!cWJY{(YXg~!)QnSE-niU2Uz?sfrgg^3@u5-wP6aG*1o1bk5^bTn~f@X|t-fF$CDZGq-! zj9OFY~ZI@YlHZ{Y8eb5pKuC#q6C519Z4Tp9X_<@C6;cqn}VT_S}Fr@(Q zM~rBgQ}7pmeb+_qcZO}`ac-D@(KI)Gl6n7UZ{=|W19*r6q+FG!hvRg3KBra`;Ca?m zdqq%Qn`i$o`PgX815p-ugr;|aU7&MamRPRyfkCH0gW_}TkTQa_NJg?HDd=u2_yoAJ zbYXT^$0vi!CUwApW*dQnP*I8)fmVrxP%RUL2+4GEHdtu`JmknN8AuoZC<*qch+C#* zim)(Y5qDLjEgDc-*_H)ds1;tg74k=o^M_-yg_Mx?Mba2ugqIn+HHIXTc>c#(6u?A0 z_l3zNhGKYJ2Ka{UXpXW*0S!nCWvM){^lQd2aJnFqy|^^kHG@sWJz1A^cxMJvNQu?Z zfg4n1p<{cSXe3AiSG#9z8M%=frjkyk5FzOwNsxNtCX`$kNb{AK_Ed{7C_K6-i~xx& zPEh}PKGz6=A_2wdjaDcwM5IyxXMo?rYtl$+W=M^Y_gZBLmD3n0w7Epw*oNJxVv8~W z{g-(!7BY>8jvgYGWXYWV0FQQ*3Tv|vhX5rKAT_7NT+t~Bw@8R`*&p0t8`;G@K&Y4I z2Z=vcjIZaHmZ%XsF?JG(dn$--apj3^P-Sy~R`C@!q*#N)X9tj3lUHz7^|^)VNOBY9 zb^EylCDf7YR(2u)p&|fSo*9#%875kP{l$ZYN@ZJHQh_yt!jh?u)EO20>b;MJe1 zAp~W&2pI7sI5&LY_=7WImqPeu59pwXAOavDsDlauT9SlR8l|FnlMeuli7*X=0HkB& zJL@rVqE%rnWmD=$f3i8F_+z6;xvBN1n=)FBcd<3ym}23Ws-Bh}JL+f18Fxl1tA=x0 zA3+(UWCWUkJY|O-#b zGYKyOI|_;gV^E(=Ado`9L5sjSbvldTMPZf6dOygfb~Jr}N~rOQ9h#>eX(#`zN_ZOf z2vYjG2prl+J-K%UN}^C_ot~v`_c~y-@ zFhk0s%lTT!2(J3Fe}UQ){d1gsT-$Ua<5Pzh(fn)3Q>^gcgdX$LfJJ@`HK#b2|I16omhRe!3Ey7+@cX zPw&AKIiX23>xdlrreP`tN0Jc(s%_!5VLmuT2)T8cYXwMfSJI)igGvHidzL$DuZW7W z5`j3SX|~`ufG%fuRS0~cS9#V0hI0F1-&e6AcAGTXslIt2Rk^)m3Ap)S63nwE>&LA#;+S8T2q@7KB>A~>_fit9yO~+D&4;y(5WIuRwI;WKS&68Oz;?RR99k&Ok#deUptDs^4ro}cgB18dFmcV986y%m&$az^h z!MXxdkY(^B=h~juHVAC60ejq-YbG)u%3VsJBpe8dplE>-P{M+G!d|<=#j~U?JOMBa zu%QLLcGHun^$)xB!`Ew#kvCHX07I1Bg(pbW7#ut11q)ME5wrGy|`5|<9i^nywN-A0dMfp0)36U%&_ijhaxQ^ z&KGmy(Y37oJv2PmeJk)(4Mx}I~vGgAPd#g&W0UTczw`HK+wKm z(v{qQnWfV6O}X>}+^W=2S2BZYoTV7de26oku;F0Ws#+gg1?OQER4t&m_{3X#f_$tsjj zOTqa!%2g@U@uA>SoUjaiFncSB>V4PKB##)&S>rAL2DJdqqM}o*mSYVEc*(VgR4Y^y z-Z?~rfsfr-g+Oph`^=R6bkppR%Z;#i@W#IDtBnofrR=%g_prMB;HrJpFKxlp>EFx^ zT{mo?0*tN;?t>h>1lighBtYm|d(~Jyt}V^NQy>b&eaWYA+&nqZ-fV@EZIEa5KMf5( z3=-W6%jpq97(l)+qh7ZcZR$E|nlCUm@rD! zJDJtYf#nq7HmyPKmZ%0!lm18tdm6nAK3tOK3#lF9NL|@~{K2s6)K#f%?M)lDWD>-8 zi-cb2gzDSAm*w{@Srgs~jvm&PUgczc-2==2h3ezVvsKrh{^OvY@B3~)qE6}oGQI&H z5GhNq1uu2PvE<04Tn4yP3-9oBv}1fo%m!%XD~{h<-ir`S-C_RcOwc~Tei3MlxvUGc z(rGLuPpohad~Pn+Md`mq?RouW&HQ z4(VbYTJUZ*nDQ^Ypi@4M@1$lw`u^`%kM#hLs@(nEdBf%}$B#Xfj{wfNh((pJF65Ij z>%n00h?DHPj^#vuQHYz_I*XdNOQ*EEy7R0DP$qVgtBDXf-&VS(1goGZb42tN=1^yIz=c8>8jr0&Nn2-Z+XzVqUly!2Wc;}Z8K zmX^1qra!S?*H+K(xBoX;kM*Zs@6ePyZ#(gJi4eIwq+v<2CyM;T4^G3p1Ub*Ij4GNa zPV`8zTFVs8{@d|h>JF9;;5*}GWzJSyS9bdZ?v0(T7{09w#i$XTT4ycu8{~U;&Zp>d z%QifINW1e|$13P19Y8=54naj@F?292508rC5tUA{3@ewjoa+31zt&cAgb5pakt?!a^ z?O`Rw^t6vIqmC)U!9mnbc9ypP*7g?HQ#UsjSM~SH)@!0diej$f0!>00i^BOC+Id-e zntChxx*C?#)%z9v`y0;70nlJz&HGE7J!2z0R6f2u+*Kp`LSF(nK0mvfK7kkP6*SoH zOEYfrcwJ&c>f+Y(BlZc`DP&*Cx;7?y zXd>7MqU|_woV)CwZJOJNKV%|k2EXC*_%6fcl=EULyez_KqYQ`ylT4?CW2vN$9>Zfu z>{`NyylQUB={zI<(qpYBY*^G&M$8`2u1_%vQ9vm)@N=^{IHc^dL90aC0};UdG7u|C zv-&SUvM6mNKuX3s5JV9}9aXUkjI>OdR51j z;5t!k+Tt?$jyXhqgKx_Lo)}R1Oc7cU4nO@eJ5iI^lIzSO3CU z)Sg=wF%}a4VKI@875PcVPl)$~F=C(ystVmA#p#XY(|nu_$nT(9ia0BR# z3*rn8X2*=A;K8f-KvX&{<4mMT;U?_ zZ0?xMj9!{)%7L0Vj8Bs?mJ5D;i;A^IJ02L5swXvh$B@D5N{~xS?lk4C5PUlU1QtMG z^q;eHq4wTw=lyrzg(tv;1FF481G-eHlIK=oeSYU#ZOvI!mwjeSwZNxGhuFxz6_^tO z3=m)Z1ImAJZY|ks+VY2=(TC$bz@=7Rkbk9if(?=^*V#6|K5k2gvw}lbBsluzCtb{N zN-cQ*MH&Fci%}x!!gA=Kt=4d@Z`5;F_JYwJc#+OHwpkHQNZ2?e^ub}sQd-0?$AwQV zW@*F;)uMD)7}M+}b*RJP4h5vcPsI+1XmJ4WZdb(M9T5R8s8NVUL_{YJFNslnA`)%q z01ZSyLm3L4Et&_t>wVEdOtjvx>hg2Qu0UMbt!whfHM}faU(j+0UF*&RaeIPWG#=^F(^ld8} zgk!@4@v|XA{Cve$x#7so8S~@ILSHA4=xgX&3vBD zyhuIg{py{~{1q5I#4~kD31E6jQ3vl>M}PhkpaGrZ67;yoW~Bs2p)*9twuX%Tc`zQ5 zd?YY1P{@f+asrll7l|<1(F*=1fovltE*$v52%dC(5=f_h5W}-j+(%b{`+{`*n9fsP z#Ap#P;dw}arWB5imz2}xd!)ms2RK5Qz`T-!SX#rDG81|wi=J1o=_yVH5vz|V=T@n> zMC)l2tit1_?TV;KYduSyd?+7R$$36{z7?*@V~jqd0Fear)1P?N>m2b4&;s=TvyFj` zqXpZCEJwlYf>c}RURv^scf<{$d%NglJzCi>NTZCD)hum5h{*_r2uqOtBm_gMQnZq? zL5cC|FjN^?K(gj)4xrJTc3K1yh@fDRtAg2h%cNa0V<3gqMtE414Rnwpjl&FMxlmh7 z=PuBleR<{$qY72k`OrXF)oxh1D@70rgAYeE;&^+nZ~;tUf(974zzH6(0G3ox`^0aP>f2&Sq?Nj@5$d6? zW$hw4y4lIbN2MJ`QinSm(oCK>N5H#Ca}gUaSkMbOmSovHNopqPY^SFGXoc_}6~MUK zI_?s6O=)NLVN|0Od4-ot;4<IqJsZLWNz_s00In1^P1J%<_fI&&2fgan}@b4Giv!&@zrzfNarC$In};P;HVpL1Cjm$7}1Gl zBY_dz07pL>(g!wy0EYY%TtxY%pERigpvl;&6;faa$*hMhTUlp}n$#C1U5}J;A^thG z0*^+4tRoF+^NLc(8V)4|H`d~w&7-xEg%-iQM%y2Q;Bged<=86cTi_zu!SDL*6^@(S z`Yg9Q$3*G258G#3J#%}`q;8g{E0(NQwaZR!a}n2VUU#3@h)b;hF(~HT?t05P1NW9r z^k|sF9TI1p0T=jI^+4lTG2y-CB(HqmP(1nu8dd#JOtC7^S4Ceupah2WtT~PXkI%XR zlop^i{(WV~wD`|(Xx$;Y;!1I3n6#*txkrVT(S6oj#2abg#Ezk%27tWQkVbHUB|Y>4 z69DC_d~sxW?U3R*91d67v2nX3&h2)nKsA~kyGIRmLrjJoB1SO* zJzp(XQ;^jaB)Z}LWxGED9r~8@-tD~rW=;LfZ038u317)d7T)lLXTv+!i)ZI?SIqB@ zw};pY-^o|cLqb!JN(t~Si(?%0bu{qCfBtdPADGg@O@qn**#!uY$}ORn&`QWq1L{pf zC)K6itQ2yt`x0GJwx%88=o_8jqAM8r$d@$IJ*o8EMsjKvK)`|*Em)Cv8@$9eT$jMn ztWRg%*o~u8*AHiNum{zB(13Qdt-TCDq_IyUg%C|s&TZ@$oL#W;H>Rws7wb~0-0qI< zx(vVcZq5#H&AjO_6oAd#EX@#a&LFVx%;9I~v& zLFR_#ujqgQLXQOnO5-A}<4g|)3arvHCoocHXSgDZSV>DL%-ryHj>%%LLG> z2*h3z4-yMc5)CiSlF4)IEfOQ~u`J*nIMEXi&+m?hgH|Gf@aZHDu<$H^5;rYW#%j#i zt#`Z)Xl4l2hJ}qFq6KsDpI*?>V$kDCZ`Mqp29*j|zReOsf(K)7@hYsU76T>vMI~}i z8YL+Sk8rcdXd{+|(Tc7Lr?B`Q&G-bY8x!nyBmx@3LBSXe3fHl~4z0hykkN{#v+R#< zMr%vH&peJT2}VKLEMr`ntumTK_4bJOE{o*<+V5eoZO2%Q9-qsZxJY`o%%1w-S2CsU zuxbEX=c*pT0rTwvGZ6xz#AjTf0UfXbGf@>&auwMj0b|tTh+vj>>Q3+_PsDBx>+mwT@F30aVsvNiev%g~ES?f^q1eevGLruo z@ho^KG$R4uP}1-|F(pM3&>R30ClLbwQ&Ki5F$i4}Ca)$YPtqf06Y;1+=5BJsSaR?x zi8rOg&jb-M2CWa;NQvGIDWCH%=aSZz(ia;*0no1s@+SXs=7*3`2+v_Td~oF;LC`)R z0~ks%5MmlVj4`S)SOgMA$fideKmdrYEfwr7-x5E=aSSDm8x^b|e91NUathV)^Qv#a z=FuM4(8XkGrflNafQ+D=aJSmPF{h2$2GXdCiXdSqP->{D06lS+fVuc2$3>l@l32 z7U!)sYf%jjtvRdoO0QHpyOdmk(c=o>OF1L0qH;{DvZ}6y91>(UI88y;;Lg0#P1H?I z55WrzCPM=PJGB5^47E@H*)l)tQVRbRP}vf9xZybl4EYGuKnu(-4fIb7Kz^KMKT^vM z@DV~I)KdE~Q^_C-SZApEB+1h6QbAQ^IY3nF>gxoSn6h;-NU*-T$a!KFQ%qGY$Sp-r zW&pRR5IM3ZbrnYO2}z?>ILoU6V>2~Z@+EmJX-ksLh7?(E^N)^nrqHa-W-=x9M-{QP zXC{&tP7vYXh)TmXTx71u`FkQ2N5XDUpvICo@a$wTndH(eoRAeqHp)}5-j7SzT zv(tE}Jv`RedF`xN}4`lOj!&frvv^L6cUYazFw=M?rEEd36G> zR%w&g@0#~mjkP$JS9<+UNO#j&l~;O^76FmBSH%|dER$Q;AdbFOZNrrpsq<~ml`@q} zo*Y3cqQN;D>_X#{ewFQ^D1plW7iwjNuo!>%221pYNfZau%7>53ntt}&gx5fd z7dE+!T%n$Cj5dWj2Qn__uIfqmTKRfTM65M{khxS?ETN0`L@& z0XR_q1FL}tnvI~bOi-+2CM;hx*piXWBAP*ymkkR#*+r0HJQ4>BG&)A`l22fQMvCD> z<<9^@-~mDarELTkOxmQQfTd3wre9j7VH!MT`D37o7NkkKEHb-PG)&p4s*2WWVN_C= z8H%?*dK(7qWuDL3e*^A}$M z{X8{~GnFKNbdD_{BNHx(n4k>qH~{+B3+VC+3tGV7v5^xwkt3FJ2Q1RsapVfM_!5=1W4?yjka@s32JDSrb&R|)Tzoz+2& znmUZQjEFdyix;}DyO}%Es*4!AjntWu_o}hEzLcQm4q%Q9O{^~tbFtKn%Q^-RtY6JJ zoyiUXR8Wd+8HbFUy8y$k)p{A`66jE`uP@iHA6Ier)PW0F=nCK;DnOqbd#~90<2nHK z2)gttyp1>)gA`Ks9OWDGk*DfJ1fJs=@{W`x0ED+-#YB6Ak>!^tg8@dM1V+GR?Kg(d zAq!qxw_SR-dtAqP9LRUOn(*)dY!g z^m=>Lsi76BQIflTq^YwT6|I?Aw^~$A(5t_i4#GOU|I5uqZ(^a4^eViN*E(X7$iAa! zERvfHDlY-*`dj4L!N_{(09zar-M|A_ar

HM!;_W~;5^aKo`3arwtuk;)_vJo0w zW(dMGye3Aswlv!bI2#G60CiRT#yI*|GULUKA_8Q5#xsBe9z$tnn#XyYwr_h1ejKKE zJ=ll+wPpLrf7-}{n{MBU$&1^#ZJJeYRlA>gsUd(+x*J-f*`advk+$23v7LFze9E_6 zntc-$!TXv4EzN7u;#48d-~1>Mv~r2ha^oA%)0uZ><92|@mdgMV;7ic|c zy!rZ`6LwXlK^JIDwC#!|C$Gv(~UeA z0cVf9s)BThr~GNFyvt4TAV3nCzun5KKHOzes`Z8I-{nTl9IHje;PS#0*MyG18s5to z(c96l-3U+a{U`GM&AO=)M9_8uegd*G4Ionu2Ak0lo$wWXFGWxA1#IM8;KJV>I+;*3 zKs{SSg4BP^)OAkX);9_WJv-yZjR7`gSsg{RgREwL=3Vy9M&Nh+hn zPu6Jg3!ndiz$6?NibVi2i4-u6PJ<*WQ8by#D>kd$8V=oq0_>PX$PtAD{0*zuYjzqg zS1Uv$5I(Qp?;m~%*??dMp_HLPK)~Ok;g}iY;)SFnWTjsOLxq2Vv-l zh)Br8YAfq&NUZGYA}#E#EN(38?yT>xtl$7*S8>;{@iB4#qHuDuAaSuU^K>;eGj_K2 zHn+C*bac3Nb_Rnv1bTS|x<$Ktx&?#dJiS@{Sp9^0Ccpl=I)rlX1~Q0e?_j=u2@@`; zkl-A;V)%nH;j5fIv&80$XO%tZDP+O(R1Z z@a)-ONR^;Sx?C!1^e9r4E?lr&>J$Ogs1K!5eOeW2nRPC&CQxALYf_^W2rwZc#EFqA zXwf#&Y4(T~D@%?DIIze7&b&60(NQ>uj+ihGtR^6A80sB9iWS?cpdgWf3gy6}WL8vx z!!wlsPzsxr-l@=?AaLLSLDVN+xm*F8rTTU2*s^E;2OHMRwzOy8a2xYdO?S88!iN(N z4$k&CZ3Y7}7dY@a1P2)uQpecupP7UH01|pWa9{QC*0pDc2+(`GhULeL@3&E4caQSr zjRZ+@q~|DEu}dbKMXOwT2{<5uaq0AvPYDtcluTf3_F#SxLa4w}T1`dO0vKM%m4@3n zl;MU@aV8;&U{T;eTW_tkVpatWy2NHCZE30*3p3%|x6E-P zH4{>DbryFgp45l~XEy--_>V*cEi~vwg^~gPUIOj`c^-NNS!XDu?Og}epX^MOop_Nh zWZ|M6z2m|&BB3;@A1Osx31l!;JzL1`Min~Af(GQwq-Vs=tNDW|CB+L{R~tmbdLNn;Y9;3#8KoDT~p zNS+zz8K<4taMv6>=ous_d+ULBouiC4dNRo+ua`+fovz%{%Pmi;Rdexq+5i&+FzfHH zj__AOw@SFG53NZnO_LS2nt)(}DB!C9YlEf&-7l)b9;ikb&6e0rK~PMjOW(MaqJ8@E}E@iw#?Sz@`3(Sl z3l|c)MmWP!aDstLK;jhRfD35}IK@&>r$b)^AQ(POo$B1Qr-@ZeH*=DP?vkj)YwQMhw%fotRMwqFZOUdQqu!xX z7LX~f%!=Xx-xZ_SoG#YPWmwz@Jmk|62>e8D03gcyj*=|w6fFrytHAm4n8&6O0cz`$ zS{tnf!x*|iXoV?O*b-Sku!YD04}zJHXm|oqOtO-mXxdt;g-I7h3vLR+WQ?+)Kor45 zf=Gg(1uLk^bFuP&ibPxnK_CJTRF0N{FyZ4+NXyGfK$lsXR~hyfM@yvYG?qLjB`HaY z8;XW>JA5YUg4nv%5%Gxsd?{kYn7G8;;c0iS^9Jqw)G6n@Lm=`<6?K{yDOKIikmIxz zJmbMKey9^=U*yz}f;Twgr4u8S(pg8SmW5$nCW|d`f%smf$A#+AEl+zWf*{HhAqbM7 zz;X%+iKWP`b&WrU84QRp8nBTb(*&EO)>>3#mNo`OK_*2)i|pt?2lj1)6u3YHwBV|Z zT4gClBY^!{$tO*n&0@Q6SpZ>a5gcnxck_7xR|}@7>b|OA>kRdNKP_(v5ba& z)jRb`vy~K9QSXuejHfoxfexSmkOLIJ00tQ$`~jAHtqLDQL#s4f^#r0jd74j((5tHj z#(!u)Km<%V9j=s%qYUT~BOk)m86vZ{B|v6G=|b9-QVUxJ2yPKpdRd`}tD~_MAF;gV z%2ZN!x{w#BqM&_v&|s}{YFLOqR2bkV6%`|UWZINL3H*&Ae% zrgp{;RZ)uncBNd0y2XuhDQaQ-)?@3U>v~kK?sZp5B<*fD4=mIrA-)`5rap@w9OXqtr4@;?fpCm;{|8U{{Ps7n6EGFcvYASG3xo**Ln(%IK7-+=-ZZ zB04M)awS=C0$uZZ*S_vGSVNI3)27;OY!NQD=r$t({I76UPUW+Wp_SBR^_4L0v>&B*~c(AwnY zaq4aV*Kj7d(D4M)Yx{ZOIkwXb(>{2G8|*xXS9;oqc8SCt{P2S8bpxpl?ygo%P;`PX z3l}Lh%JXvE<2GQ#C~(4Dj_`!P$idY(zPZgYI=@&W0LXB51g>)}boP$)6T~$#vfcLL zC0x7d>cTeEoqj=WBUJ>5S&|3{7VdEW>xF0jL(gk{>tKKGOJkP6ywlDmCQ@t}Zr}6l z3jAMx)19uz8P0Nk2U(Ggtg(i5G+r$=>3c#L<5slvq}8+74*wIfm$tymSz5q1em%ts z#e#}cu6dfEwkIQQBe9dqQt9TL^oOKvIX2|;b+g{q5kPmZWB=>f%bx6~Sa#mvhWlFo zF{n}-|Gn=E4o~0{U)oVO{sgtHLBt>x1b2u4DQGGE=~KTnwA_FPu&@2cMSx(~YkdeX1`#c6PT_Pu)bG#+(4oOi_2VyzR) z_*fNXQNbw|q%B;ey$6VC&vF@;OjMphW!@Dy+~#c{`m|i+c^*q?(u+ipl=V~zI>iZ| zV9te=Mywv|v6GABb#NdIX`=9pBV79|*Vr zh=>4_aLA606aq%T_esDe7?b$DAe${fxS3G;%^PIA-#5Y4oZz3I^;g~LPQUH{hHyZU z-oXdop@ZN3orN(9g+&}4IvS;|M5Q#ER_&q7>=Sl8UOOOHi=;(nl^3W12m7XSs2ub0f7P8xh1>FtG9x0BZUi9FsU{dfI;VPcPrLZC_wqh(Y z0M+q<)%i%b%EKHwj~+5w9I_05EFwFpV_;ca;vGs+>O-{w)O@7cwVavdu~;KcfE!gL z2Rcnm%|)^yWI}4xNRp&6t|ePaq+3oTuz_U~K*dG|-z&-@Ue+Z>Djy#x6}jZ1ES;WU zrVEC62{2yGO_`8la?ndQrVUZV&RJKuP2V!g7kECOLI?tyk9U-O;kMS`cHXqF$W5R}5_ zFm_h;tsXO{;kTR)|Zs$1wrYzlJ;3Vmlt&&$1 z%7ZB8R7DPa{_1<;ry1O5mWG!ycIghSsav^KG@fBKooRuhX}|$x?Ys>BNap?K!OkqTivEQXRkT`HwPKU6}n zqU10AfqMcgv7S(7oNTc|)g~aT4t*)a3@H3zCN;&~8ltIy*^W*I*j;ra|1C!SzF{(>^7Shlwy(V1&XsTL9=h&7l zjehE<;%LCwmPMXJc3u!iJq3^8*0=zv-A1gda;LT7?Xv+UtyYl6a_qIC0l9gsBbjV_ z@+#s=X?RUX1gPxFqR_Fr?CF%D!1-rQ(&V1hB!Nb2&gL1kD(Ht<>!k$k6&WS++~IfZ z!%}MPg}&|t*e&UbtJdnKqW~=-N=?;Ttw~}3E&MR0XT>XudacrgQXZMWzUr&;%4okf zZ|#wtvrSj3vgAw6C07_M_M+g$?(M{OFW>U5-$ENpn&-v}Zf(s`8Ayc!co(onZskY@ zZlzbADjA3>@#X{y7l;6eExrO74Wcb|5*;X@ z=iK7hPl@o2@mxomFZZe{wV|i@2C+GSoO9&%mGIYP3}gNAcx*dZQ>@DXbB5%BBwC#RVqWZaPlti^2#tJ+u#U9FAzuX!8Yt* z-s%r~uXlp-_lD>44Y8~$u_l{eQ&=C$9>n^}FD%Ehcy+PD0C3IXul_m=8T;>N{#j4v zGB)C!&!T3;$)h+Hu8$S1921~FwUXhz?)@|y)AlKmtZ^G3$x>{T!9Cswg4~kHgbs&u z_Nrj*>c}0Dss!YjNCZgU&APdMjWZj~sU#qQq8g zDI`V>)k$l}w9W212x7;FEI%t~>Q1F#~BhM@0#qi|HZKX;L$8q9!JDG9pQu8u5%n zE!nP!Gf!tP3G0YZcP&yEH7DiK2mqTFzH=l`B#z!K$8D%RpSBO9_EfL-J#S?A)~(*Q zLlQHwSYvVIei?4pHn^4lwHJ%&zU{32p7l;%bT^GAf^K8EB4s_YB*}sFQ7SMS7vUk1 zvmu+3;ziNS;BNAqFO{WHK%oU>V`5K_aLysJ8%ZTPXZF`_cKPsPXumhFk#=iS=tQu# zeSfcEW3_%yb=C26Im$NqQiN{H@qFgS)kXVzil7G#iq`F#jx^ z8Zd<3byl_XGUrcqOE%rEB=G!bDytHd3CZ4F8k4D1`!E}2|86kCoIUpanSc~~bn z!$34L1a~yv8<~dxZW$lw+f}%Q!!#U*s~ozqgcjhUaqBs5Mcg(g594u#g^fw4xNNb* zrAc0JqJ;{E_*EeCOB|;^C>@TYkB-j@Jj38f*LN%?rIKGdep7lV?>DEzVlHm1Yb)V- z;z0-$c$NFo6-V(D8~Dm1xR=jDgS(x8W+sET^@RKH|NdlKQTW~UY+YmcW#hG+hV;|g z?Eo&vohQ4Wn~P~1vQgFTUKM0cOty@Vx9dX1lQrt@HG1xuj|+st5w9wcr}hF9xoV$# zk?->mi~EpwdfxW1UcNTG2lO6<0KQ~-$mTn~@4LSDJL8(VvA+6jMD)zstV9FzO%gYO z)s;3*>o_F;H}J$Xb93n7J@;b`rLr%3N5NKyd$cxpBR)+id=R3v=M7L9`V$XEtNFBr z5-bv^%C>WRj-QW^7(0^dccV!irtiGXk2}xvysi2-yVH9fK=BFXb`|rx()&BpBQ8U8 zvB3XAtaoO<@tH{UvM*=7wEDJMH#~yea+I12kQi?p6oGoGh?#y^CR%lAvC?sl(k zUz=;m$HjOTdZF7KB6k9-s2Bo`7OD14(G9+6jA~)vL_;)+&&SNVW4sWzHstSoZF778BdzI7z1}sI(#CK zc%iZX@jk&h`i7BmzoW)KKJM$j$hUn==kZ|4N8Ll0q1SL&bqONNmkU6H8UTPGBy}XZ zfbnJv;dlSwJNg!sDniwSiR-haulwbv|7vf#`nP|)FUI7@wnhZB)u}qZV|nQ3ztg8W zzykqCI3yN1m(U}$WHy~oXjD3N4y=ZOL7UZLx!tT)+$EO}=B+rOR3N1!}J6O!9fVYiVO&g#6?9#2A7El$w>!F%Sz14!-~wxK*I`9#!<&W1k=;h1=UYN z*H_p=*}+Cq+ge;kL{i*b#Ky)3fMMcd;t?0*z!BzV=V$2XlIGCJ>WM-ZIXiARfM?_X z@NjAJa^==lR8?{IVS?B8VOI1(`t*AIbY9@uSo?f{_ZIZ?7tr88fdv&NWO#5PL5KzBP)#ad_jM?Mm%O5v>(2&7Hr`uPsUc=r|qm_--v)clQk`?L~ zuwI}hS({sz?%la~B@J|2=+|4gr6~3#yy)*?!?xNA4Gb9W;l_>wN4Bt`a%ExhhMAkz z8JcI%p+{gNeJ#OQumqmD&|ECpvsnFPM;ytzwL8ADPO&54!uRiaBN;N9oYDCIMTy6e zFHg?gVe^8{rB8=W-MUGWB#vvu&>ee-12S8V7hj${2gaMyr|;h0{d@TF-ydCC-@_0j zQT6YCdbJPNGO)^HtE~k5O07D)q6)#W%IeB2Khlz8px6LYYyrA7=|n>gGYO=$#SqJ7 zun-eN>_ih2+sBuE5`oOc$%=y!uFNzOjWZlW!>bxBQ2PqFRBn66o>@kGF1Q4MJcWSU z0O;*WD5I28NhA>>jvv~D({j5bxAd}0Fu$yBOzXHT^UN+q5CJ=nNPvW%D%1JRy^>CH zz`XL-V@y6i`}FhAKP|JbJ_?lJ?@&aUxWWsoz!(rMsvafK(H?zFkefFD7{pY<1?PxR zEGyU|%rOh)(#x*A074bS5I2dD#kXR;5=L4tt2IT)D2q|DS9`tE)*9Q%@kU~!!SI@E z9I&*TA<<#7oi}I8QWfBy{4L6Bv+Yf@DY3Pd$}NlR)l4$WbxzDM(R@x_bK7P2T{2Hm z)1x&-cvGM_fus|p3G@W?Uw{L4PX!DJCb+wU3+7XW4elHCJ|qlHbmFM|8v{TC6yW1x zNzd?jQcxRIuq#aygpg!UQFd?*D{oQJN`&q?N1arZl&vrpjYM&0TzkgZWf9}DwK!jE z4W_kWnW32>Dt}v5*=6mZwr1OiVe(sVyVY82n{z{j>!!nA8s=aBu|${bv%|zU?R430 z`)!TNgnMqc(WY5jaL2yeHwXmw`)|MlmoEdPmdcM}#GiWftc@K_YtjJqC^>4fCg0RS z$w!8ijW-fntn5)3NSd>IS^0d_p-asf=(lcpQPy02mHky-YdPdH%Ph!D0AhIeT{WfE zj#-_1PK|(aIKz9g)vbe%LQpgEJo^z8~E>oHN)Bp_xz_mONXp9&g>ALoc5)EyE ztFsUV7bq^G%_?=PdyLM?<~!aMz#I;VUYrgPiAzZ0e{N|17KQNPJQz|fI4dGZ39t8x zqPKc}g8# zvWYTmAHo!tu=mCCm9u=GD`k00TILd#y5uD;;ln;E3LyztBxVRW#<7h}<%|6*5E)H& zIZN%YfMn^S8rN9C(7`c|a)iatR0lg-@C-$(`uYO)Mh)Ou>d!qa6`wN zce^^k2qEewqTqp_Gw#pdh4RE^@9Q&jgABzf1 zD8=L`8o8JTG-{=PRdFUX>sdyuXli8%YKULT);f1a14ys%=I!oyQw`>{gRu*pi}Ywa z^|VezieY3-i*?kZBDFKxSVJUF%Cd(_?@PXFsOLD@&#ER=tK=$WS4}y}t%mhW1l6Z+ z(pSq~rd5`Q-L5GCmwF=As()1(2cvuOH)UGm!_WOJ&8RAzf+Z00@X~_Jy2A zUCm6tDk^YtaDtPSYy@GYqMj~Nsf^uhXJs1yf`Tw>Yg6)U12vnYe-e}y8XRj@UHjT~ zJvO#N6l-k_>d4zlHJ>u%trO*w(f3dke#32LaNRdcx#lvt#VxLI|3p_}dNeEgn{Hn< z*4M~UcZ?xR8FvlKIZg#u3%h6w8)0O}H*$7n(SmGH*F%{)Zjh&d6l%{rILOl~QkzQ6 z@38t?+HxdSzyoGyfeozL6R;q`3tnV{8T{Zwu2(o$#VTycWm~XjI67YCu!S+KTZk@^ zx63^)T5)+|xT09aDQ0nU4>nBb*6*0r&Ffwj;DGB|7i208Y>)3S%`_7Bu9-V59$Zlv zY<5z;O5r7`=_Eygl4IFSUdR!L+dboZLT#T>SDJ~kY3ps|I%AR_lYL0AW z;Vmo||AJ&eB0|eej$lyfYe93mtVkn(Kr_JHWmJcm%&9i*sZ}jz2SZrFv6eNhYdvcQ z8}Y-t9tmyV++kqLR?cn)A6Y@n=iee%xyu%Avst|275`Gu>@&3erWv4*Vi#m#9BfJ7 z(8VnhR+=h1Am&c@vz%@ETZx5iihxSxDC>;X^Y${m?cLc`+dAL-cD1Q(T@Pd9+0Mky zEw3?r;bG5t;0qUcz@@imh@Y?jPtKh-z<|3;6AQYz75^7-B~b0eAv&ZWwQJ+HZJjDOb6|<@s<|9_j~VG`@FP37kVOZjca`;z3BK>ATf!7TtC;oO1AmH%y4`6jGIEs zcT+}WGrnA!rz@wZ)jiX~Nb8$dR%g8pPxQkFUFnNodgGBgx~xau>2V@B!?9lZ%V&P_ zn|HY6kG)&1?|FTHCcXM1pmxL&S;&lYT*u77@{|j<+jggPkp&xb^(>}?w%Bw)%X^I^ z&!QQnM*O_9Tzti=m*0&4XTIZ|e^hb+yz`pB`fN2UH-uN8`na|H^h3{pKfU$qB>uCr z15JM8w#NZVNIUAIf^AP}*J7^McHL($GP;xdWGAcr7gD6y3raj=p@>)$Km$ZRqI*6C zY(7{kK79H<&)Yr;gtMxfz^d{-&db0E)Ibr7I6dndKS8UmD?JkQo6~c@M9G|$tCVjO zki7yBM|&zU;v%%W4r}4@?oT0fv6hvEO zx*|C|M1+iw3CKoLggt5kFG!)kwhPC4vY1xfLx{9P9gMFCW4lMH0l|yoxU>p8olG{aE5ks` zxbwpR_Jc8o)JBqfNc$_tM)Nj`M38Z-L5#dXMf*JnLJ?W`NRiyikTglJEJ6XaNk$Y% zvb;o@gvn(@ODm*Fwk%6a6w4;Wo8EZFgse;SIm)%uq8Ws}sBFEbJgJP7JxQxHrTh!N zi?^ENO2zcSlZ?l&e9Xo~NwHK(%3LV5OvXw&OU}`MVQNi<>(9H(@Nl6%+Mf7;xt4+j7!jT%g$`e z&0NX<=Jd=;3`pi|M!3vG(X>nK1Q@)0zi8@;6`W0rj6EL_xr-!B1i?M>ad9}}5wM6^77#gIA(_iWD(jZDWZP7$rQ>P*h)w9o0BN&mde{ajHMjnM!- zQ5p@+?7UGw+0KgD&B5eGi7Z9&42z}2L6w`kl<9{LO;Y3|QTZfE`s~25gwp<;QYwAX zmV8butx_4)(ixRfo4n4S^vU#r#)?W!_h?WZjRNvKAnCfzA5BUN{mZE!QZkBA1yNG} zJT1N^WzyzT!bJ2^6x~u79aJuD(JVdGLhVvPeak>?RD;A(NNtZa?GNt4!62j1WD3$a zmAlz184wLZk8Dpq-BSfLJVjkp7PV4DT~+){R97X`hpSOW1e5_aPzBXVTP4E>ozx|` z1M!N%2boRn($otr*8Hm`ILK3F)kRX3Pe1KFS#{J?MOA8rRYSejeRR=Rtybm4M?j5M zaOEv9jnstXRVttyOD#w6{7YiB&>^*hPmNd3YF2tJ(LW7WZ{=2NRaJh~*0tQ$eeKq7 z9awy=8Ri?D4AZG4Q2AKMnXQ8HEKfB!9HCXgPQ0y#SZ^WEP9R>XA!j+Z^&fJxr=ZP$vG z-|Z5k`(;-A?O2aJ-~|3(4-Vg_17ErQ;QsC44tC(cVc!Tw+zF=O3bx=2{xW|{+YF9R z>|HVh4p$N$;2j3g5B|>{=3ybuVH2)S6b7936xb~1(vuY1{5=XST#Q6oc+`&UgYbwUB2aCI=~XN#`qmZC7t&E$>t;ft2h|90l)e+KAGAYFzO z0SpjL5IkmvZfWTSNl~@obY8E0cIVEG<#)d5Th1Viw&x+n=X(xmp{D4q189L^$dpcM zgidLNc4?<}gW;@Usg~$_t!Y^P=t=JAT(**HuI4<>X|VQStN!GX9$hl-4uM8NZ0Hk7 zdTO^GfSLALM;_|2&gZ(uYWoxhYQ*jia(-*4hU=+j z%j(;!j7?!n1Fw2YsOw^$4z>L>kd zo38Ajwrq^HAqwegUDoQW?rYz!ZP~7ASq*I*Rq1hl;M7)a|A=1hjV0>Y?(8eRY~6h< zs@iPd1!AX#Zs7jy$!1{Uw%S2v!Q@8d)n4bw*2*ObYHpoz+75Ci4;vg0Z5`k7oCWcSHt#&8aV>vxBJc7? zq#>*-awj)&F5mJk59IwW>nP`ND*te2y>fMq>oJ#c|IhAiF)z#9?rH{~?>UEL12*#~ z{|)0k%{4E*HkV&3XWZrnbQJIN3BU7c#Rom-bAc^$Mz?f8U$;U(;!{5K!?l8DF7U*= zbRmayQ%6|}>!c)C@+PPAo5pllhXgg>^txu8;Z}legSpW4>_i{V;Je>-2 zN*`HL2llZ}cDclx8>e-0W$9a&V8_m1QP*E(5B5rT^ixOGJg;u)KK4#-bY{hnIec$hXZg!Qg zaX@bN3#a%JRQHQdEJ*Ljm)QCa}^a1Z)c-}#gGb>Yr=n<>GwHfme1`Ay0A z66Y;6=Xa#9dXew>?<;w6M|L_!d0+4K6XkfA7yGdXd!zq){+9WRa(ZgNc{o0MU{83W zmwTPodaU32tDk$Tzi+h%c(y0jL}%knBzvkK`@KJWz3+2Mh$-`j}Myf{2-lP z&z=0jhk4j{{m1Wi=_CEn>-}Yy_}#bs|H@B(MWiyxTsoUiXmm3c%t4*q| zMX#~2vkSDgvbYGfxx2czzP-V}xv&fg!l21fyu;1JtkBQO)6vz`*4WurZrpF(-`;2o zeRnDt<|^sujbX!wPLLgh zNO7V?gBTBX^2hC88;>A6f*To*q{)&f-Hda`vZYHt5F!k?N3*8Qdi!$fdv&pA#)?24 z`cxQ{nZ%<)ivnAUROzQ{!ew#*JHrusr*sOD_*2ufJ<;uW+t5kdTtmor(WRhOa zT##u>njU|->sz<=-M6#T|JE*RNmIVuuZzcSO&oQx=hFvYFPuF<&zzCVrz4_m^ZRk1 zkKjpueE9!b0Vp7D6xBvpD^bZt;DQP==-`6`_UGS0(xGRedKgxSUS`D2=iwai;3r~D zbD1(mdl{xT;dm=X7$AiF5oL^v64uxvi`Aj%o{l{3xFd%j2B}s|_Kj$yB0Jf5qmoO$ zNF$6;HYnwkL{<6Zl~Fdy=%$xc#QCO}VHPT;nTV#vmYR(IaY-+HM(Sijdk$2oi&VZgBSM>AYGsz5 zhT12NaGq#tsi>wp|EQ`V!M9F)iK-O=nvISa9igk{II5()?h2|cz6L97YOEHUtD(r^ zn(P@2#Of?d1{7H#u{z=Wo|?!k6UP@lV^?dx#Alj@6aYzJ;P@x>Bn4DrL- z2z)HS9}j`>$e&5OoB;=*EW!aIw9K;02DsdE%MM^b*3CAzOR&cq``a_e5mWl}&_Mfa zbim3cy>rqDknFS_E6*IW)FMQ{;ni~Btaa8lcOA#pVA~8qvlEOBL9~GoU9{R5Yg|>H z`3CJa#um@L|2CILhb$&~?lt@L-+zaV_1S_TZTR7A-<|liK`pNM#m<}FVB>J{AcEy=T5TDOzVB2)^*8_kys8KfL(f`gNiDgNsvdv1)A`CZ91U}#T5yX@|rr9x+B%%%Tqg#YGrS zaZpU8;klaj#4@f$gj!6ZVJOH(sc=h(Ky0HNub9TiZSamt9F7_Hh=(W!0FU!hV?nM^ zM{L1SkcAW^A&-YaFpBY!VT4TcNWsTS&Vh%CM5K-AxH>7tk&|;gqysalzVhVpaFHAn z4lQ{=KYp=N?iWC==7^^iA!grqK0d4&S9(uoO_@O=QN--H@MDz%F~~|64f`WV811f^Pn>HWj?EkMu5)qlktowra0BkQ2z6x4mDam zpE(&ZO>?2@+9yZ*)x|(&lyMS0sY)LjPK3JDM>dS9Og%Ewf7G;>2m7cc)5+468j@ld z4XQ;e7)wTOG@SQ521x%VQfz|kmL>Hmzl7S;ssb;nNR8o3x4JVi=ya(c3Z_-ZDlDp| z)U0Tgs!@+RQbz(upeNsVLdnCHAKk{j6vQs!_@U_OvtnYG$E$*wIE-|9{9> z>{(~~SlBMpwNH&#Tkkq1rm1$d?$|7ESK(LQ78kcuMXqwIn$+f^mZpQnX-@gb)8(r5 zKeAnHZHKzu*LIepm3=N4fGZ#24!4oFI+SpU%i6T<_QAS+A%Rh<*M&^jyQH13csCr_=nj?$Bl|7F zs(WD!Gf==N4seMl2Tiwf_!}Z-Fb2#E;Tj)z#V79XopNmAQmW6uFSf7pVocus_87-0 zPO^%XeB{XjIl&0FoKj`n2O1yv$`Z~plFtXI1~z%icUz<>oh;-a2ie17|DZA-NDSr^ zGTFUvj&nBqLC?09nSg38nS-BPkBYcR#^rzM1*X%p+r->0snP4|9!yUb3Vk zEok}L_|nl_+GQ7g$%aB%X;^l2#IQ zZG|@p=U$ik*K@9Os8{`BGUeIInC`Qw0X^VcJ3F`I3NNBNe9&6&vC+@Y_MF-~X<&OB zzJ&&_iz7*8WG`>nr`EQ*mmTaDhI-n>ZZ%3i9BWw1n*jI)>#ftxZeD-;-@fj5X@|`o zgTlKE3ofj=lZ|eH2mH|wcX+ywd*fsuJl^8X_c^gG@goDglimJ!|36V}@sdZ`v?Z~& zK0V#LaF6_FU0%1$i)z<~U;O6wrVPDFa&M3PR^^G>cgMXe@}l2%<{A!EmJ=MNljHod zKxgQlO|I;r*QIn8JeOEp|9pjfvI4e1hbY{CzA^|tL+B;5hs(U@v>(TfVMVd;R z51QZK^pZQZUiPGC_3vO0yybST^PRIK>c|l~-%&`!PyKi8kY7A5OI~onUlZ{oZmr83 zF8O4LIlFn~apy_CdAff6^y3`tl+8%?(I*$E=!<-_W#4YM+y3oixwZ(yXm`BVagw`s zec%1w_6Wn>@~8(6;mg@-5o5k3iXXo8XN#x3`#a*ar|`>t|4e+RXS$=w$Ccc5xA4+) z-uIDRw$agld&(pK{R3M*X3*JM;r}rEJI4R?71@6C)0czBwsKQfcfP}Q?DsvzrGNI< zKjYSa@b`ecB!BeweDe`~3#eu|k4Jn1*-fgV^_lL_<1)U>_zU zMPRseK4k0CdYp|_2R|HzMz2#)@E2$4vU%m^mM7Lu5gC=RfY z)98@qm~}dtOm$L|{YV-dNsupyj!>C;F|}z3q=hTVg|nB9DF%n*7?4W&l>vze$#|6h z;*@6jkwaFM1K4s@XNR8nmJf-IaCw!9^p9$Sgv4cu9qE%%i7ZuUk|Q~PVP%zMscxVb zdReKJvv7=Hsg#G{n1mpZI$4f;32%nhmyPiuLAjP^NtAQxnakLcbm>hQ*(f>rnVor# ze7Ty+Gl3OIn4<}4@OYTsq?o+Im|+Q)H`$nB37eCNn$wt?|3^U2mx;Y4oJARzp?P%? z6`T0OF!WQ6mC2gaiJ7bhm?ddWnn4yS|5=!?H=EpMlrX7{yUCk237Izen*d~Ifp?kJ zS%bDFl&NWY$_bibXqyCfo@ug?(ute}`JUFPj)ZrEfasiQH=c-zpbC0q=GmJ|>7Wlv z2xIA;1s0qZ>Wai>ZN}-J7YdOEN}-)ln(hLe#|fY&8lZuhohsUgv(}x;$bq5>qw@G~ z589ymS)+#VqRUB{2YI3ZYKC`!h8_B!uvCW}`JsiB2>SUhBs!!|>7ybkpx9Y&K+0&o ziKG_hX!r>VB5I=!`lVocr9H@_PFjgXigB0N4jWpgZ0ey+ilue5q-t`STqij_ilR`O zrztw6e41uwRG@IWpr)pPHyWmL|7xg$K&R^2lxV4^?+K(r%AroVqmvb=GuNf_aHxzx zr+6u;Pl~6Y`cgH8g>}%5mb#@gI;J?fs;Y-dhxSD7nIiF|BoY`ux<@%!XfTQZVi2o{?ZaJOqd8``yml7zd@ke+C{~8D1>aO}h;FxhxrpXikf>ocOa0dxAO}O9b$kM)Ikrm9az1L>ybTCP|KC zT4M3&utn9Nj3BWq8>=l_mW?X3eOa{x+kS6)v3+~5T`L}6yR_6YNI>hUc#F0Iq_%-8 zCv7X9sJfdsIk~Tzo7d{LJ~pev3xTlM2uKE@=`LIS>w~V^2vdfx!tGSx1UekcJwmW<(+Z%m5B!@O~_|Et5ph%)`UsMyN6L40U0tio1&q`Zs7_cOSb zTa5VVz%d-gn+wEbydy=d#b~U&Bm9eyXsL%=z*2lr@>|6ZD8^u1q&1AfdF*#J$(4!- zzjr#uQ#{D^|4WO7+@~&e#%X-SjGPVJ$dv%Bsi{b@2av_WHo!aCx`YgZbZp0OOu&r{ z#iT;XJG{KpB9458q2gC&K+mV)||_{ zY|bKl%``E|;)uzajEAqxM96%_%6z;uJj+1i%f776q*2cNj34YAEg z&az|6=Zww{oz4#pp%7iUl-tmR%P{(^&HU%dyu<&T(iW}K zEL{)m{~W*ZthnBMod})K3QZwi9HuKh(Gktl5DlhY+P^J5)Ny;#mF%}jorcD2(8@E< z!aS+=Owz%uO^k8U{(Q! zz12IN)w(;^Ud^)^eaEc()Wm#lto+xILB}Bt*qhkE^=ZjwO)RB+*NolRj{Vr`4Ax;S z)~;OEXZ@`Hs?~A5*>Bz1oW0gSea=b?*`%$`FTK=bZNM+()Ev#ghz-myC(WjO)tYVB z6>ZzKovFB;+oj#xzU`HhOxZqa**XS$!tl6_oY|l4+04D%&fVP54cfQ8+tW?kzs=aE z|4rD=n=JARSc0wE1}W9pk|4)T-b(@7B@GSIY~4a#-B^v=?Hk&j4d2&Y-}Y@2_$bg3 zIK(LWm&E;@2YMSbvI_%V;O6~!1a^;V{oD*5-45R15T4Gu44LvB;m)nw_nqN|kcz22 zZLD3`+Em^KKH>!4c(1KM@vz|T-P80vxfou~4<6$d4!14t;u?PArQOcL-J1UmxEZqm zW)tK>KIBAh;A*AdPAua}KI2R-!V}Gy`#Rx~?9j@M!&AQG4_@Oqp5YuW(t<2b-Q7MP zF5c*;*DquYMV{sipyno$*n7R+@15gxjpa@L+0(qs6K>mEzUO@Ih%?>cvDV{I{|w}6 zUg&8~N>$F{bx!AuzUYoF<#8R!_;}@c-sp|q=a^m-9KPcxG3Isp&YU$_gl_1h9^~EI zLm6GyTCV53{m_*D>acF9~&7e+=j)A?Tk~>O*b;W+UuFeo=`|>&PDC zS3c=E-0T4j*3iz*%`WSe4&|{vvHyGR+P>|{Ugx%+>)^h`s12&$ox#dENx)v}kaEdM z&CFlI&MD2_x8CZ~J>%9+?cI*=wl44C-tVW3HipdKhrEZ8;p@d6=*ys_rCEHzvTY@@h7wE&|B_&y|AO6?uOpxdyQ^c|B&jCP4Dww z@iO1>Ghg#>{p}#1^Z%~Vj%x63ZWszb>_Q*(LeKCH?=46U$#Wj_G~e_#57ZgY@!QVr zQ(yJiZuMB7_1gaQqmA=A|MfVj(RmBfWbBq=pbKk`@@&5Hly&D|AKy$L_b{K^PG9$2 zfA?>n_c!i{1OMk{Tz}2m?uU)VYZUB5PxOd?`2Q7c{`%r`zw~w=`FOwRQvd1^uJxFo z^^^bjdf)lYq2tI$@?{q0+WkW4p7v_L_H6$Ri>>#ZFZY`t`;tHVvd`+CfBXN;!yP+# zifrbqkKUbt`bLlV!*BF4+62d56R*GgwNLxbfBDs(`Ltg7(r@`u|9}0K`|bRmyDxt0 zxPRB6e@~+Sx+0DGsbAzxum;oo{c^tVHBR>yJ@MVY{mC|FW1`N{LK&b zS6}_qPxF~>{S8k2N_^W%tG!1e*ZBYaB2Oseud$MW_UCU5`>5Rbf8Wes{|sK?H;?A^65ETm<^%ftUASELy2Mr0CGBr0jn>s5&Cq+X^ zM@<7uPcJ{2pz=^*V*^!FX%|y#Wp#IXdwqX_gN28Qi+jU_ux^c+o1F#I?(C$eqd%Xq zv$bPioo#c!Pl1=oVZ(V@S4nXr5z*PU*LTauE8XMq^Y!=n`~Cj`7Bf?gAT@&5;?a}0 zaG^qW^3FvBH!)l~i5Dqmlvv7DCyy8#J*tp_4!3S5L28^8>fXbEFJZ=%DKiKFA^8La zJOD5NEC2ui000C40ssd8K*PX56b_Eaq_W6t3X4Nx^r<`^n#X{4TYWme*D$ln)soNI za(Znx4yWLFxJ<8?-S)hG?+qUme}D=OgMfR7h>41ebViMRL=F>^5gC@3n3&*5k`G%28Gkw=;`X~?CtLF@bQOIP*!O6V)=4lOIYMdRQ5H|7x2wL zdkBpw{1;H*z(`^i&Z{SHo*zOOy$L&4u_GgjNmdPFb!iu`TPIVZT&c1p%a<)u1Vu~rwi3+HAjLoL~vN72~ItV z9ZR;X*|Y8Xbd^&mqpv`M;K~)$Bx6K4VB+eX+qMwFHx9=XY$%Yg+_!H1;tgCb?l2^S z0UMmmMj|n9N;5y${J=oxf@j>|xkoyi_KdnihJ{!k>T!`U0SX7E0)$h9-*WqKY=k=%bEC3hAVh zR!Zrgky@%LriFG|=m(&N`hf?cmTKy$qe_V8o)K2a9;&6T+9ri+x>=!~cv65~t+;ZQ zXPRsRJLjx>=1Ktvp?ZKGgvf&W-I=OBNUX7W`fBa9*y5S(w%m5>?YH1+o9(pWmTT_0 z)S~~hP6k}7I-u-44%#%EG+ z0jbhKC^M|BiYx7~AP2iFeCk?Abh^DZn<}zDOSh}e(DK^x)mUe(_13d;?RB+Zciiq5 zT1bKK!a5F_@V{r%ONqb-*Sj{9he=7d!gl|Qu$4DP%&?MN-VKqLjitR*UXlQnu`*K& z8}hUqTg^1&li$he%P?b}`R1B;&bg+ZhAw)irb1Zr=~7n+?Sspv`XJAvp6Y?*b2i@p zXN1byYPIS^_j;|)79ve~vApUWC+!cWx-)yy)mruAVMj0h^w3lPG40id&91|9yxVr( zV*<$~+dtyiW5Iuuza+q@pnv!Kb+f(q+-Mu#H(Y~5>}A@3aBs2VjI$lqiiSPoWshqQ zI+OxE$H31u@PQ7D;K@W)8|j?RW@nPx>o}mRR&CIO7ED&re)YQ8<*srgsjqwu>Knh}Ha{)(Wrg z#PKl^GwXvN-N^T_#A(8d0W@H@Y*;5TQj3f`w3T#tLz@n$OoDKPpd3+_DNfn{adddh zqaI%n!VAW(RHhqM&m?5D<#CWtrfZ(#fVDdf%#4s|EaM@6)iVx4Xk!E<;jEUGyPP4? zcDEXy@6y;q9(D~~ktAg)bJ#9@WZ?ryx74||-s9=r3b#1(;M6Ci{$pwqH(>mVm30-*h>*x%6^>(8OiH!1zoD)iaa-1wCL)lO`HN>Z>0DShG- zqG6J`QYUiHnHWsU$5a$W}2H zvKC5h@q8AcF*;JJXbSC`WLD2bwlKBufoZT#``F(4_P4;j7Wc?jBLfI_h+`@uNi>?# zz%{9^%r)s=okZ7<>W!Al)e#jjrc(H!RHEHI%xpqY0oAH=l<1ZJDfDLQJma~gF&ogV zd@)O3pxXDoqNC?fWyeX-3UFzvbC$9oblPKiP*4X%hEJFB$brMR|4E=A*-zUo#Ku!?D>UI&}k=GN=4eBIJztLiFetR$~OVM_bg2O%j{f8amFyn7*xh_0pvYP?pJ{cCx7dJq4;e;`dXq6J${*9K5A+ zIlXLx+JF;0JS&?~0vViSu6X8EHSKI_>|8Hofw$EVXBy7PPBw^*Wny<-B&CNADWapR zyD-PXh<({HbF_OQ@Q$Zp*8$>utwpn0ZajF zW62v=a6rB32-JH!{nVq=cMD2=-%{7ioU_9)ms?%`YGN%}=qrowsYBWHr=Or7arc1?OI3y)HkwjD2w?_4 zAb5uS`e}FdJIxdH*lf4E@P}_?$EMX?H5x!L{m%Tr2418XU7s8lbJ+OFve1A%^l|kp z?U^Hf(MaDb&dWsWwr85X1mCnE0Je-VWwZ3~e|pNAj(So3bn5+<9e>qL)t#w2_;2EC zxkfYTBsI13StgWDZbEASsA*kwWLmX()I)dw)a%Mb_3B2f!w2V|k{aZo!3qjx{dDB3bNL37{8$L@0j| zbSfn?C_uwJJY#7Ra7ni#T9&gjXou(@6Z(IG_Yiy%uq;#xt0-LAFLGtn-Ms=#l|QkOoPWxoB(|ND=#! zm0lu!)1{Hj#gP^n7*pU;(zT5Li>GKh29jJ!OyPra*072v2|yC4IsnOLH1#HfGk!8T zgg5z@HF+sNV}F=}I=EwA8uUWu1b33ugba9VQWz|7Vwb+=Gx4Y^xR*jY2!V6AZn0OD zpc$IkGG`rUk&(t{80B(qqY*F$gTI)XiKl|!7M9(2J{#Fqa0v)kl^&D!Oi`J3k?DX^ z*?ZW?X+3cXG`T672$(jxoImA9801N0#(IC0D#o&4uvkd1^Hp0oY~Cq$Ni{OFbChTo zR*cwHE@YLWsb=3;l_55k{!p45_gr(8c!e{P-c~;FNj4+ckf!OM(T7Ao#*Axen{8Q* z!jPM~iHeGalD;=_@Wz?{=jkRFkrK(toay*Y7+Obz37w^dVN|0$trJ4yX-z;`D^yi` zzQ&I+g_+=4h&p3?j`&;Uxt_`Pb_!*qVxxw1wvY^&OIZY#eE}C08HrA$ZLevcDQIXQ z=!Y)23#M<^Uzhn+>{|D~VvjV=W^Tj%P$XMA{QRaiJR;m`sP0&iRSZ87Kml zTCle~&H^ft*Jy_f-mJit7227AB~H>SjDxs2KN}K-3KgnUVBaq~3L( z;Wed(r9XyemX3;+&_yp|8FYxv*Lve>>fncF%+Nx(NDX6lEuESbh zmWr|XCQe8-N3&-CMrn`6c8eqSn>yN=!^%*k8AS5tKB$?j8VQVycbb0&gLcSr8!3@b zYDGeHXo`cMOG>55cqG_}9zuo;fjX#LT9V>82xMvrX1b=YI%@73uXr?I07qei^ppC; za0@tqtks?SWLh;us8k82<|?Scb*gd)tT|e&qp7G@(2Jj%a+Dg3kp`QMmSUiKsoVyt z$~v1jvZq=5s_#0puF61tBvk^Kn4(CYiDW?- zCWWNfr4z`oh8l)MJA8Ghm5i#X5JjyrICw#Nmd#eB&AN=)=W!nSk!%@=4#};0fsFul zA(mHiw?~lw!+EagsTDr4wxn0HG%K%g3y+_~I|%rHMCUY6l{!e(o_HxITAHe7D{QwK ztcZHFE(>|QNN0R^n-58?`GUAcO0f{5u{1}G9q6&E33A*;jEyFK5qpjJSgM-~2!#uL zX&bY(P#bUSwydk16SgOrX;oCFlz^I#p#m*F!&#|ChO_96FFUxUYq-KIT+}BJYglrG zh9e&6v^B@9%?qjc<1nIHwXw;Of2Xx`NojOQT{q{okG7OlGq&_0xHBrS=G#@Py0+}9 zrf}Q3?aRLH+jQAUnfR)OIBS&iBxN-7p+$D6#bPJO5}d>;oag&{!8^Rgibe;jw2RBM z9mT!>k4s}Mwx5(6sUr)mn9G9B7l%8>ybp&^yJP3bkFlaTOer zmg{0yyJw3AbBoio$VEg?Bqaanb^k-De~fGe3zHQiX)El;u)LxCg--5da599uj+x5; zZQ_5RWraT(Q;is&u2N>Nd&tI|$cjv~?|H0Ar^G^ z?a|u|%CdP;pZ#o%Cz9SknGIHQScIc|3!&-knc|qKeeo6T3fHaO&gd<_u${ZCQ^tO4 z&r79YUDmtx9W;})Qa{D`n{gukqCk>=)$C#8I8uQ?1SageTn9XU$X` zpl4&;A^pMCeV?8kgBgL_zoTxHO|Fl5!s>$648(OG%*=+w-LX=PsSymMLw?m0e%y~t;VyXLUfsybgy0p zf^hzAo-c{ycvuc7uHx!F=IgzjfaIYXT{qUM9Nt^5;|USwzEeN5sLGB)1|Uu~P> zp59FzP&ysT;y#Ph4oKGo%vGhluyo$)tp3{>K7D&G=$d`xm5bc}{PM(+jJOfL#7aJ~ z(e1^R3gr$dvI_2?ZdcdmxlWdDx@=;p-?|FtP3C1z?bUwmc@*mUC)ZtAKn3>QZJyuK zxzZ9U=LS3H0dBMxXu!cD(O1AG2O80go6(iasLp(C5ANu_F5zB|(TmRLB5mMgf+w=@tjCp&1D{RFx+nZ5Tmt=rvZOBFI>LxEr zNNg=X-iI4$>)&VV%6+8$p5(tSY6?v@w3!5c5$m!5Ws49IC52$0~cNtmWd zZ|zKv?M}ar77s{FMYpjQutX1FmdWnEvTC9->HkdFT5s6@CU3YXZy*huxbnGI{$BH1 zQQS`2B@n&ulFZ=1{%ruS^8!z@8=jFq?|c4y$P5q2hgAV#PVp1}GN_Vj&^c>aU*GMy zRD6xc{`%_R4feq+_G1rc`q8jtIqPakZPY8;_dd$Y&F|Cum3^L?9*gsxdgL@8epBAb zldkN?O!-_+^a*MWl5Kg@{`9{8`|`SkqkiUk-1xPYGdN>rvPIgI&&&M`w&;HKL^s&x zZd>Y}`4?yT;S%qAZmq1($(4Lp#qQwTTIg~=_x-Nuc6IlY;ES)n_s4GV9!|tZiE4_#opz_1DOSq~i^1V5nN8k~ zv+?$vI;UIZ>-%2cxBCJ4{ssvP4G$3$6&D#B9Umbd4JIioEe!?&0wxC>6h1)@LOn@A zOCuamNmE%TR8d`DRZUw*Z9idUOGSBmdo2%udv9%hjgN^>YjtCDUul<_BPw2ngRiZ5 zcY$wn#o*$)3iBe#0}aj30V(S6B)mK)L$k3JDq8UIU-tR+`~Up~1UQgjL4yYcBK)_I zVMB)xA@*Z3@re_PLZW1RB93E~3reCSL1X3{9*}9iywPyz0=RbR)WLKoljcmDHF1{H zxszv4pFe>HB~yaACrS4JSst*xyBrQ659V*aQj5 zj3K+g%_7F)lPw}Uel!!B0=PwJgvMm4I(2K;uVKfQy(nu|q)3%*rRsDy*r>UEH&fNy zD>vQqh#wFB+ZDA~zL&RpUfe6R>I$4qsPOIOIFH}p=nU6!QO>%V?NZQtHu-_Ris3Vs zFF0a-`}gtZ-$xO$6wXlo8$t@W{tIafl#(zaiqE*n?4y=YBapmA-e7Gt?^H7{y$daz z&BG5t3=tBvD$#@zvw(vRxnii>ZJX%-^t$R2zK$btDHmZxF+~-pld(l@)_Nrk^wePk zrMVPAQoQpvw23_jvUzfY_^6!E$||wEQp?2nGcb$>TLQ_ykvO2Ni7&wuGYSX6i;$xN zyF-K>5$UYc&O7nUbC4PNG-{JzvSaSIvUFteM&+2HgZ}D=X3{;x>C# zv!aW?5%9~6s0cIJGWv^C+mT1bcI18Svc$leetUu-DS7N z>mUU;W^^T$S*@P8+85_9@+dh(kxkB!4hKomBGt@X-HTCj9jE}-hjFD9*Rjbad%*++ zyzCp$fK{*}lGqU`O))bY&?7e0HqEZsr0tuh@%CO>@My0c)#<_i@bMB8<@t8pWyBS_ zUdX3XT5)~R`k7lC2^Fiy9f7kK^UE{MH%Pq!pEl*EB>=^Zo5a}*N~Q)lbQ*`j&K-B% z7k(RU%7$foSp^qlnw>NMl+~zs(zuhtZa-Fv8gar2QF;37{k0x@GFrCeWdm$BI{cf5 z9^Z2vcfNOD%XwSBz39xJZgLdW=TuS4WtsDM2?O?yNS|bKE!9y#6$2yBc)D18wTp;Q*MeHuz0?>tNybU?{au z1u%v~>0XKcq8IRDICrk!}AsJ18Bz7it;7wU(T7vOFwkBx}!ip)R zUJU`cvi<=MYJp_`!wo6%JsvJneC6ArBg+-UCB6@6?Kxi~KX=4bxT009QW`1nC$Hvc zCjv}EjUFHbA=Lfvk1ZVF1r)%=nGi_~Uku~9D&UtgPLP+q^kuGuu%F8i4}))n!m>K! zJZ7cgcT8C3czzWqB9u59HrubE9lZqtSO<7VQx#kjmYGJcVJ+z&0~$<0+#okdJv z=LQ9@Bt~+br5oKmqTw7*F%NoZKvrq|Ny9wpkD#;UNJ~znm0%zs10B%iF3%WIi1x0S zkikgX&a^hQnIUb;LY`zcsK4jIs;x|#6Qk8BCk==qBBVkHTNMe$5lceOkZu-tk z4vnVf)7j7e(q}(-jnX+)EGX>w`A_XFHG5oi+Wx3!FoT660S}$1VSbratun-!AVdPQ z;J7?BL1t~qD<*_S!@s*RPYgCuVQWgd(g9jiu5`8QY(`fbMandM@I>cdiFQj)BywHs zl%zX@CbW3;G<-k>>Q9HdNqusMkSYX&TI0}1*2yweGYP{5Oy{b#SoNx2+3y*i@;2f7?ypj!cNcJuV{wOHRot1rrsn-GytzzQfj2 zH-4pOPB+Y9u-wv|(;($`C%{Sy5%i>iJP)IU>HyNBmd2*F@n75ql8Iz6V)~`8MQ6Ll zizu^LVUS~67o1=vFS)MFZP{MO7iCI4n6J`J?qD&z+#xy^u{sTxb(7(!c6KGPMAc3d zVWC-30W?B@%n7T&n5h}Vm(F$0>Q~tqvB_NUtHmR7F;&|ijJ&}!b{nCZ@0w%*GMUkJ z8l<)IY8!D@*~;fsEMQk;VNEv|r}gt0y@E615`S6L6wdIe;Vixrqp*744KJRML}J1J zX3Bv#wsnnh{Q?Z8%~y8hb1_v?CO|tz&^?~mXt03fMk{*R&DK_MkJ7mEq1d>Uu6CiO zqGc^VSHmHOZgi;(;hI7g%)Fpl*G{}@JQnzbF*Jv9&}+Fl+nL_=9-FQ@YEdNc2X0;! z@{slYn2T|&G_RHof}K6rgkLYxe3~@PkQ-r3WBO>O-gLtyIc83KnR1nmICihx>2}=@ zfnuQc1i}yjgS$15x7;?+YE7kF-ME~^tt-y(+245nY8}(MaG?(!rB2p1VWa)h zaC>~|O$Q>zw@k#SE6~H#owkWlTy=6EM=D0;44O~AMe415vuq&oj7C9g_~ax19~p?b zz2UA^Bj7{6b@#P1#bEcFBf-B0;;e$1?Lwjle(+6AG*2+@y@vlz&)9>fKjZ1y{9Dn}2%zX0aY(C7TKA$T- zds!%+nUuCjou0_PRss@{UBjhM2QCE-ugyE|^~cri@t^aZ4|~X!$qZ$yKm!1bS_qH$ z4DbNkAn|bMPln4<775%SkNeVY@=z_~Jnh@G?)tWG+{DcKqG8>rCf&aO;`|D5v)YB* za7i#aK;%B*9%}9dPb>fMujkGtZ3d3_IOu@7z(95j0e$cXfdb1a%;+49`H*PhjF1D< zMPDcn0~M(H=0^jwFFQsJn7C=PTe{)8&^-fkb@&jMA? z{@n21WbpQAkfQ|d-<+uqg9Sl$P`8k+1k273Ne>7OPyrKZ`8sams!-w{?`N7&p)7Ce zJWvXuZ=SBubFvPBrp(+}&Pe`ld*Z+f-KP?GLE641Al48C-2+x=4+haF4(0F$2TcI~ zP1pov$L=UhPJ#?XBM{+A5Ph-eDDdbw!4M-b10@mSDDj@Y?Gd~G&HKLZ>ZWjJaHjco z!5QI4yJF$CMCTK}i;q%i6UPv=s1S_eExkzL?bfi9Ji+Z+uc6-X9Vy5aZBMK=Xy=lN zn08FZ8l-sKVNHJV9|3YiTIx13VZ%U-iMlVk5>W}wr4bzwAw`TTjFG3rE%G?;x;zH} zkBXCQX-K?LGX&~n!lAkpM-)dv74ZS~P%p;1W))fSjO5Yo3@x*KtR?pB?f@=%QliC_ z#rOj9CxP;if^i5fY|>UTxDbh*Adn#qrOPx262t5WjZ6D-#u__MyV_(sItc7Qp&S3; z3{)&69SsakZo9VbU>Ycj&};%t@+;C19^rC7VlpR{ikUqBiUM@(CTlSt*=CjQ5)gwj zFca(mgwf$1jt~*4Fcq_~B9SQ>g(y*x1url4CbH?M(J?#l8mZDv#;^PY%7oId7gYss zvX0!)k{PMOs$6n4A7(D|F(tf&q~>T3_p&zmk~8YEj+jL>Tg|9uK`?=HWrQ%xh%)Jh z>nJC4MhX)tSyBR@f-yghEdWqJufL(PG0`R5(k<+!5i_l037=8>v{5Zlt|n9h(X1uF zybkonvC}?D)+}&ETyQbs(%xKiFTn~7en+G}Yi{(i?)=P5U^5R_$eLU&1qif24fH?{ zR0DD$I2rWW2J$fVN;yxBI4N`}E3`rrs@$#t!zxfc5z`3?|D_5wPdm#E3%fAgE-L_6 zO+4F-Z(8ip&f%#_t~1Gn87pUrA^;`%!J+7nHGPyI=97b5Lcjhmc>v@i_Os_I<9B>d zL1rjSS_nZ2bOof8N>|`Yv6M&N&`HN2WsBpr~*;^#E^;z3?<5PQase2Yzk0j z%1;|FT4{G0ncg%s{X+r)bT*i5J=I zl$h#czM&mU(=<6OGFwm}=IEaZ@%WL>8S1 zt#B{UL@L=xNcWcKRR7>!vov23bX8&X3jsE2@hUOP6i5Vixo%WfwU$khk2v8?V$t*h zjrCnZR9Q3cxG+{ScP3*&&kXZ#dcXjtOpO!pb zRH@!d7im<1$&YWl)keiu#?+8z4Yz~U|B&uxFlRY7wsg`CTdKsYGWtX@YoU%iU$=j`Y))s_b`KJO!AwLgI6OfT zsWwR?Kje7+31B8z)xeWw)zXW}r+JUKbo5~Za!G8WH$U_tCg*j~L?ag4=4fZLG~5*< zz(8Z_xcB535By1w3=9xtxN|vIU#XN!RnG8oIFY%BYI}GB7n4Jo*eW>`P8stAp)x}) zuL5QFb$?a*y0(7fH)9>^s7nF|HR)kXaL;-HnBAl?sbM~xR4iAk*&EL6jn{EmK8AW zt`2WZ(FWSJ=d7t5=xa=q^DbS|4Oumuir! zR9{It1*-X)QMwTxw2`g03BA^KlQEnP_F$7z@(h{~6_Dc=@{>FHxjOl$>(_qi8O6Mk z4Bok(MKdf>XpR(g-SD{!qByGgmY`A3f5DaYCLou~^#wWrG7Q736G8^Z3P8wuCe11` zhz)KKWI-%9q&#XskQ8(C|M{u^n2J7EnmgBqQ+lsyg9vZ6k<+E79WQhuk0sApr)GK? z4|`&9cYqm>oq2UlL)qaH)@zkcpwY~V-$9_2Ej3c?0n^rpO|o(-`2KbgF5nEAqnqiZV!jQr!$n?9n*0Ql%W;T-! zq*@%{3TaJ>MH_VraY3ptSg;nmxm+7dT7XAquXvJx?eHOQj_Cs&itQMjKl5v&O=xjP z>i6_bNzEFIze9wL|1^5um}4W@UzQuM4^*$C`@{($o7oh~g!qQOnXvtvoL$%AQddrE znlhs>jE@*BJh{EO7U`~A+HA%<+t-4*6{HnT{T#SSRCa!CMdH%zL4CKHQ+%Iwtwv6q*%=&}aoCcW&xSut zn>!Xz+?toGNTz$7I7w4H`MRx;63My6XFReOwur;qDHF2lKrGwT`_j`p)BA+IC&7_Y zt{LQ3%B9#R7wx{6yl^f+d52mSME%KS92LOpct4`H zCt{7K|Ch(|%(Wz*gp{YK7Z+{{H9AvZqnXr3uw5xb@B~pvca?sWt1Pv4W2pz6l;XtGqgH zXV!15tDotW=^FQ68O+&k*clqfUQyW-x5+DJq&)JYaZ^U~d|F%;eFa%doqO;BUFm;( z)0fZCA9)CwGNvDY>a^Qsg1oyCo3#-8Qj`%I2Q`LEpfx& z|MGR5PVsco9!_RcHh%;P4iXj$?jh>YX)v~}X>e%RJ=k4HQZhDP%5l;?aDs+@j*60+ zj;5@vq_DQSzPhTeUPL50oZ{x{?(+8f{sIRJ4-*#~A0sF4It?^8Gebu+2T(&z)GB1# zGD}G>)n40bi<6I=XMdA-Go53juSXAfPtm{I!{bxkm-E|(gDW{FuwA$&@(vn6s4yWn zb_(V-WOrmDk_g1u)XG3HT&6h6N`+b^L?ghG;EHTaQbFSoe=JwRe3|lIzIs#~RQRI> zi# zwVs@VB+`hQcRH2ik>%xQSmJ&{w8H2L9W@BZwMdZ+PPx9DfT6zK*0e*-gso#EU zNSPe2RkAjbHakk`5U8S=#w8fizF936A7p~anQNxWEuKey%oW#v^{T>3p^d`JufP8Syx*hoRKqG|DOzaSKas%*|Kf)oh6*FX1LrvF zsHIknYGh3^>@Yu2&gki8CfarYvbcuhgi?|u>1R~u@hY!y^*QHcZb6=0g_S$Egz>Wy zV&~Lx52H&7Hi{=(6%+>51PAE6)Gqeu)Q5CgPS8FuWN{=$+?6lwKW*$&S z-MiF?4AlX6OMe$g=e2vSAiZM4wY>9j2?(6q(^Fr)^>4!s|KZ#Pxnp62mSUKxrs@(V*epsnQ`VDm zHWR|JFB=sQ+6yQcH8AC2EKE^a9nN&MJ6!+|-fj5!{Y=hg0Q1qgJ*zYmoTjGWeR=$25Ac>#SU&P{8joGj- zV;Xwj{N!gVhE2n945F2@B)32Q5#b@28z3;irWZyi2PUE$o#{^Zn<}}aSe46+>dKb^ z*O~BK*sews0 z2r<4r?4pB#RYa-inNewPa=Z*CLk&Q__JNV3uX3Dq%y>pLLeziVY9kC4v@|DefQu#} zUFq~_y8cZpA`y&08D5tiaUlYDT*z9|(6Y`B|IYAb9Drv$t6I-Abfl2|?B{+sX~VC2 za-aiBC|S#TP*FaFa>9(w_CWMKV#d;yOf+A_CKJ(0;c`U~+N(!7)yu@gKb*f2hBr3rJ0OJk(W3GJ`Akkx=sN0L*@ zo~)L$RYNNS;HUyNi?EHPE#y^x(vgDSTkOcvhJ zHqB}-s?`>ZF)Q`q$SxUZq%>t4^- zV)re^q~-NuGF_G4*SvTNfEKA*ugRKvsJKXI)9GY&nqN;lS--7e@x*T(nTiKZODR}1 zrEod#a(#3OE;LQ3wpYN ze9B(FcbTAuciRETZ09mrCjKV!yZgKs}ZR+VZ{@P;1w5PfqMVy1z z(U<|zM6oe9N-qoOl;1u}Bd2J5k9cxuT)L9#y-!L+{^@cwb?HxzYCN7Er-%cIoRm#u zKxh?QmdGAR)YSd$lf;<^1!O=}phw!Fof#y;+ab>eDv$9M+p#IZ@`)gFHJj=6++i5j z^aY+=g^zUk+v)ku|E&EP<&n|(*aJx6V0dL4&*d2P8J)Hz8oL?Z`;{Q-TokLI4g+f7 z=1?FUu?(N>RKDpOsu`jGjb4z&T%@f7{cPR=o`Kad(*n+(7%rd(E)<^!3Kv0*U>+KjyH!pVz6dy5O z7f6Ml&h4D3ou7HNh$D*8r&LCJJq#88o2@L+*R4!Gfy5FhNhZ`0;L#CDK;Ef6*$Y~e z0BYeG#gwg#7RE%J9G2qB$P2Em3m#mc1%`n=Io6)!8ZRnC9oiv>VS^q*qxD1=C_*6^ zT1uj+pbNHO{}dV`|IJm5TpJBi(OaPs_+b$xUYGUtpbvf`5O&`qR>9@1-l<(lWjtS+ z*bx!r7DN$;$)KSL;umg+9yLB+lu6ld&?1jvTdEkH*U41uvC~tCg$6PlL@o_gv0+AP z0dBpPw=A8R*#$G&A@V)r2qKF8MdLGxg2{=_%_wA05fS4_$GBA+B4S@9-r`1?(RO(V z4lYtbj_G70_9UnAVcz}H54zF=xFAq!|5U0+^A5q9HwAs)BkNTm$UAv)q_5=LYICE*p|bvY(y z#+W|fTn>h1z~N*~(xi5BXNaht7C|2h2BH0!R3B=lOx@iVapN{dp;qdjY|iFaQexv> zhUO8D18z_u84y-MTy?q(2b>*S#>at?Q^vhrU2=k7IA`2J=PlHoEAXXt21chLB#X)8 z|FpdrVIG8GS{hzO!;N*c;!P%e8qR#Os5Yq|`W>8NIMF`Y4X@ zWaKcKds62331x?n+JJ^>ja3)s@fqXDs3Sh9XISYD4rq4n-&SUxOI9P&^^r*ZjDw1X z=UJqm7MGW0=rC@iRv4q4euptqq_6>0GxF()0;-_i1rqij%K6u+ip*$Qt53F-|3a=0 zcf}xj)@BnuQ8>b6;9zTz0$flQVpPhal2(y`CguuWPS8oIZKUR`aDvmd>a1qvt8Qtm z`r*`F16UI2m^R+7u8Ac8Cxj|&!u@I^84Xt5)0`4(*d-?2XG8*)FLUZ2C#}~gaWeG znkHj3$u4Vgh3@^@X&nX2G+}s%8Rr-hp0W{uU@+W9=g4)i2jfC?L0R)YFN{)aN2Tng z^(`T$+AvD!-f%ou9z@AfJhLohsrmN1S%ooEouNHBxJGCyIk zaP^SfHY*&P0uu|Syppi}p71g=^9`r)OU9(&(p5u@s^Nlb|M%KpGe;@bJ+6vDat|Z3 z9iytrh_ChTu=i%Nxl(dOaIQmiLJ6<4H(;eeI&pzQ@##+S&{naER3sMvVlT_?B9PF- z>cqo_u`b_JF84D)znSp(gMb9&LJApeaD!z?%JCLhN%QVt9|2(^$NawCsd|IhKT^)B^bwwp?)BVA9MCd04( zv@<&pN9O+Iv>B~$kgieR^VAA0kxs`x=R@mCbuYsf8d6{jRfUJ{Zfcw2LT_y>*X48q z*9{4ZSZy%LZ4!!_wP2{VGKgnkmarZJH)*EtM{8^Kiu7P7-e#Lm<|0}k*2+$6DpQ{? zP0~%Meo=|7us9pjHM%f(YT|c+U(U*NZd_huqi}ns7*WF}DYqt4Q(-Dg_qjg701K7D zg6y#9)>%fCck{CiHur9)|KhSROry2;F!qIycRhy2I}e6AY6fPH z?p0N?Q(Xd^_<`%`qE`fhlETrgt=R)zy?buxT zgCBzuCv!(%s%FG>Zx1zgL26^%^`nJM9y@m>vvhQCst(&GA%|xWR&sapoSL6^cu#gT zpd59WLS+6qLU^NnnQ~GynS0B3qaOf$M?(T*`J}^3SYRlZL9CadcG>zfyv+76FG?_T z@W*=jU7#0*qq=r+RHf!zj)l0%vU3Z|tcqL1H#cS5& zI?R~AW1f4|;C{Ej%lXRI(NtQg35V2X9D0+ncYAw0(mA@$J#{%FdF%$vf}N=(K+Ss) z(!^?jy&P*AlTDylG^O`~sK0wF#QR++>L&Vp%bsINx4L(mdBSWsq4E0*lPV-4?KWG7 zvgfRcduu@G)FAUHG|y|Tx4300J&eaVPfsb01L!CCu&T?mdKdM+o^qE8tX+SDkON}y zfp*CQZ(4$TYlVj&6zjw8!3wCr|K9_?3aG+|Zm~bjJk1w4qwI@B+dM9S`pzcW>Fc#JdB@@4=Xfl6 z;3?yB|4fAf`x+D^gsaGS|1&HUoO5J+lxvD~l$O1qX!-c?DCW@6xX@S`ZXRA)d428WmA#d?NYPGN+1Bpf zp5`6DCO6L|EI(gor*3~=4~L(pueZO)fB*C2-}mna`0o=ZJ8QTw^2Co#za&gJnQ7BT z%fL|;8-`($P@tbG6F;c|8B!$3h;1x#a%e=Q7eWLy{`tLBaxT6Ou@ zo;^hYSqHivklj_b|4fCm4=SJ~6X({JH)z`m%WEsRox=!<-M)3qOIzZ|l`UV!m-Oe- z&7C`M1YL%r7?NYIJ_KzuR2ZOszz*DNQZb{BcDSskm@K2IZ`W z&J5#{CQrErED#!J3X>2n$Obd48)zWh21IihQ?yY>ABD8hGothi%B7;9l+sC=0!0cn zR%B61B$Mo!SpgbXp1V7Ofb&0ijo$8G?qpK+;bqgW!EB?KXV0*bIw_mdF7Y@e;M%5 zW)6+UP6k(bkX~;l+fWvJDvUH>fd?j-9TAg6m?PUL;>4hfW?Pa(6ssiBqu7{ivY@hB z^{Pi7|JTcPC$h#I#3I}<-d5pWnZsnpOfnW@R5poSSqpHr+BS(?ID=KzZxL#Wi9^1e zHmhon#@1+)!6McsC21YYrUwj|8lwE{#rn=V)$Qf3ciUY?-$PRfw5>TGeB7*u6tCPT;eeRP^pSY-A(l)Xjga_##) zYqr}uw|HzWAkZ#p@Cy4dZW6Q;L;cWJd+p&XYf0|wx7WV=fDf)X;WuXuatMciRvdiW z|7csB=JAVcZr3V9FUawcLN9-E`FU;c$V&s?+mcY$g-+4USqFrY8O}35Tx_RVkedz3 z24S6B$*yRm6WZ&7G#ZGcfdC&QfUL>~t}66PcwFONx`fBBJR##1&NB?z(6yfkIv@|! zt6mBS<~Hy3(1$-XnBDG%x{7HkAVuO0By4vZ+0cf56RQ!>SoOsE^-N2CBj4rpmp;xN z4mfG*jlyC!Ko$+lh(@Hz=!`N7)5)lUO+mz1-Uz!3((x%2L`V~z=o1hUzyKj+!{m|` zytJIqkd-;)*OGCJ7b5B#(ZE(5B?+EMA`2T1$|1i}C>VeN(UYJ2q58r%O7T6d|8wCw zU;6UrIhd`oNEaJLD?{fi`}Hh~v~s1e0B91*@vmD9X<5@^Il0E2PCJjwlFu?YyC$f| zceWsj5c?oSpiK}FbG()vHK?@MfsNfpk}KksY>48Ng` zFzLyj4!8h5&6bTqWm1RibPG>FIZ%SeEk>inTM?BKK$JKkn&>N;9=Sx#EwXZpwQDFB zsknt?UJ-IkJf%gwGsga5&Q}7Y$W4q>9*jN^fkJWVbnZx=(lzj*0TB;y9^f?%SkO#j z%H}p_*G+CJkqRK`;|D_m(V)t42EbZ+ z>Q`efq@zSk0=C+Bjhi;hvP}eoG{CyqztqunMhhx7K`Sx>oBwrR(vse}x*An0Bba!Etj|`BtTsRu64t7i z=a|9zgiX_$PI61cvz1OOS#?3f)VTw$ z27QO8%2!);nh$(rFM}Cfj|_6H2gaC#3)|wr9`1Q%gu8_TGd2$mEk$8mO<9WU+s?Kt z%EPVg6?0q9-Zu2Jqy23>OMALi)D&BD;^-Z7rFV{<^gM6-yztC7hM9hzXob4;pVCw~ zqwaG604bepZ6Fk>UYDL)?do33I(W~^7q7Q`>uf}vzV@0>_AE50i?`h}I&W*T$0v7^ zrZT6i3^cX@TjPt$_~h5_o3{V__rT`8mny$GcEdar}J} zX?ed%j!nM5+`ETHyv)UWZY|ZE@Ra+me$N?hy+6O?qVlk&m?|Q_hi88PQV80UExB&B$(Cz-#T@YD%rCyu>-)WiZ}v{ z28UA&+nW%y!@UTEhmF$_Vyi&+Ij;3+xi?~;oOI zI%~mFF{}fMoZ_Rhh?qQ)06#B#q$H~zN9u-E|JgTxbFywC8l=HFQVR(HSPci*fDOQv za)77)qcEsCg)vLG)sqZ=g1rK?!h&%?2?Rq=k|yKIxDv_0Q~42)BfjXXq3SX` zQ@Qwyx}|bC>jORMgE8s@!M;O{^b$F38ZVmixm7Txcl!rWJ3uOtm^x~5QI6&@k4?@!gKSh)`&T8nIX?Wm7Ys(f@+r_8L{WpZTR0s= zq_z8+1RbOg{Jh)2^HH@+Jt3;Ip!d{d)JXE=2ydvl8 z!(9}|Hhe>9Y{oxaLjEETl2fm1)C11YMom-1Zgd_We8kT~N%m`r5WxLi#JTpX4fzVA|R8ETgMds|#WK2HdWV8!K&M+0e?s(9=E4foTkL^<<4eGuK zz)eqtqb7j31mwo;v`KK$%xKxk_M3npJ)cbc(MHxms;<E1H5@fE+ydRIm&>?iwyPLyj6i&OU)|%y}YsJ<JIo7Iatxu>I>~FPqzsVP zd&SqIMOp}8S_^1er-j<5joPMVT6mS(q}7vn+|g4-icAgIP5q;=4OI{l+p-+4?IDb%2Lmljk9D7YT$*1U*0+?<39UX?NxUOe z*_N}*V=Y;X|NIRXq+D_PP-?9~oGr%UTopyYw4P0Yd}~9S^*q&5$3uYE1gLbp1*lpVc&bpWMYJVbPBq)%72YK1M6nf8ueC%>;Zau5kos$k zw{_TSpi&JT%9K1*`5{MpT$Xf#oZ5**VT>P*qz;_wcQ3z02gT2-0dR)pkNAy z;0Nx+cap-oeOq?GRNw_+w6#Rx{aWPxF5C+?` zEnyNCRS+^*KvLMWQ&rkh35hL-SDlADSe6;?zgEBtD;<#=mKzaxfE;cI+VtBpgvc)q z;v?48H#6c=7T;dwCCX*uU(�>11Ag5a*8S9+}|w`ryBhU-(B1M_~3vYVLsMoMdjunJz@KB;jSB+S1dD! z|42P}h~a=*sXbB8cbQ=u?pI6JkRGN$RAyz4DNcacQiWp3jo9bkL&kB7oK=qEj_X+C zN+^JK5%Y@W1Wjd;lU1F2kQO9APU)(&w3 zBu0ymNI8IzW=fHU5|2(tA62c>0cnviO{#gdJhlfiXxj&#^?K~&DM5pBhAyCb7Hcl z5rWogC?4g^!DqIjO$0*dxYKRKIy@<=A0K(BZ<3(5jyP}S;=qi6M_p-WIcdD^X43^} z>%Qyh*6y6PX{9Z8j^_!SQK~d=w$^DH6-|#RVUY0MJCPhhH0NgNdT2 zP^5ODJ=-vCf#KV!UI(bV?X3po$|~>#M@tq{-0NE9dsgk)POJb`@E!Id+9rDx#Q?XqC) zv*u)jrn?eLVh??9(nR72SM6%!a=3Ht-|puOoW>$hNh`+eJYWMjDBmy!@k0sYP7?w@ z5^R$AYad@q7Z2n>R%YvV@kHNlrt~0DrDH#1RV(i?r8I-39O?C5Ew4?gbrsm+q+_Il z&om!9Oh3;zAZJ`10Q$Zj`<_53@8!E)o;@Iux4ZIr$Z~U-;wdKB;5IkR)H&=!;vKed zGoQhmYuqrG@Mpg>-4yc$cTPJESMvbSPk95Fnxj9jNih!CWIA-?{~cu9&1N?i_3EDB zZr1dr1nj63t^f*Cx5H-JF5uUo zW+!h6D*omv0nttOzKLCL8&u{NXrJHsThMy$XW#qhE#>$-v+W5F9o%N~vX=ALZux&+ z%SABJ&!c2*$5WA@!SeHV7yGQep5S%-bQpJZNY_+xDu4-IW_FMCllI3tmUk_ghx^X> zd|&Aq9c+~jbrAk^^UhSttCNBhD+SndR$mPK;DaKN0FhY-SziofT!cxDczQD*(5Q?u z#Q1UOb#fr^Tx}JNk4T(1wl23tzE^hPc5wD}_QP8F#eef@|KD=4up{66SzITHZ668$ z${?+bI2GJ4P`6AO-RTxz)N}u!qgU5PFL$MX_w80`r0O&O^Xc!M^s2AXOx^d}&w4?Q zX{tBPc1oK|A08-FB~wsam5g&yT^MSKFeKN^OYam zF!y0#KTs|8^1^q%;tow*gz&xxe=c?T_NU_JuokQs?uvXmBfxn+j}^WHgIwbVAkbVc znqpDmKq`|=$0P!cIt_yW06-v6tK5M18!!OGf~%P@#%9+pcGe6>%g^w7J^cQT!C%8e z@)rmwI7o3pScs^&he-D4I-wu!L|E~<-u~`Zv89dl0pg@86>>aCTPn|?}`zofp zxQ|^wBvPDAa1~)z$0@s>>?+9TK!phN!ZaBPQWq0fF`d|i8O3B2No=|p^l~eWm^o-N zS#oofC>8;r}#qUO^&PegLZrshPT>FhWxNqUuEN;}9bAkeqBVTgV zB`f8VmVm5)4t8H;A(3-+yHZfIy$>sW(Za*Suw-1af_9N@0jn zLqRedSSU(e8>WE45X5*{9btnIL83{WHbUhx#ziD!YBc@|EI63u=%banE-8>`xQv4t ze$0xK+dV60S?p%V<_cxDUMO_!mEQ9BB5}YCH=LG?k(twnU^+Lah!6DfQJOBrdggW2 zX;%SvZ#D;>CIV;+1{``KsD&1Q{@Gp{f!-ls841NUFnL*wm;spNutC8%edt z2ElKXp@~#i^~b3;4}!V^ezy`*|BwL{PB5xn8)}efoFS_FT*MTV2u^5Nno}-rn#D-8 zZ^16j^f2N;rrK&kcX{NIDUBn-16lvEwFFsz9SGJ`|ESrIRB{Gl+Eac8ux#9}-7Rg% z&OLTS=5C1&-Dl4EM3`UxJti#MRygsz=z5odVCcemIO1QT#064s+}Y9~4wII2cP{lDHhx!b;D9X8OqIIgst8|N`w7u@rbobteJ!~~>C;2VebY~TU2cGqVD2_E66K{ z11b(^?B*D-SQE*$E71cJSS?R)h*DDR)%67CC z^`pNeya{q9fU-y-tV>`z!U_1 ztUUI4%;{F}JPEZhF`Qh97JH$DxNL}urPScQmUztb<%W{c+TKOX2+v=IPke(k*Zt}V zM)KJM4C%Y0KkW$6un}T?;RuHaZgitaDik9NY-qR)xlBGtLy?JGh&ufzEjBK+fB4+d z{w!(3Ef!A!6a=RRBVaG=WU!on;~*qFNVx!tVWzdT1L#a>2GOLjgM7k@F*39~>P>6_ z3K+t=*wjUkCbJNZ(AK5g5X~RD!DLW+8KYK}O?3Xjcusu6NY{1&R6Gw?6)kHbfridU zs#PBgObw3m|5J|P#B;7Pnr96)iq}Xs22yq0N*o9A*HLtBuk#1gHL5e*VZ zDGFJ*UDQZ{R1qK7h}b?dq(@R?>X~FAN*O*OadHY}cf>Tx9X6AdF$I`^gdz>t!cvFq zp{WiJqD%gGtUg4Af~bPo3Y4bHJ4rPF;)F2F#&)ckn~18&bV*f+`Oaq>!sg9XvCXe6 zM*=g8z#w83)~v>`oaB886tr5S0^+YR?qm^N-)qMGBuTG=q!zUF=97NfQH}v6Y=8gh z&u1Z&p$HroaSn{&;*ez?t$i4`CcDv1A<&_UrSL-iS6OpegQWQ??@UOm(ouT&l&XEh z?LMfj{{@9^0PtZkP8n9G@yuZ!mlP@&E=kNO8pei>OM)xU;!LHG%s|wX4O~irRpMzw zM58=rA}KJ=L`8PW86s|4%@<*3#AuAYgs+#sVb?CIk-!x;$3a8D&Y|&@RD=~QU{e*? zzRHn)XHKYM%M1+^7qV)A++T(>Ti-ti`p)@eq@mU7jznBgdZ6s^zC;{az^S$kdP13= z8C*2l-uA^g#bquhcMDF;!??a(6a_0d+~??67>(l_O9-il+Z|=M&~!>t#6ZMJ6nPcb zM6z{R_N8*mmtjR(w# z&1~V`>m#IO>71rg2_WM;TPK7#rz^*XF|23Q>AFC?e9~%J?1f#T9aE8EbC!}xHRDgB zkLgr~*9bVy+h zy?&jeAmV%(IiEYg&~m!KPorGQVN|m~=V9)&|Axuovyxt|DBG$>=H<$eB}1Ryv{UOaaB3|2daA z)g>V8m7;41I?ow-Pp=d7j=i&=Cq#xvbM)8?LfhN+a~r*FS#f~->C0$3n%SaXtJ9I) zTF?4_14?{!tGl34Pq6bJO0J2wwa+LS%q*FZ<9<%(HKbZsFD3M2X{rn6&_;K zp7fOSnG~(FP=EsX{n9UXF{g7%o{ayAJm1HLnRrZoz|d;VF@GqRa%q8U4TfnjhgHco z88qi6(nBP*&~rV9R6+AmQ($bBWx6MC`Ev{6u!6qFf@{`# zG=g>i6hF)tZ;16pMKf`$$0S%{B%6gyT$OfqlRa&hf#dg9TLpK_#$j~#{}Q5CT5ys| zn;?IBMo2j2V)j>mgm)B1ApnO*3C2(`{s$$-g&oiXc@GC`S4cuCCkdE$a!=AJEYmVd z#%rM$f$Sm^%C>ADvNnu%UKsdbY!i0*<{S6WD_Y_hA$WAOcOpwCexE^SDJTstXo(5M zPvzET{&aP37EsIrZ+}2PU8hF_Hei?~Vf1Ei(I*R|NM>e-El2oK z$54Jw=x{pmXucSQQqV7MARy(TfD{)fYq*BRsDJelYyPo0l9DYq2Uf*HOk$z{1=uz% zc6k6#K(N1>$Bo;fGAx4%Gp8sHFf&S3N~ZToRAvllH6e~PMpoE?Tec;O$p47gW+R#L zb?D%G2UTa|1`U_!f_^{&`4xizHi=EAN77a-kO+}rC{wR!QF0@X24qRR@GabTHjm{a z(eaVIh>dv|Y38?Z-6eN6fiF)$jjAL$AhcqI$XJ~wlb8l229kiXvj<`*5iwOA0SE|b z*iw?$j72$nIUcI&lA zwWyA3rBB!AmlSzV0p}WouxNewXrhF0zo-*faWW;NT&*a5PQ`Mql>bAt1t?^oEhhmY z*f@YAfq6NEZ^?LRFo#WbXiVR5ULFyZH=`KE#zaz5Nfo7i7w9O-AOc_+mQ?ea^(aOa zp)_b|bT;CMhyWZaIGrY_hyfsw;O7j#CkVI{2<^ix%=a|+^?Gb+o)9UR&{lT#q>BaT z8Z)Gxel&1KSPo9&o5=8XCYf-;h?^vNIC)1WYO)D!Aeyk5Ek-GJg|r59i3M+HGUO6E z|Cf!a8AC0V9L=T&NU1PdI4LR?C{b}!aF}^u&?!b`m3HVmPYDv$l>$>qQOT1kgJ@D( zIX{O)YF>3AU@;5D36>-PH)M&3TXq}HDSIP$SM(KOl&FhFy8lB2>7;0Op4Iu53dEOw zIbqT_K3LY5Qi^4fn3zN8k+v8vJxHd!_jDq;NtZcd{0RkWT1s<51(bIQ2Kt&3WraL6 z7G?sClyG20c6ab8auvFoAR?lpNit>-M0+O{_h$#L>8M6jq92ivG2}usvnneP5^Tqt z(g|9cIwd?RmZ&3~@mM48Mu>N+oGnrjL|UZL2^bOiMe@db#=wHE=U7k|Z6Yv+!{ikvd4Xeb3@fY_!RP+BOtng9v~E;%5ybA_FW zpfzcy9Wy%wswNHkhBIXhJ1H7C_LHmwdOg-^i>jfwWdDC!xE5F_qT`W-q+)6)>aWQI zl`Lv=U+PVdXA@2IQ?vL}#u=8P%9R%*3ZN>H08^HeY76?nszr(ykcfl+n4KBebha8L z>2Qf1C8f-1ogWLUQ%W44X_&JbSE^`XUaFY2cr5q1SxKV|11MR=+I<|gVYq3Hp*5{_ z_owm-jIkA`pVoeK`lzj$p)TiXE>>wCgSDkYQ%^EeK~Wa%Iucq5RK_-_-*Kn>SEw-C zuhgItYfE+Bg_~H#u*`s?WF>?kr9N-vun#M!n@Slx3Ido&kGMw#Gr~O^+la1e3-qw1 z;=-~6`J|SJkBe(2KBB8Z%Wu1yfr#lX4k@hZ+5fr5YG}v`SzkIfJSTKEs*RcYn;@xF zJ?L5JS6&x{T9>)H+WM0~LMfHfjICrA-sobr_FP#@J1$VX$J-MJgAnbpIb90|NlT$Y znXNC8ue!FV*SoJLlv8lnsG$RtzVnWSTboBjGXtxeC~~k&lr{@10wln! zlXtSP$%}lVFtfhPkpO#J#_?OJ5leN>{Cmq3Q}{cnSz5SjY;@5%mv3yrbQ#AC8TTEL8}Z!8JN_G#6iqx2ig%eXUQy#Ns#6MSUb_EY}kc;*r#04q}&??Dve{PISB!$ zs2Qj(bbcz0*|)TTi8fN3SN|$Wa1h=Re; z()^ub>4)uW)VmD=dR-aC*-r*l&3cB-aOGZ9{bgM3B9$9q>sg6geLbUQ5iF=rSslRD z*T>N!rY`%L8wk}zIB)j0Z_dpRh{VXyCN$PL8i`CMZH>7sy|pqNEEk$-OPMuSf}w6dtt+AtWP}CZN|_ zi&p@pP1+!S+M^BIs?FLTf!a4+;soHuBvlLQgB?Z9+hAPVW9$vU9X>Az!Gk2#>44nV zRzC%W2qimaylS5pvH#V|JqTWHaQ1VUHY>C2s^lA5KuC{mJCdDhKiS}?l`bV@ zy_p^16kezq{9R3i;Yz@uA&%OSF6ppc;;Y@-fl>xUJ=?X709c3HvpD0sUE@!10l=xF z?)z6xZIkyKq-iV-KMv%%I-UhtPgjx=N$$6GiLjWf+V{wKx{qc>g`o-mb#|?g9f&$_%go=8o>^e(pBS znWa+{Wq5#k4*wK6X2d{wevYsd=V2aPC$Kj82P1!wBy)P%!)5le?V$J6DPV>!L{O3ZSb+m{0m3xls_1 zPRa_mN)gb~>*I}tX8pLeUNvNx)?HfUL8ngo+~r;#(5YKCg-qskV|Iujt!#<}6?>!o zX}bfMLHJGQrp(&u4gp>7^5`83Bo5ovJK>u>Hh|D2U<)Y$(g3C1@JWmSI<4Y*FMk98@nDH{KP}rA55~TXOI4)} z>T~7QLjNP?Q1X6@uv}!>zY+|q;Cm=6xtaS|*5{-!Z;3IV>&$&myuRJt_L$gRrP{3` z!_nQfXnhJM-aN}2;XDwC)a)+%>;Mgf;kN9?(2LgIJlKx3_x@8%;g+->4*%r8MwpRp_gG7?%y(tPE_{q>5 zFi=oL%4+Ix3QuwIAui<5zz`8~X&~})fKhW(6El+&6x1QqAwqOwBJ?3*RCTr0HKM|l zGylZ)_5gL_mKW1E$uhSz*w=D^7x|VoLJwKQrTNxwdQlRoQYrXh@2M~QcPSiOoOnFk zxELILUA&z<(t1)}%H7?&k3BzK-~PV7JC_214g~t^7qFl^edU&wa%d3XyMz7~$wD&_ zW5#&{8ZJwd#$iX1x@PI}NV3!c0xJO+Ab`LHOcyK(0H}G>W=ownUlwrTlVv0$8>hT9 z>SD-%nFQc@Dzwn46)Q;+W)ygb;J^e9NRS9A@+lTA4ak-)YXa>Wv}4Vl<>DnQ5lZbS zJfI2>QUvRH7C`vV5uq@3Yh|3$sU0-py0`}cELmx=dBEto@lnVE(aB9 zu0C9!p;k`^%y;&MQHpV&O8p!5b@4hQGWc2HclGP>wnxY9AhFVY`c(ZVE*He@^S@@j zk1xD$=A#RbT+-mfs)RPOiX>EY>kmSc9QYwFR$7XHC`JSZ zCrQ+M%8>PlsSn0!w1TNC5+EQ;i?xtCM69)ZTx$#<#R#$tF}^6zohE?Gro#%X_+l-; z4x6$J1PFVJ$T7V5f{qCkVC@bVA)73a5F)EA#3NjhX&cQ!P?8a5mqFS2jgMfKS;)-gk_liLM{Rk=IZmwE$Co|O!}QQ|K$GY{@h}y& z(@?uhv%^p!I@LW+K`XAt;v!o0GX%4;Zz5Y~wGPpd5}H*ZQ}0nkw*wROZ=;Ti{U|~T z8!X9L;YJi7ikM`=5LNQDoo7T6k1(o4mu@nq#Y;{6uBN6EwDnE~w5X8;4BF~T2r!70 zvdDXl+;U$mEW3~ZpqN64$-J)cLdYo}+C|JS&#Dwnd6{Ej66^6zw)adAXUlL;rvMO*nImI}SN! zczY%RUY?@_s_7&m*SbSpoQ4YkdL{Rev)Cqr3~3vSLJIMr3?j z19IYS#B>!Evk$BgU_%78LFkH&AK~&a3#5kJmv-90Kx<2)#9DX^rsi=Hc8K-Om-v0J zlo)v|L_lTb;_TCS4p|%t{NtH5F|*AnP=QlGOkNHJib%ePITc18Jf-G}Vn&~~19)D4 zlRZ1#Z9V_fAmy_v&Q^XU{+g#RU=ZE+}9xFS^nF_pTs zk>Fx@Lm+}|^)^J^Eh4u%&!>b&sIQQYfO*3Y3Kc;CX8nyI$qLD`K;o;06b^?c!J!Mp zg&~{BWF{P{$%T$!6Ctofa+a}H<{Ia_9;PTBEAbo?l@hvvxy^%EK}+i9K!OAS26xml zkpLRtE+a(@jkBv=0W4sFExc|XWr)&YsP~i%D8OFBOXD7mhlaO6sdhGD-pW!qnTurx z6>3939XS-TryW8!NczW@R#r(@U}t=A zWGur4NJGHU9!ZY?Eg=Nk*qhlX$dBuEuM*yIiUaB9wf{JcCu{-C6_Z@l%=Ms6Z8ch( z21OM@F~v=8;me0Gdv!u!(dug`<3kJmwys3YgF!evoZ=*uI0RsUB~1hho!e%`H+L4i|I%XsW znMq9;Ba=Q=l2QxA)OL}g6to#-p5PZhUU0-g^cxUpinb0{UX_3SYlh1 zaGmYUp=4#ZlrJnXh;TCKQ_{3ge3A&X_&icPKxDR^tYrZ)^&ll5aJq*=lmuba*B2OY z(KN0Rw_$)l8(W%BxeP$If63P&4t7$Mru3yB0su&Q`%>hU&t^b`iE;*^0L3(n0U&^Y z1M2udDHJsnm#RQeQ{@iu8dVXX86RxS=hUUjv~TBa?^Ig@Url_{s<=QUGPW9y2SuQj zUrm~%lA($^hDw*|Ddt_XrjBq5Q-NJ;PXfUv*K4c_Z*eUfgxm9n)qPlPj^XQou*lZH zhO<>KQr0Tzy31Sg4G-n~8!tykIKmjthX09^>`C5fxz9E zE}n4-u*ngOAhmaH#vouK2P-WA1Ti5mQJBZcO&s9PiN-SnHnb5=$rrs%K*#uG5u2vc z_q{voBvdg?tg2oQtXW0IFICJ6QVDp#`Xj3)A7Iu=qBVts{hEo>O6sdMFs^d7CI@-F z*1N7F4tiZ(23ZUsh9D%H)I>HoCkPP3KHJ!;8Ww9H3+;he#8)@IV~!Cw*=OmZCe}(y zw`5}By(BrHR@Bf^*sX3SHaS(Bo&PXSTu>Gim?b7xvBT-Cd}W9|43N1+uA4KeTo-h? zM8;KaNIk=5;q5C+3X3>lFgjA>l5)`{z_XbG@eGYCvH}p8CcM3SgqWT0(2Blu!xUX2 zblL;Dk*0?CzLy5ZFv7jBu}a$f(L1vqgz&qQ)<` zS>efbv*;_@Wy3E#C;23O!2j_jaWeGf#(XUJmMV#WjO&@>a|nS8O2C2;qe2eh0!GP8 zW(t%yM&*lMfTCk5)b0&HH!`nAQ@$sQQG$#hoH#yJcOIsws>0CV|1=|BDe7KSZBboY zI@3!*-~gP))&I!l0HgiZ1_ul6SF=Cd$u(rU8g_yrEC3@R*g$M#;Le)-!^8$cu%0jO z-mb+A2*aq&SYFJ(uq{INZUg#m$2t()){WdSOYuN(@kr1FL#tdYLtCzZikQL$T_Q&k z2Bcg@0WvQT-~|MDp(dzAUn0&2=L}+2PXJs1b3)KsQVR(aU;&^b&CW>TByNr~3s=Ljl{)7mA}5O%Vd4 z4Or+do8-egSb^^_rX+@u1O3h<9N+|(u>_lO1P5lbh)>s&0&K2ec#1>-a-k+X3=m?g zcaY)Xa-{OcaU5eXVOCFj7$6C;OC8%W2?OWzoKP@=>j%dm_O?tGU}Cnw;OXuF4%Jc7 za*s`{<@W#$1pmx%ZioV8bnejFkolA_4$Uxo)(ZM6z|y2oJl4y;{P57$@Tq<-XF9D9 z@ymSvD;c#)?64{ViYXG??@{I?{zym(=Fb+hLxpIKf(B3m(+YMXMjGkReu7f&76?;R zF>MZzgVt_vbg>&m=oCq%0Qc&fSZ0~VDaJ~oT-YNCFmNRHj_;1q14{xXj8M9m@flN( za(?n-IP39JFe~xJbqYWwCPolqD+>q!93zGt=S3X%QXG3r;hw@B2Xh_Wv5t-lFOcfeAkac?b<8jn+j6edaOV&$`4#{bB68*C zsvk1oBL8KM?JDzqXyy++DGfMHszk&3^zhq=&gfj_5iOx5Sz%E05KJu5RcwJex+W!E zWdM(kRK#xhxGw)9VM@}ZDs2%If5C!qFb79SZ_@6>7El(&VGlCV7`PEBdohLV&3GO>-#zJNAu56=RUJ#Rq-MgXU3 z6d}c<$Y!j3YF}ezo5D)FrOsyk-^8mE! z(f@|QDT>ok-VY`Nh#wNH6Mt!gZt^|^&;hMRSOO+6s3k;pk=ZzrI9BI^rU@rIs1~7( z7X5*XVkJ`sVJa(eJ1KB`>eCRE00a9&+xjysJpzvq)Ij}@$%v2mdZYn73P*AW83e&! zoZ-vn5eReZ97(T3*%4LO(E&JsRa-T9*60aUYRd=%3yQGAiZBb46?sud4na!l2;{>(H_BT!%WPhdT#3Vq^Z ztgYA7Em6_aaJ)Iv>p|cO3<;O2onO3kX$vu1Aw+w9iYt+ z^BTiUxl$CKW`R~Ybd!+KSLKjU6vIq-4}+4UHQmzDSmD1oC?=Y9HS2ar*$YXXAPJIc z{Mt5KLt>^fqFfW;edCZF>sD#$Hf-DRfSCn{oNhe7;eNv5Z;dm51b61-HEMbzYw9&8 z9Z)(`j8mR7|LO&hc2o-VbZ+Mng=G+O57!DJim zO~Y_?5|}U+CyA0y8#f!0>kScKUa z3-kvNcp(}j6yjb_j~>@}2TqK~xB^-=diR-CHvsjl5X^M8jQ>&Ljk!0Dv2c>_jQz?L z7jbmuP@#DNC!A9SAj`FoZ||a2jwEH12`E@~V}k-PW^Ru#(HMY`S(5=G8I$u&k|#ir zn-?JGw^2(%T@kmizya09w-->?f{U`%c1eR3HeWq4A*%y~fyI|u!UydoE!e`TbJ>>r z0(-?wFmNP{zF~B`ahNA)hULK78uo^NC95IIPW6hIyJ1?T6UJU)7Ff4Mj2OpEQi)fL ziG>4%sK;NTQJk|yip#m27a%PURGp86okyloCZ#xJS)-13kFGbMyWpOCHUlxmIfMOn- z+Cm?O4fwLVHI$ZHbg;$1WqZdDUib#fI&<{^G{}V5Ja=*va7m)_U{~>iq;gQHQWsTs zyt4AG6*aDP=qu?OnVHR?B$Rb0xNsAG$$! z>L;7`p@E#WgIvj%Jjfv+H(xsxfHX*x5R;L-r2k{Ow^3T9hxC$L`lTVdS;tm^X?h#B zArU*+noTY4mU}m%fi`QYW0wpVCK6q}YQ)Et{yg&q5`$tX%RTb&Ew zEQWBbB_o>D#!O1rram{BgEAo$qtUw&#L`CU(psE4dWgNGh@o$ZglQ17^bO;%Eq_=+%L~=TWwbcFO`74OA`Ag=PT@igV1Z~QXUl7 z7i(*)5ozma=qPcia4&K)Z~rjqF+l;eNrSXWb#(!NK=w9w^|Xohc6fJqx%qi`!SuIy zJGwi(HuSfP3kb=5K0XMDh<`u7dnEP+1o27%!i39MC0u3X!UYWwA&S6faUe#E0yB~j zvCtQ;JDWm$ENRP#11YLZ9cvj16|yW9Rw`pCOV%odrwB0EV4;JB4G0Jj)fMJwmuy@H z09fi2DO9LTn;JE$G-{Z5^|mHJU}k{}e_}m}D9GR}S_bvxvBehc?b~Z|XBgwTRw1g71Zen`(Z`TkQ;U4X~_#8}(M{+^{YFIKxG;NlI z6$@1-hqRT5+STlPv5~-dxh!TL{CAe=w~SX`%)m3b-xL$BFK+I;{PFfjyEb_LfA$ro zP6N|~MNmN=vGPhl0~N$iEH{}{9c5IM#m_|=X~IE8)J^J`XRbcGcqkS%XbtIA^c;X392v~$5d7nN1k zC69oFeZdzbPkz#gtD;!LS8A#hi<@C!rN|?+*U|#$uTxTC!Bop+<7^v2qSsckpyR8 zTy=%Oc0tYv9jw0`V-kwW*m`R(@p|V4s~7 zgkV6~N$8<37aXCj1QqGSA-EiR)XSktkOZrTNF=vRsO@Rd%PcD4N*%VL9FPrH_YV0Z zzy0zXB#-rxHn3j>6MV2=$O^kKvfFG~u$N$^DrK*S!Sxmb88iXx5K>%mr({LgX$58# zKrvZoLqKA1hE1!Z| z6HLgklK(1TTRmp!A+&Yp1#~EmG7=ZGdOcpd?vdTgdoJ|HYsL1>b(g|%qqDYKa<@&` zt_=|2?6?d*s8K=IHdJpYEI)8zgW-M)u16y#vB`Gg$w;meE{<&;LaD^$;=K00Iw6n% z`#U=6r3)!A!l@g)`dJj;{koYzstOkEt4cgE>{$-G&0|hr0|1;($k}q`Qj{1O@hQ); z^DV@GY@5u{9}-C&HqVx5ZnRMV{m+UFJSonVQZEV7z%AX0(@&SJ03!szM|C~k&k}wg z&Hs{wtLmX!I47Q$RCebAXo~p!DtCpk20IuwtOdej9&B^Ru^3jLc`OS7-s;eYr1dw9 zJpV{RI-wT`Ax9aph@~UoYS2bFvJmU_uXE~}gDTp^qNs^VUh}#Z2&X0(uJO=xKI|b7 zMHjjraRGIzJ0k0jWWXh!Np@a&V(p?xHPdI`-cuSTn&1yN7~_5L69oMpg@``B zfq6<`p8|@owJvlGJzwKQ2qss^%b|xE@A+IQboIb0t_@gr6IMKm6*p2+!$5oMM<_J- zw|qS4Tn%wyxi09S2hs9brr-oHnIl3G8bL(uDM|74#K~eT?`$pdjQ-}8s`#y8U;jV6 zCN=|>6vB83iGBH!m#QdEZa8r$=A7LV>k$|#juQaSyw?Cw0!9$gDVY&Cqb1;E#y5g2 zk#_v2BjKnANkkGv+xsU%f2OnXfv;#w6WSV}Lx4UU384&t86oU&fHAnh2>Cl3=87q` zXlfE((Nu=oT4~C0Dod4fW1s@l(0@5i+W_mD~g|n?DpHR=|oBfqliB5j*Pw+SyL6+ODlo zOy@e;S;bLs>v;UU!^q^BnJ%V8CbO^}o)X$dK-{%la#TbDyf{2H4z{t1JpYTLFgiWu zU=gBZSjI#lg-DB*PdEjY0Q=V9fEaqGJmVQjCX>O*N@@~-o-3tX+a`~>tn`$&eOpc6 zv7h@CsJGYP1y6g*L23n6fFgkfFpJ5Nqy}!O$$c&!jCm`2E;B`%gI5-YLN%hCvwH*> zVpqK?ULSIUyAV4sIMJ$BaW=`V?sY5G)?1%Z@O7TrbJ6kcnTIk4)Sx{RY(i@$P(1i^ zQH9k5V*#vS!FJXUl*LRRB}!4rO7EgZBLwZjq{t2qIdv&;f5XE_1k{qlHy_!O}UDFE$FBAU@!8u>D@9%@dn?Qq9trQkD8 z<+2kKu=uehY|*d!1uPlq=U9bOVZe4sFdX{J&%h3P!Hq@(WCiu$DW`!vnFR!9I2!>* zO7FWI9_@_YgUQx5BW%_6-&Cj8QWM8kShTHeZcm9@hPkDb2yCT|ZT#v2=y=Dwtffr? z7@}Yj+XzTyEW*T56Q;D4v(4o#Z=2gLAaj_*O}YV`8NF$S zb6R>8<#lUSon^{Wd*2$x_qtm!bJhzk;`L{GqCyW@C^Uf~v;R*x9@No_GV~n~?FLo> zmU{+&IK(F%X*Nh$vx3mH!o3?hPmV{VA(rYVq3z+7is`Ob9FG>q_&zDOb>4C< z^O?&WShcPO0fc*OUK-$*KQ_tZf_^f8>^!NIL$b4vdGrTB#u(exDyFg>JP2^R+f~21 z)w%3tadSPJ12{L>S$8Hkqr2?qg}2$$emi^5Ii7X4FF*0DSCipF;D0rA!3SmG`Jn<5 zibOb{v$kkC6&T`=es~2Q%+QF2^4gr98P*)@X)kO-hRrA@si~dnN|wUbt&X)_UTs^H zhkhL@H9gH!ulih@@*LGD3$=V2dsq?s?16n`V*e7=*{Pjwo)Nj#HKwKz+U`5m zwXOUsGvCVK?mEA?>(z44`_1M?l9{+`we54i``XV@k>X8ndBYVj_jY@}`+aBiGvVKL zx4ZppmUqEBFY0DHMg-(d2ajFpU&_ zwa`Ma2*{;cNt8*7(T6xh5CpLtUZk7{wpe1&7;UY?>ZRc7o!$oi4mm(W&dm~$C5QmA z!lC>FvIWGrm_ku88)`9!$K{IlLDl2&ffL}-X-wVnEn)LDpEW_BzYs-QJYSE9U-hY7 znhnf-EPZ9PZ6OOueag4L~NbBl^XkJl&mt&0!q5-+sYg{uN@j$wq}xK;hvA!~G!| z(V+tTNn(wWqbwpr8V6UK-Azaq8blMQZ44-4i?TUTx8Gz;Sr=yZxEr(93e$f zBpWdy)}4kmUgP(1U#d_cQk*0CePh^VWA}CAR(2&Z*U(NX zL1!H0ej=xGW?hh8nRCjHCHjatS;cor=XQ-w7*dUt}@|~9tB`TA(Q^Cv*MT+ZXQ+EpC90`B&Y?LD$Bnw`u!|v^@4J_Y!DsFM$1Zjh*P;7*5q;?!`;vyTha8N@oz(YVpWkC$4 z6+)z>0n28tAAB2Vs1sFo0nOGd7v3z+=IpU1>*@L|>RusOsHf06D>)g!(RwNEt|!2m zY13}&7bz`X>Hi}^QrU7^Ey{eWAtHd*;%V2a>$&=8*V-W;rfqM)D<9Sy^xnq3T1T;& z3-;Eb$0}Vyti??oYzry|!RD>`l4!&JE#Ri_-d0J5YA7~D%S_&);J6Sca-(G0B8DK< zMJys(Q~(DAumBG*0Sj;-74QH%;=!bBKNd%kni<(41$2ro1=}plTBqq6t68b82A9?9 zW+$_TFlm@)?Rsel2Q3PpC$$Eg?wTp@244#=nbZQ_6-+=Al7?Q6s~ujMoFXsS`Y`jR zE1fPRLSZeRj)0$5Z$cF++w$hyx~(u|&aI{1i24WnuJ5M0@55^G+jKFNsP7hA30!cd z0MO$6-v6Slgo?9iA0Lc_j*)=7b_oCrum&Ko9p5n?+wrC>*w#hoIn`SQ2Qt+-#gbO% zS7z`B|7_~^Z0aI%X`H9frf>;oX(UH-mfkL2b?cnqSAM0boe?BI4#)A@X`q#<@&+*~ zpR!~Su@Wze*|zdOF|qF{s@p2b5qQE)L;z`qC@XGp`+{WXUGe(9q%f-{7mKm(#P7rc z;{)K3GaFJ^=;$9LfHcQ3Y~(REXR|gJu*u4g#E`C539=y98wP`=>TWP2pR)(6vjzvP z3Y;)Jqi{UWvpl1492QS~;(>eOUl?KQq0yy(?(ksdFe!`c56fTjrn1*2G-c>1ECULH zCjZAn$MV{;t#3BXsf?CafG7&|R!6U(>Ajp6i?o%5;4zQ%NvAXzo8-leu_z91TDE8o z)+R#$#zZsXSzN+4?=(;6aRxBps%#%Pi?iqga_M>}J125FFSQ~|DI`mBC8H-j$MaKT zzzX}@3vaSL&ZQ>X(|IGCk<1p$(lgM00&vgUP^+(fO z!V-8%D(_^D-k_Gh$vuVF`J%{jVqc%LVHVR9`J%14f>-ImsOoy%d^7Kq; z473ESFqw}zt(UdKx$Bv0c$zCH7z7hTq*O|tZ(jfMi5I&V8++hp_jI>7pJ$2uW^7WS z5A;rBPCvGxUppQ@7^HV+Ix;%92f3xQ4v|l~q>FYEEjhV&FuG$p#Av!ayY`n}!UoLo zW7m7VPx%{GIead-`w?Er*jZH}36QGn1i<#Vih0P?y2AUVViETdNB>W*KfJpZs=WaF zE-wqFl6alxImhdH$A3J?UktN1dzSn;FusDI0|KFAyP+exqBDlKKYDxzbNHPJxjy-&d}4K(!X@(g-N56jn;!VWM}5@adV`*M1?;+-gTbSQ z?=G9K-m2KbBKyX#c*vK1$d5e9+sCsTu9!3=19zFHTRY3o{mTpAu--iDW-#9SJZL91 zIn1(1OSptr%B`nf7S&xnymyZV6&oMpYe@*w2UQYj>V2yVwo-=Q7YP}y?wwqwDZy{#Q*9dwk!|Reajp0qSyT0 z+dT2_eb4ti;3L2CbMUdgdm7ld!54i(>k|<~zP_RP8~l6JSw5TQCDeC5_>Vc&A7)yY zOz3ksC9wiW*R`&lpw|z(oQu8uzdq}W{7Vn+{nP#u9d@(>K}h5(jf}-(Qn^GbPo-xR zI+a!{9Bfv*<#xSaFgLTAFkFZ;TAf(87i_rsEsxjfcYA)W=QH+yzd*o2!NNcY!$U1t|g>}&1lZ0_yvZvSxcaPsL_hh(Xybtd+9B!*x4 z3SRg3`}TbO{dR(YfCB~&m}t-dg~5UZO)O+spkV`r3mp~^Kyl(lj1n_qUN01mR zS|n+)A^-s>Rjy>Y@}$LW@61X#89$;!3USFd5gj^$LA>{+pC(XLh7m5>SxNgyyar9s1lyA{Sjuuw18 zEx%KAO&C!u3NTs@7#uO2&hP?O=E{-tHo5X;%#<^4*38-S=WOVTjxHJx7`>yURj(%M z51>GT1rug`xDcYmlQmy9zuxTo_L(YSnomDId`j4B6CJE3@y_TG|XDzkh5t%6mdgohUt#A?o5LUMHWv)(X{~v zFkqtKfa@(c=`N~KI37KEsW|Cma_+~Hb}N#(g^X)1NG5IKC%fyoYq7gzh#7$t0kah2 zzXa2>iVwHK)EzS3lDXa#1YL!^xSn7ZAP>`b7fK3Z!fi1-Xk&{U{mGzO;T2u zdQ25a<6b=fugXrzT*eJ!=E8O}R~p^TJ4aLjp?4jW38;de z`t*MOQ9WX|gESlKPs0{lcG_p3oz=9}X1(>AP{$|q%6-wYPN?LNDy@?bN8E7c^=|(8 z<&|HrW;Y58esRN&h+KQqdkOb?V=l*BbI;2s-+a(mVcL3)C50Ml-{V&T#(%!%9lDO} zhdAM~$?pIE*Qx4A%*h>b?uQfk0S|jsvH)G~mVxL=PlC&$U=Ai_)3OCA-i*g-5#FpDyaU>9w6MKI#agySkGLw_!WEbT%!|R=qgbFg@DvGAYHl`ANuvkp{6u87>j6fH1OytyD zpt}0u@rb%9BoP1jOCo(FRHWKnz=R091J)9h6JmrU^M(j)nar75?BuYbNzH0{QJe7r zV-=G(x@9sGLW)F+DF3nNfK#eco$JiZIq2xh)bK@6?^I?!?8r7)?UI*!?Bg*77OGD9 z=$E*A-Hq}nF^tg@o{Ln*vG60$Yh@Fgo1`dxB6^=nW>k|BohY1A2F`Hu36?aBn%v^C zm^7wSGVDYpOVj5J#emdGuO#OzM_8qq`i_qG#HZGJ$*?~D6PQ3XBG^XK8@3UkA_S;s z@Yp9y{!EivQf1yYI||W8Se2t4o#^zw=*{Y7ETq5tQ-Vktyp*QXrE67ZOq+V8_iaz1 zQ_7ykc~p#N=&}yb%&eM2>uA*0;axoq>5XFCptk@t#Z{(8b!-BJ4so| zW_DVTmDA{u@&DPeiqo8oOJPgdYFhN!B{Hi;R19UB(}mU$HI}NY5*K*O80FHZ$obU& zgc{hQdhL+=Gb92PyHKb~bE8v@EOWog+2&#sqS4(db*=kcux8f{2_-8-84EeNiViZU zU1w=sN&yZyMKVrdZ+hL^UiFrX1L3_#G3F{;`o31bG)=-?cPLnW2FxRW6(~_#1>gYd zQn1c((s8YXiJGkml35jtw63e!z+ra67d@PYms?#BZ&M9o^{!~6cuuvtD@1OLj9uKT zg^zK-yezyxj6oJ-_+|meOX-VkVGF!m6feKX<*x~gfZNq2m{s;uvU3Xj(*yIi$-~|4 zkZz*bLjQCV4w0ndhAHga3#%;58UAvJs|x0>qFK9IUb9r_8rrf3!pFT}0C_uC;~eMt zy?gdhYWJ*XK>zv0rhs5ADb+m(FBC?wX~2}FEaii;1h@;Pa;4Fxy9V#~m08vv^SUJ>t;I_-QM=)nO*K9e)BSk0?nE`u?sA9agKG3a6M}Q zQLN}YpRZx?geN}m9{cwx3hg_Gw-Vx$e-SS4_+KY`3hUDz33umsbs~rzdZxxE+|O)! zly^Awr%(CZKWcB#iWCy<&|BV}sCO{re5Fe3!qMrbo3?tlM#J^dK!vj45X zoTL79uv@-$Jk00jWunx>I+!2Zj*hrJ$_bN~yDYAUH|kf_ESGFe7hD5!?^oGEG$FvtHLUj8V2w^A@n&`NFV*vxd%WzzZh`B>B4Dj8N4GwDH4l3sm6>HMUUjhj|9cF6UqN)l(#rI zgDKI1+dIB~G{g<@HFS*0m6XXZn8_!)6Xn~q2c)lH1V*1E#?`R2uCo_Q495t=#4j^S zP}>{r3N=s6Mx{hUsGQ1e1WA&7xu~Q{Q>3?uq#2Y%5t`&fTl7DA!~%DGMK372T|7Qw z8w0nz6Sx$aryu?VX5=J9PwqO`YznnnJdnYP;k!9>k&HA+G!K$a^ z3a5O`s+`P_%rL7I!;yqLZ|tTt$e5%E%dm{a{cD8-1Wg_z5JV`A$i&EOA;^4C#0D%v zpR@;{belsOO6o{V-OS3dL(0c0$t&bbLCDO_1Ww}wN3Enx>#9iQOwRw}>_*-kM>w#; z#9R$uG0l$gz4JjK?4-ZFAVSw{s(=*C2&^ZAWDOSpm=Dyt-6YQFT21EErjpab^J`A~ ztW5lr%KMB?DpHL+!OoUcz{!w|?lg}L;ETbOz}XzklWGJa%gLo%P-iR57hwtPTTjF) zO2y<)Xk<(dmByreP7(dk<8;pbEYTB%&XJSO4K+y<9Z?pI(f0JrIw?@L)WO4}3q3r| z)C|uA{ZSvaD++bC_H0oSJyJ`O%=h$A6-`kTy-X;L(k6vb%|z0$vd$Xy0-LNs8?@1Z z8%@mdH&(c`JykQ*4`+&4b-h@Leb$!I*m=EItu#`M{aF7Q%~q0~3P~~4F7VfXb)aw^ z*j`0gb3NFYeL8dv+4OtY5$wWt^)$zf&p*A&di`0U%~)%dR-0`zOKBp2U0E-9RG#$J z2$fk`gIcP6*{GFTsFl=S09J?XTKTY7iwd_boJi7h(WCWRjy+qoOn+u zg_xNTd;6~{|y zBD~#_0+At5!ndkj)2kJo1?5%0rNV{%hsZ73vSZxB4MBFLQ=)}e$z@w=dfmx&+)Sn0 z({iHBtYZB~fI>NVOnao&8PQtzMwD-p5_u z*u__`>`Y5(paQK`3y3wD>s<2H-0~f(;%!8(v=VyUUM8CfpyvREzk26-4|}*&o$r9UEqIgUlL|r-!x%6Wy*2WU=R*s z*&SgaF5<2oU>;V=|3%^=bss`|OHORy2bN;crC}JRVl1}d?x7YVJ=s}UIgVjEo@4(zeqqJp;vD7#uT9-OR$~(ewI3CSt<%;d}jJK{n$M{$x>2<25E^vkl3&`V zjt*&a&T6~EYAM$06VVr{evgKo)8RB*cb4j>M(YjU;h0wIw$ACA#@4w0XSxpSB~Dlg z0cxS{>4WZT!Yo~sZfCNl-f8CKv_5Py3}eK$&$o8#6RzvIE=Pm*>W@BO;jL`UPUb5% zvV1As35ILCHfpqp?2WqL#$N1h7Ub6LRMb9gj1FxjhHSgmR;#{hk`8LU=Igy4>fL_O zl0MH^P6A=J%;;Th({631zUKcegU{qvYS(sb(&kLio^BM5Kai&E^etx>&g|{pY_8s1 zV7+a4QbyS}H(O4gM>uZiUT<#>Vi0BT9KKeKrtau2Z|bCP*tYLBSYOh)Ms5aoZa`RV2RHBrf9}_=@BE%yzQ*qE7H{tE?cCmQ&Bp9!8vu_c z(vmG1!#?nLhHnb*Wj}6l7LRZIo^k!I@kPdK2kvj()^QK#@0q<~1sd>>TAU+DyOieH zWbNSQMs28`SO}l+2#0bQr*ZqfY7M{g@6Phgz18f-t=!Ubc&O?Ghv_Mo@feq1`z>=b zFU||M@iuq!H=iQfu9p7*&+$6%asLMK{z_gf&$nvLdJnm_oh_IFv2 zc4en)>ScL0Uvb*qW43aam$Laz%SERr%@v{c{!tZ=4WzAf-ea;vC3O{|>rme`I{D2?+ z8m@d;R(t>Kc6ZR{edgEwhlTq$Z~jg<{m}P)>rdpoZ~cMSe(r~LM%;6dmwl_Reew_F zBX|DAm;T;AMe09g$TxoYCw}=Cei`!e7#ZgxP<}U8|NRF7L_$a$8jHxIGRa&zo6x5; z>Y5^}R4X>C?QXx`Fjq_-n~C5wdaZ7=%Wtqyr}vad&Be^n&b!Cd(bm$>*xK3L+kVA;;osq#ow2az>FDc_>hA3Dl<)2F z_45Cq_W7Cmvh?aMrA+ag_o!C4gWeD-Ovv!s!iUbrL7dpIB1Lf+C29n>v17)M9z(_{ zNAeLtP_9DZ8>mvy%9k+B^=lcECZkaSPqBPw&|)J<8PRG@?+K zN^MHjY1OMBvx*#3Qs&8>V84nDOLnZ;vS`o#GgYeVOhi4c%EcN{>)g9^@!HMHckbT5 zTLE(v9JnyyktN;MT+Fz!Vw;X5OQwl4AmmjeruKchS##&lo%^{cOE)>Wy7idqUi1Gu zRJm=SSaI>19&h|A`SazImqE`yy?gb|oyXS_z2)uu)A8%i&wu|a-K7K&$u0A}$B=vl z!lz(@3@+$kgAgJpVT2UgMSu&?)#uetE{%0vb{c+oU56`mCtzlWt!Lna!?ow)iZH&& zVvID-I3ZvmYS-V69)8E;kN&+y7J#KxSfi0P9(f~@N~*@>l2AS=Wt8C%;98E@ZCDm$ zB#I~*h&papn~CCaCS#LlrWqBQY^s^$n{9qZWph~i804Km=85N?Xb~B}lyS}p=b(TR zYN(*SA*yJhjINeposbGCR-|EOIjNV`k$I$>fp&W6jGTVzX`^zIdLyNJrpo`PStqd2 z>Z_|(3L84QnKG)V6XM#Jsl4jyYp=jADy$lhqAF{w$L?t7ZI^usN_PXaxMrxgnmKK? zzf!9$w%leL>>9=*tLnJomdjzSMGot&y5IiF?z+3a+wQ#b*84}X;dV)GvyG+0ufMX6 z05GMRO1mw*2zRUS!3#I+?!5V$EAhUyQQT)v(I%`hz43BPoX59n~HD50$Z#l zrnCOrUuC2iY%Rmy((LffGjFW3!XqOY@x(x@EHr-TW?XX5NZ-*h$w@bj^U+W{Ep>cV zoQ(3yET^0`)|TpBJ_uf;} zVRhSNxeVsp`7NuT)?wSMINggk&iLbcN6t5EfD<0K<@?SpdEb*W-MM_5bN;#Lql@&n zv4ZzIBG)N>&H7G>LoU1Hq1Qb7?X^!{`Q^N09<=7Alb*Zq!OKqk@x>!w_35d<4rKGr zM?W}hu@8^?^)hFhGws-Sk9;&lpzgZwSbp#Fp6HbD#GH7{Kij@PH4S+V0B7K=T={9tdEI(FDjj3dRqC7nGj{Gf2PwNsoUY z459rpC&2<*(1aMI2ntu2!WMqd3=4Fi3>$c^5+*Q%INaa~KbHT)9tz5bVCbL+M|i)M z4N-(3B%&)|Ho+Y_(S=ZC;S;CG!5GqThFFA01lhtvD&`7CI$WCk)_N@UZ}$j0exjYDpX zB;stixI*eNkC?oqCJ`wHKKgN7Ym^(tx_C%VmU5G*H03E*xd%o{l5mZDB^#@k$y%n; zfY-XEDtGz9Fo^GyS@azs2g%D~vQn4FG^R3p=E_*whD+0f=IXlE2wEc3me`czHMePu zWVUY$zYL}tftjaaF7uq+Os6;3$DoA!Q1-nQ?5d_ozNgENyqvXq<;0R#O!5D{Ru|2y7j5Plxj_> zdeg$ilRK33YFD+2%f0G#t7=UvTcpEB8P2#eA7Yt#HR=L)RMjwXD4%a+llM*iQD1&O(<_r4`~Z)P*x;1ZMg#8ip!grhs13UBzu z#sw&hXRJ&Qf7r&85pjP<+}rWm4#hz3@s3j)RTZyTa55c%i!aP%3=5gYO^$LNaD3w` z*A|Wm4ziX_%;hZ)S&ZCeub8i#D*tg2%2NM6t#;FlW~Wj4%_DsYMtLk_>hjjFUM8M)Sy5^~9VlX-LeH z`q)^(wWklD0A@Rz*3hoCveVUS#)di7`~h^ec};9&_ctK3j`p+_edvI2yV{SYCy4Eu z3QBvs(m#eb5^()&XlpvxiN+YN=kRVlMs~Op)AqRs?qfo~*3jTKK(ohL;`DZPSNQ964 z=Kphe&J(@s<;dLC_o8^kBkgWFbNuLd?$iP9t>~LKoa9aK`L<0BHI<_p+ge=p&!-M_ zlzZJ&A=kOc#~$`vn|$kNCzjByJk4Pn{p3h5anqqrcDmdB>|9?a4F9dHbrbi@c~3i$ z1Mdfh+r92M7dhSu-}SZo{bxmodr0Fh`E7(+?1z7P;z?e4s?Vz^zRvGJ1kU)vx}e0NVcRr=LUUU4AUmzyA2YA6?jWuX^5_ z9r@5tL+mZ?ciMOU_DRovk9Gggedm*W@z>|()fq?+<0n7*=!crkHQ#;qH$T@>KmS{& zFMDWOU(dkTd-!kF{@%Nv`x+;I`G*VR2WdwIa{`!v14wfV2q&x8fB)xwzL$O!xI7*9 zYpMf6_qTr>h<(J7d+!H<5@>E6$bi36fDB}Q6}W;52zDXJd@opb;ManIMt{I&X69CV z4fqi|7Qd;MPYG{RX5{3Qee^c0SYuH?Hc7tG(hI8nHU$|4yB7vm9 za72i6M!1LZWrG332(JGk8%)TCdDw_}AcPWhd7<`z&ZkI@m_v1#UbBRUWk`sgXcbGx zdQ3No%LZ+b7$Ac`SxjWD2K6_i{NC4^%DMiuNqrcG$*6PFc#l6nwcWRiQLWDqBlM!Aw2(ULtGl1J%t zc}J77qLDzUllupfewc4ld2CpS8w`n*P04ta=#np}jbDk6P$>!s=#|Qpm0-k^hsSWK z$d)+ic~nOXLaAziSC(q2mp&JUP}gai#+Pb%ma!m}dr6f)NB~MXhNcj10YR37CYL;P zmx^eUf_a&Ti3WWrh<~_j$oMCanVF16n2TVTb9tHtCz&Utn6+4I(YA1%#%Uxenvz3h zLsFEqnVZuXHk}!6wH7nFnS6w4nm?Ic0Ew7h#+s41Y07p0gB6*``J8D*ndf7f!HJou z$!+A;oQwY!n%LQh7fGDoIh(5KFu(b6+~=9TS)ApmPKJn6oH(A>Id#5SYfE{b+bM0{ zIiIffp5p11(OGA&`I(z{m7{2k2x^tIxSaLbo?Eq*`W2u2IiU`vpN1!dFG-JLnW48B zf~aVr6dFkM_n+B`lnzP^^?9KRik~dHqAuE^JeQ)uxQ!n}o$#5LAu6AzWT7wGqhjcm z?$>%f>5@L$d$J~NI(k8CN1_2po;nzyb&;YlTBK7did1@~Rf?r_H=uDTolyE9)Y+9u zI;IuEqZ;a=9Ga$RDx`4OenPsT9U7-s;%C-tyrq4PBx>(P++(ks*JU()(Wod z`mL)KhhEWYoVEHy7h;MDI*{(#t_b_B@|v&=d$11c zefvt69LW)t<0t5vcCe|U4a=$z>#OzpvH2RP_=>C~Te2p5vM4*UAPcNW%4(72cDeuR zq;FEK>$_3 zRI8p}K!?rruTCkg(77>2d!>w-v|<~!I9IT-YPJOnY8qRNXsfhtJGO8;k+bHdKo}A# zCsM{zFh8`n%&4k#6%3i zpwCpi!@gLoz8OTb7VN@_d%;lr#AKYh^$5mj zoW^Rr#wi=W#M{CqOT*vtC%L~Fp8yvk(j zxkD_y3mnUT+`w<_e24tWFFeRAipz1#%BxGtyFwYLyv15v%*1@m#+=N8LbJRK%56N! zx9rQhT+7ppxXpac*qqJU+{|$7%+MUQA_vTgvGuh^`1%k6r< z;IMd1E}PHjw0g~MyA6|!d`_>=H}^b#ukZW$fPaB@gMxpAhlYrYij9SjiI9wvjFpy` zn3*w)(F+}qyW;NRlm83V z6Cwm0D6k;Gg9Z~;K&UWbK?4vY4ycE3)hc-wTjkTp5!e(d@jy<5H8LJ7lO#_Lcu;a= z%9k)>%4Ee-<;+nibGp1a6XnPMkt~B6crbLQNSq;C{ye%8Xi=6Tp{{&Dw5ozQ9kOcO z%C)Q4uVBN99ZR;X*|TWVs$I*rt=qS7cgS>G+0I`EU* zxNXcGqH5MhWGWl_`Z3Z(P2n+m6?b2WE9+ zmT4xLXQnx$nreE-;sxrlhvA$W)=B4`cIJuao_hAl=bwHCD&L^(6>8|Ah$c#?qKr1` z=%SBC8fjfzUL}WZP@0&UlUSB?TM>W>A>Np}9f0GI-LWYho2&wNN{~jzdgQFJM&LwX zQ+BE8uDr(Ptj%#kWigL*RD@DzqBH( z@Ek$}TcX32Rp}a*y;j+2P7Vv(TdHHWNU*BkeJt|GZCdwcwb!1ja=9$8>~hO6zbvWD zG1shU2aGPnr9kS|2CBxt`kGtACz`C>&$$`vn`ri(`7!a^Qgn*dDm;-FVxqH_dtTz4zaK2iihb8)SuO z1zWR=2hz6YMsLD!%aOR0}Ck;81xB*Uq}@WRWDDB#Eko;;+?D?jt|&_^$Q^V3&veWQe* z=D;&sHEx@wJ-?f{lo7`kxu=muK~;-b5AkBKyrb&5?@vpuK!o(_%ZMx1lt{1!McgAIXnK*XPt;FT%=xl1orT8aRlB)izP zPD|x89}lNUMFj$nf$2dR2e;_KE^_gU6!fAPA4Ni~-6nkRl0ylJCYvllj*U=4k=88d zn^HypA#v!V9CLEG!#-9~IrbAGtx7k<5y22^^K#=77syvV1XI)4v*cIUVw- zUkVJJrGiLIs}<6aw`84RN<=XyINmsbD3QLho+X{br%ty**Vv9|wS1{dMHDnBSyBS1s%d%P z)XOsoFMWay-g26^yy#8;?|H>( zR(s|p0%}|<5^L*TSAI{fb7d($9of0OT9mhaHPc^%d({F@N0@|7uyF~3SX?C3nT}m( zQ=`cm;pD}_U&^U=H+AJ37BjZO z{Op%`ax1|e>lK(e9UgFjtJm!kx0uI8n3C+Mi&t(+o=be{Go{JA2gShg*Px6Ea? z5l_hARdI^PEatJQ_sot;<%#IY+NH|at*m`>es43~CgONT{@Q1VecW3E1Ba9T{M=xZ ztl*-SYI}&@pRH_|<@y3G!i^>%eY>XNE_3;c9`0p#iR?+wmbjF6D>Iq@N3CMh)|k;L zm9u?mEaTXw4wetlYL01Sn%xGs&)M{6phtFOI}los2080;i}1Kiv9Q=vu5z6veVN!P zm1~xc_J+v=+)j>|)MQ2TdbQ2%ZF{@h>3yD>oeZ%XXUx%W9noZ*>zdo<+1G!Tb)ZAm zoLw()(1k{_p}&{tek+>X(5#fI;|b}MUE^X{ayq`bb>a{aY^^Kz$+~2->S)up;;(^O+(vG&Td*959 zlCv!Z{qh{`zy_YD`6k$oYjb9^u;WT{u-j8bQ7@aym3AYhzdY?9 zo;iYzyY^Zoxkw8>>^uFUz?TGo6?L1mdijSPJN08TmnOYN zdqOybwnv0Ts6AJsTzbcI>ZX3mWn%_MT~tw#HQv@&Vs9dDjAl zUsf%1!&!x*f+Ki>r51i}7;bMEej;T!YG^1d*n;P0XXyttNYiTi31=vhd~!MPN;56fuASWIdJ2d+&G>a z$#{N-ET?&mABlVF8GztPi5{m5xj<}KW^5~oj$H|Hbm^O-xFR6+q6$HxxLF=l^p#;*dyt5P zY;+0k+JD{l6j%KG8O09pLklLJxU(dhbSG8lSN9PU2332$^nPsq$;DN5xS_1 z+Nd12d0x?RG&*o}%2KFfqv@8A0=Hkg0-`<1csIyb8W0D(M}O&e>BGh}`=zCQ9d3Fj(cRHs3dD@9<^Ne_Q9u-$8${DD}I;1G~U!Bw*saS{3d6)q? zUeS6=YPbiB)|sFYUr-2t9yeo^scdJ5b~A}Uq&Q%PNF>s@j=O3m>B*jH8kTEHp|1+3 znmTNe(U9ATY>5Nel0L=_obTJ z>5zEHcj2n3i)9n|sXmIQtUpdK{{{qQ+#gl1P@VDR%XFj91xj33Z|>vSC|T>$-6NWeX-|`mRS{=ik~W^!s)b#o4ATAp%S||!kCs*8MCi>l8@FI6lshBws6cyt`ygdCY!e@ zsuxGSXUp|nUk zt&1zI*++ikr+HoXxRcARN1$`x$hBSTt)fzw+Vql{1%1x|Z+u(3pL1mH8ouHiuWx%g z5b>s~W|7w_r++x5ura&vTSd2PyVb&_f5~0>!g_-Xs@in8#rwYi9KcB_Qo&Z0m07na z37<5IsW2OSF;Qp#4OYFGOPczdy*UP7p$WSDx2pf=q6p!#s0&vr#JVJ$x>=i@7Z$%N zj8(ttyZalsN6WuNE5o=8wAhD~0-VDPYib7Ea|Ud+F$=vF8;tmheCo#ph-Oi4VxC3& zlendFm$b9qTPA4>l4=XWwkoe}nz3 zT1lbH!K})%rR%bh7^`Aa!tn_&Xl1~n{AML_#!ZK+>I#jFoWE=g!^mong4)WtJHR)5 zly-c}E0&G_RtgX#=e}}#cZMX$%7(zM^u_kb#+wVa#Pi5O6~#*YuTwmY8!W!m46j*S zzDBjn@Cn93AfkwDsq~4$6B#k4T+We{Yczbnu1vVLTONfYH;9VM@;uMbgOE^}zIlki zlDoQ~*R5~qi-)|rxMeHQ{Km$yxh9mssyxBjRmq}DBxJcpA54-BEMH&Tx}hA<$)prtXZ>0gl?yH(FRe_f*WhR^kdA7 zB+iV#);pT6M>17By4BpO*Id3_Jc!=>)M5P6quhQe+|fLG#;6RiF8sUetjHoAV)4w< zVm;RX6PumvJ24fh(VYpsTPrcz%)Dml$pWDPQao?8rqG%Tto93mx|6b&ytbqZ86%{g zN^O=e{nRgd)|MKZt!C98{n59%)gt}MCEe97EUFq1)@?}EoZTqPyU%ZJtFwf)-H41c z{kpETy0mJqh>Z|GwtT;;o8npR!2+Z(TtDrsAYlY3ph`)TTNwmEXysA@StDAj2&4k_JR@|Og zEQwp()E{2lG)v(f{?;5V-~t}mb$!($J=r_`)$36>nhoJd4zb%EvyfZiyxh&|TLc;& z$g#ziQ@!8Ylip&YDhXYC5}3@RDQGNy#U1RbF3w<_?9`up+Evcl^|Y_>&Ex)w+vVxi z1uoay$?!?sHh$sgYp+p#w{qU?m&5-`d{W%cMj5$P2Fl-v<4btUTD0;o9mrTDUR;XN6gKpz3-=eP;Oe| zh;HnO#d|x|?w-15JY^gh?B(aogCpLEV-D(|zU?v|pWqJciC&c&s@9W3BIf?_v5xKo z9`ZoGzpXq$^q%sIs+5E-Ec#sR>x<<^wT-j{^GgWcs?Ef94x%KR919Q0Cq7Z~{nuy; z!Y!-qpiH`d4B{{vj2PeXF-7hm|LWTn$9p@!cuwmFF66fE;3|*gytps_s8O9qCG#)p zwj#OKYd*I&KM*W;(&?SZug8;W2lO;a=aj_T5}(v9I>ZAw5W{Wi8E@S1qPjEc;kYvO zQ}5#^9@6PPM z-ps_3)jTxxMDL!q8m2MM;Yi@^Y~JRhZ>74b__NFSuzu&7zVpRz@a5rMw_f?o-+f^( zxslsoYRc~o5rZR@i`l4}gCFH?uj0{ifk>SE7ntBn1I50{e0u-sFAms9@AqnY5J;s? z5V?di&4i+E;r|W%jveMK3^-93~JW6ciaF8WSD$2m|@@ATKcwH6!yn z4L2_}sG_ViCPpO}r9@e)C?+y0VrLPdnv!sla(8)qeSdie0s`)2jgOIOm6Ms5krOKr z=LRF9t)a0$9Sb`%BfT&06uA#TH?{L@Z-Lj_+27&g<>%e$?eFpH<4IlHkNQVV5f~6) z!GZs#h~ltt%NsH;xbRO4l!obxRImA4yo#Vnl_2jB$MULeNzeUWS~VW5na3X z=$y)$mN1FxC**>u3Oz+0)Ol;kv01b9(c_noDHH?4aLSDTDV6F}t5vZ=A_SA`NKNGC z*c3bVD@r}oIEnnEu4!AhZ{a45JC|<0rE(DpB+7PYU!GTbdgwTqaACuT5uf>ZhjFD& zg8=aX6uIS~N@8Dm6@eL9jR$(06w)tF7FJG<>uVqs%9)9q|` zRy=60W*cXAc*w4xZH67m4Rg@Fcxgx1C4JW@>er!9w|nq;UWBKYfcGvzJa^RL11C0f zQGI*&?>Tm;5EATW&Xy}PSB+e2eoCK%i$=5NaDr*5@>0`EG@*1F>Im4F(dRDg>as1W zta`&x!)&(Nk3Zt-8&E{G8Y9j*y(XkEI~7@M(M7ud{EE>a^U51#tH-wKoCPz1TgWeEOX2k0S#183DudAyFS6AaXc$=)G;ykApKF& zN#6@{Nc$-1EXhsv0~0y_?(;CfC_^h%w3HG7FCZ*eTky2h2qGe;qmbfDFI&l+E!HB? zv}D6yfejE-J4K{3tR+n~>ryRc%yrObp^bLXX1k+SQ5&lzFFi`R?bh3kI6+8g@)YZTL1ayEX2%nKwJ5t&jCQUaH&QL=*FB?_f zDc2>I-D8E+1W_3U&RFA(+Zt2Gi)};@r zb9km|kcBu(Q>Y}>kOQv?&%p<#TEl zj(&wV*~}(5AE?!wI^n6U_SE65`O~+okia%n?9n4H-E=d>v^I3r4P8~J*IPGw14$#T zJ?`6kTnwZ-Ncx)^u(s+@lbO8E8%a_BiLaA$V*8~@b8j{QN>s*)rj?eZ|4ci3(sQve zYO2AvJh;rU-d#k`vsAW3woPx}{r9y$yIS_Q=Q76)PR3pT{SVWEz)qD9eZ{+$v+Bn^4X$v7 zVM}4!=(oDqZHs?3v|&cXm7f6OZDu{R83El@lA0CIaLeOR;l_u=%%lnnW=a|cou)!* zQ9%eC>|pqQ#Wmn`4kT#V+Pu2fMFVVUS5jo*8POQUj+qf{ZA2Rt9CbrE&QUQLAP0VS zcsI!a(K2N@AOi758OofaZoo1B72z_oz$|HFf=Wb0871I=$EERUP{GX=J@_WRRU(U+ z$rA}Dwup-LL!{t zD>y(H;>ajQhqFa)B2r0N>tZyipeHEcAv+?q0|idH&aAqWuK0-QRcm?yx9OCxZ{w*h z^XMeE_H>uQENUQqI>-qKiCkuNB;pbp!G%gRlDUcK(TadqtWtEER=8+3zls$Qz9~u{ zg5WVhM*&uR!#oRe8neiw%aOdauC?8c3vYYZH}(Xlef90P!u3m`&Mb4o-BdsQDOpeO zA*onvo>Q%|NNqIE3C=u51gax}s`5j#YNYHM33UKVRt~hG&BP?e%0=gWQ@RuzO^wdk zi!sVdX129&8avznQh4N-FIeqKxPBX89CjgpifO@LkBcsX5|OaH%oIT%oLC2Yl(ER7 z*J+h|DuhnbLx*;hqoi~jDQ#nV-<8fa ze)}b-5?FeLI0~4^@Cny^Y`o8Qy^N2Cbux3inF^jc7P<~(YKAp@pejg}nV02WPnrm0 zm1?UM74?Ogt2o{hn;6A_Q*VoxWUR-oWgts7XIx+5A06+R&u6BoZm&?_A`iNMIc&*5 z1-li;81~74l<*Fb`arKDa?X%lXgD4?0qUC5vJpk|im10~&F<;b@+&c$EjQW_viQx? zU=D^g>^d?3o|LgR#&Lgn+-HsD_|HEMTR<2RE5a0ONvP>~5WG*aRo=!MR}Ygd_aG3|IJp?aAOfl#5e$yji3FD9A@6w!Na9 zv?mD>Sxk$Z5ftn$(mEYt%wo0NcqKKMo49fxXLbpvM)e0(9TVie+qq!;Wa0wSU}XP~ z5BoOyZR5i0UN1c_u46?!1s-)EwP0>Z#uOmzy<~DZo9h@qV6-CvThkQZFQgc>?j@SHQlX{Ot0ozN?hOY2HMfxNV`k3zan$BpzOrE@t47x829?lCME(9I_ zZUh&u1i>#3tOvjh?ep~LsNh7(giFC9jL|eMumlJYKu(}O?k42T#Q@I-t<2nZA@%}B zP(lsn{;%HvLjYfpybLg-hL7D?&G<&6C1)V&?+c2gU+0M>=o= zb;^Ij0)JM}-bT;-I&axbuj`5m!kn!A_TZZ4q>mD*eCp)N>ZSgMKsT^USNQJ;1+eZs zEdVia0F^}VI%$~tE&<`q$}(-agw72oA=kRF6L-w|tjd%`?BDJO4b#vA)vz4!2gwSL z^om65%Au@yhwC711~<+IpM;Q351~Hl(kyZ3tWcB$Q3$v#7W~ET`ps5`;s{Uw?y{(7 zmYOj5t|$tVh!PW!0k6;nhmI|1$`iejmA(*H3_=XQO@^{yuS`+EQc;rZNb_v*cjC|! zT+OYJD}m^+(eM!U<}eqltc2$85dfo{2oV|mi&`>^8NsF?b1z?v1rk?HqZEq@vC+aV zG5s{LhXQ53zHuYL3)i4Y9A`-j&M+OP&ICo!1VNAl8*bsk?*x}@{1WV^7HsRXBqrZb z7`h6OWDpkvDi}pVA9?V1NUr04a2NnfK(xOHkvEP}2|x0~N+!(6gxrEotQ=CC5Ks!O zQ6eo76Mhms(IRy-tWM|M9uB z$T9lxi`EMeV{Ox*-~!krDch+Tn^6~_(kTCe5lJnmo@6Q?avKnE3jZ$Nh%y^5@;(Gb z3&XNApYQc>3p5LbjLdK()3QBA(()hO@~RkdmM)<5AhVMO3w$UOD=F;($AuE|%lRr~6FoDX#*!&z zsWn7LG-oJ0Zpt)I6J@p_1C|d9iNqx`G5v-ru;|h?y)GXCZ-lm{sX$80a8DBCRt5XG}Zkf+V_gL$|OSz49(l{{#!zFg?}a zBvemTbS^@vb3(P!IxFEqHF7&S^C|OhKsgk=zSG-SNo{5dThNL;{O$2b zbY)Pm9oG*InJtXsv_9WOHYbBNYxK#??=JUHKgVial*tz{AsRNo!^pEs!PMo($v?@A z?s`wnnlyyaOG^E5B8Rj}vyjQ0;>Ko!JHHh15S0@TbyP`_w}fp?OVk5ZQUt><4q);f z-Q`7vvPon#<8<`Va1=*x6&G`}y)bPPLT)Ir%*95oR)sOoQldzY|6sdBb+eLl6~gRQ zLe2MZ&Q?Z9j2~xL46%@^JAta9t(GpcD zWdp+M-iXUs@wFD;a93p#JTWD`|gYmXsNlwMobBvWo%Ve zg|fJ2^iFfs2Jtjkh1Os5D5Ng4Pj?p9hIL&S%wNcDC}07i(1ROain?qMAsHvA4(17= z&}uSPTRT=lFDzu~CJDH*Bh8E+%@t*LElfX@EZY@5G|*+;|D)gtEDl&SNOZ6s{|Nc^ z6*mzpfPxlqN00sZ^!<3$H)|=3T+2QL)li>LY1>Rw4?@NUg@?CQj}(A_VNmKMfEl|3HL3-X+CN6X9KrJ zeO737bw^v!lRoEHORg7{R=Vs&d92p$1k+Hlg9~Yn44Kh#!;D(JigOQ85F&G1LHA=v zcNYJl5NrY8Kr%a#Bw|1q`G{8|40-Fc zE)8}Y@{=FOqV)teULNUb0WO|W&O-@Q2=xmIm(X&{|Bw_F6l1y80NDjXKbKoY7gI;q z514=uhJaJ!>veZQb>p`)TQ|h2k8ws6Jq50IVfJ=y_XLgc8rU!v*bpW!u3wWY7hAJ< zfA)!ObT*B5Xc5?MfHY{3)`72BN@zFA;%ttTvGqeu)z(-O4EtD--_7N>8|a z>t$@`nCQ~w&a?wVNA-Pc_#Kdxb<3D^&_g5(|M^5y5@vrmh!-PRR}mHtSvmL#UA9sj zx=wJ9cW~`Aii1~pDVaX6co)YmY5B5Z-uM&@w1UGpLW)4kOc`}w4*(x_|A^CapVX6M zIn_2v?5dG`nb(%D&|7mEhO_j}<{*1zh#-JjemyOisi~Mx8By|>1CrSzWETY6^T;A# zkxy^J+*3}4D<&n`CU4U|pKO6o1urRViw~A8*39M<1?b0pJqtx=$Fxgw2avUNE>On0+`3KrjXvnv~qtNODk>4rx$JTicg(Xe)J zS3cxPcMnXgxtQp5_99;`RxA05ffi2tRf^#{fyG&I;C$1mAtlA7Q0qvP5(mALeH_crU2M*|vy36^=LrgBYu2t>3^ z82s&C{9!pD8e-1n{BNg)*>Xn~QlFc*`3|XzPpP{Vmzf%fkVeHHpd~y~ye*@J{h-C| z9JCv;G^38IS=E_q7OeNzRb`99LT^$X`M&#maD^9n=QOUNxW9Kbvi$&)b$i2mJ2Z)) zdUHd|hdG03`g)L17jl|vbz0`u_`==T(m6cME%L+Fx?J0n#iL*vwtG_~`pz||!9Vjk z{xb_|oJ4`xX49KRZ#IC@|8K2vHdql|*^{`oEm?^4)wK=yK3iRR%9lvVT%C`!X$+Lg z9YM=nym7GP8lJ9Hn`YESyFA2;8c@B>GW@aeI4|^EX6dd zmmAC`4^<6b{I01gbg>*1CKY@r7IA|4g7s!<@RA>+u-=nj0k#)g4I8svwBT7ew`qAO8d9?d<2WV0@xDgmm@%xsgYqbpDXU-N?JB2-n9Yh(TVn5sLc=iasjVW zdnG&}yj>AJJ3$tQrmq&=>*CbyUXx5XY|*^KLDug{_tk;fp`1^gL_D<^jE{A_@nIaM z)wiCM2|%O6%2Vt?&t zAA@3ol@l?r6Ef&K{Rng)2qqOFacC?ekIE!-5{W1dhUvlKygswot+t!Bg1_RhV(ciN z(P=e1>aN1y|8gn4Zl~V!)CXTearf2vN$3^!9U>+wE;2UuT?{%(PEuA{USb|XW^#6V zMuL7uUXWgBRBUv#tg^PczQV@J&eGPps;;W)vhMb_^mahOb#!dR#o;BF9c7>%o#72o zNlPmmxCuvPS-VLW8c>6aSBpmg2K3iYXPb0Be;%77H0U>)f`v? z739Z@{~3tcN~?A)+giTf5{oOB*oJf)>Nd55tr=g3+2Wzó=Zgqs61FSe%Vk$oM zu!{zd@#4z&UN+&mGbd)5Tpv^gjj@6T4y7mD?W-C|M}?(6iyV!a>&K29PMVC}x+DeI zxDkO{DW!8~Ffxl@iGmL2X3fr<1C5R*FZIUOiyA;m=MkPj*QI3z8-6@_tf9-JPaoEg zsZ_Yy!jCWi*RRy7SO424RB$0H4*NQwm_3vwlTHMsa(TouI9XA0&=V_ASSQNoV5?QwS_8TKAu|yn9HKkabga&{Z z|Kx{5n$eSV)gg%;e}gQ=q!ircwVrxXPFbaujg-QocvNz^WmjblLJ9{Qpf#VFXVOy4 zPajT~WFC9{XGDPQ5Qv?TL-f}Uofn#8CuMT1QKw-P*5Tk{%jh@+k(%+SBagJ1vd|#1 zAz>eiJ4)IWr8b?HDIy+G6=rLtMY`Kqj08uAqdPv)Bdbp+hewFd(aPqLMpEadp9vnI z)KL$p!&_ok5?idXVr8l2u~Bt$YL{QV_8ywkPD_jvxZbL3QU2iQ$7m zEwww(hu6mHWJ{D@%rxX}VCZsF7@|L}gW=J_e|K)q4gkv>LT>X`-{X^K$66!Es(BMOzK|F3J+>n+Iz zd`ld-ORb9k&)~-AH2lws=jw6DLBZZW$$tDwGv%CXZvT{EPD)DU`aj|l_;XpyGIui# zq(XEGOpCRka5^?jA!w{?0@C>7Et*sS8-Y?1J)riqH)&&obx56_UU#lOmFOkvV^j*i zmXJ*m0dQvPSK8ioz)ifUBWX(r%nrtzi`5oMh(d8vP8L@L(teoNt*K$y)$SGD)b@F576$`qnCe<>b3N_-Xl(o>9 zWipfPDW)2A;j@5RD5KQ8SklHfo%pfxJ0%pPGH>U!LjK8<5aQ4sdZ*EMRLGDF3Z6g7 zdCEk}OP#Nxn~fOq&bh_1Z59bCA8r@EP@0osrR?V^ZA!0z|GrX=StRH{A8OUBA}owo z^<_jy0;V)lRE>CoDoBBr2ZD`nj?si4WRNyOZ2s|ESzM|&;{Y!cerufn1ktOIdLm23 zX?jHD-Z~9J&sc_PJmACU6G(v%R|$@(HQg&f-xyVcVzzNa-704xs+G-lwpm>TCP@+) z3lHcltZO+YMr{W%h}mwV6U+$hR!3FdnG3hnSkQtFncLh>l?)slM{i>aS8&oaicZC; z@o=J3OaalbLR4Ew_$NT*0acTkFls0nwgsg=u^sxHEE_0WE6Wb_vaWm|b$AIC(1uoJ zzjQBG;XAYW_U4ir0G%;C-~rXbM@Ok@re z{fSq?s1m*E)njI-3QJkWx3l(*ibIo(If2l%UVWyTXRjw9m$!cI6tuDb7 z{G(LE9AQG8bhm2O=!7Yp2X92yb#I*qOeK8R@hX?HH)SsKns*%9!0>M}Zq}%PnJ?|~ zfs=`NZH_2bs@8BIhJcH}Jb}0{Qe03-BfSuimn&p}4!MqHWv~0h=;TvF@5xODS#lEkvzW_X0jPR|jLKIH*1* zy+Q+Nk&UbeSED+irT{nr1Fq`P#@ao!-T-QE@c|Ef02sLjur4x)?=$q)9>C709~%_N zOk-BrFI{%D*NRBGzIDUkJgJ>8%-m|fJmyV!?r}0uifb!6lh5IGcZLzlEp}cjhEl68l@1|$7*)w@I|GnGHY`rd5#3D|opP?Jc7(QCRFy8r$Z`eJk zoco1J&jR+wCUijutaJBuH$^^bnY(vd?VygjUWjb^e(!!=}*o|S4envaJ4)6P;n6xR6w%nmZ)zKS> z)cM=g-InIL-s{=Sg zu~puQob?R|IXGX1#mabzTHI9)|FkikfSALyWd_V)8@l1seR$sz6`FJ%R@Q;UB-NiN zM4l5EogZM&YmCagDBw`tR}p!~(Rkco6dUm{liv{_>17tb4d5Ld;Oc>!ScKsN=F&w7 zQ^93!dP7#G{9CJDhDWOAhShNhP@dgvYaxG znoulVU2WJX&Y+>_)3xE?|09YI;T4|WSRnUsQu$59VU@=bW`T5#mZ;3j7{=nvOxPP# z7BR{o3N6m*;bTF)A}prNC%&3F@*@-i(|!R%YOvv4Y#DvzLN2IbfazY^ZQ${_5@LWJ zD!SDiG$SAJ7K3CQhiTz8c;R70A2se#M;@XfC1Um=Q>`!}BlTGi3SN!$9R85jJEmU} z0bM&HTn@>G1E^yysv0Twnkmv?DoW57qTVavVn7}oC6eK1fg3>jqh~#(&5WB4^@4we zW9-of2HwH~3D^dT+BOK~wJ;mHFyk|#q(?rVpMZ?Zb-@d=9b5Jt9$ttIGJ&V@Sq`ET z%{`(_79mtlWhbEH{}q^{0X3fVG*&~|Bot1e(_!Hi8VJjwnYMslK4PX*FePUR;lVMK z=WM1xQV#;9A;U#kvdoZNbfrSFrtFzXl{HXAy4{?_j#&a%3H8olU}VRQoZbCaAA+QK z6^bCDL3lC7T-MtkvO`mCBlqQ$U;35a$kQSqPtJvh`a$AAe1VKzA_UgtI(nv)3?+1w z+_3@R@YN#=_S?tI;y#AvdsWvh799R@j==CEVs#?+#8{+QO4*4WEZ`os$l-YQL>?&- z)1Vv&&D9I~W(r1Id9D|!4We0%)E)t1hl$zNEFxcW<4cg3Gh~}gkfQ?T!SGGl1u8F?@1rc16lqPDSu4Gb$=vYG`x1DTj6+oRX41GG@Y<##UBpN+gjxaN>?SW{Y0e(J=)!?4OK& zj5D5I|8B0!aXqV}is5Kxk9A%uVtVW5CCdebAd%Wfd7(mB1}Kw$C6dO4Fn%1Wew>-9 zDulA?MW*Q=!s?kJ>X>dRTX6`26vbUN==Cw@hW=_I2`gg}>ku-b_%&uRGHX|{O?}$m zxccZV(WkUZfUz0nl**AsK24HH8+)}2IRGiNmg~mqsV><@r}WZ%Icu_-VaE24rQKK) zJyQsg4;YAQnv~e@?dCP8Dy^Cv-(e_lcA;=8ZIVRho9NbsV(5W<7{daqHiqcKLe_x! zX`^a_z}!tF0^F(4q06eR)iFhdMy(*`Dv_Biukg`^Sj5Y&45c#a*VQSbN+I5Qou=jo z|GS!%WIDi+uI7@$Ca4nXy;c$N;Ry+nY4Ta0p<-#62JCPqsufrv-S(!T-r2(Xs&fK_ zUv90vE(NoAL}F^syN!oRz*bLz;(U6nEcz|SqNR}OD%0w#(jp?@%9rNpE%a2+m{ix@ z-rvn)-JV*aVww}!Hicav?rQlApb+Shl%*T(1N31LD*hT^tYED2N`#W$)ZXZ2D(c)O zBDHC4hj6X?qLm}^l_<%_W_n0xzSE>^)j-jap-}m1I2xuouCI zoPH)f%_Qxv?E?a-qOfU$S;43NXF^&_^x_-T(!maCU>aCcM*3!DI4HTaWlDi?|GZG` z)IiYru57|CC!4k}={~1ANlk6i7BX-Nx1PU4GIu#J*UY>@q>m_X3vq+)T)1jlPFuvi=_ z-0o_PhOZ3MLnY^}5)SX!zDyX;2%icZ8B5eNs3i0X*A*tvTdZa=EaL;+=Fpg`8L)8G zA}k6oh6#G06a!KVrsbBxionKb-}$TP(hA|?aK^l3w-{eGrtADn1l|l+y-3 zBwoC+1eNd%4Q@I2GBCrP2{I_%`5WifF$^ao)<|SgrZ3WF?dUda={ggNATk;g>JLeo ziB!NC4jerfkg&Yg|9W$heMC4rE`n920VDKqiCiFn2N8Sn13R$rX6j6?u4tfGHsiDw zGqUqCVw=c`JZ5k#cV!06a-bd!)$+wkO0=Okt($4-FXN%+%GMM|H6BT>3a^}U@~Sek zBn}4(_HFc~*fAn07*T|qDr>|e>*6kL^F6b(J+m!E#4(IYaTN1w|41#y6jJ3-Zx#1^ zXFQEb1rC;1T|kt9XI|@dd+MyTzTZXbLn&@5EO&rY7wSt#M+qy@*%UM$_b&Nr>3TH= zIlHi*c(CMdE1KQ0G(YbR@8t%?kxY;_5B`NMD_EJigb*)YzMMh4Id+#Q(-E)q@)Cke zUu#Tja%!7)>Q&7E%P~T30Av6%xxx!kEIsDz)MI2SNh01!1q-=E*JN zvU?Y>g{D-NZZcN;@@YGFf|Ar&({NeaaGNgO74BWfC6}l6UNVDGhW}^|)vo(p8XE3g|8?zL<6r^aN6K`N_p6IvHzSZT0dZT{@TWv9 zj1yEc#~zGP_iTu$@lL?DG~TN?AY~#`wT(je3Y+QV&KXJi?f^z6)tGQ{{3nxmUtJdcsi~?pR9+^^o-ikoaf?B zm$InDpV-)3tPO-SEVm}?wV#*q%@OiE{-34i?uH-R{}m8)D+Boe6Zv6yb<_xXaOUxv z>pI|Mnl8I}Uddeh61jsn1#wvUYOp!`vU#v60ULh#OOV@f*NomwvP#3}IzJhFf(&5m z;hV!bA(exiCwqxQ;FPI`8#+U;_Vs4aY4HH-nV=!^tT^W zW^*z!C#U>{P7rKpmTcpUfppk7E9cI5PchxaFhQl~f zV3f8C;%QHMudDOK*Ym3lak7|3I@t$cg)P#A#p}w_sV3fPkf0vJM zyYHa=k#jq1gL}xEya4uW+heP)s;_W`_+{wj|IF_u;1epky}MuBx|Rc=TQh+Ubw-xh zUymLF(T}*o6ZN#7XUXQe!F!OIQivv~i_{myDd!*dWIfhfio|c`>>4+q2E{2k*tEC# z0JFD0Om%ddEZo0v(|tCgR!H9yAV{G+uKE^P+p63nI?TCX;MchFkNt=0{08Ox^MhK3 z{>>R%%!P~cqNQ}vgMO_eg->ed<@-K*>bm0`d1?a-{CElWD|;?h{npX2aU2jhB8rLPU^pg`OJnokY%-B?N78|fc6X?OqQ@3 zK7|c?t&)`~6u*C=G(l{K&d<3%EOJqxLd;_;&-OGINC#pt9a(1;dmdO>wcPb^lrE<)@mN|#k#SQrx1w@^!1V`Q2I zaM%zbFNAI9eZ{A*o1K?8HKyZe|F>htIcojB{p(h6;K9KD!Y!u3gJKQ>13q?4xUe?F zlAl#Rc8^;>M0d7kp{PNtnxsPYo-3^c6Qz)J)r8Z@@(`OPhzxW(R9EF}bgNLhc8$d8 z#Y;$d$_{5d`AK!KfFp$-eJRx^)3L4^u8Wk^sMxD=rQVq|*U#m5b<4-rVX#UJmknq( z+46k(_19BKDefj;{&(>b$}1831|;w+`7l#&7_t~^3?PCY-0lDZCyWQdV|Yo*itMxq z4HQQ<(hWp25)lr!yM_WukCl94Q4k_PG|>_yU^=NT8abJ1H~)65ZnWT(;|WL>k1H`X z5|2VrI-)+jjtJ|*TS^qD|G1kf!&%B}GR7;&g7@MB%t8z4qi~xG z?<4a9ro{LSxDe|c=RZ8#-v^vttAaQUlsIsKH_Hr)!l z?Xxm}{&6rWD%GXe(F=cz^4(;Gc%n=?lPIH1{6rLY5$K*MMFDmaAyOw7X|&8bz>AyU*K|65cxR$hZGYGjf%4)0cBr}x@w`*LIyowo`sjnmQ#^>>THv)RJaX;6HboWj9N-el@0#uW3Y?EQ^6l|Ji}Yup(em@nu*Gg&X-wn3Sn%2b?hkg| zHRFA3M)}b?7_{T9MXCsPN8a^;_ny|D3zBtS_#p<_+AR3sn>~A#hw72-6=yYV@Lb0^ z$+p|?j0s~H|2592VZ7b{smbatg`DTdPo6|(%caEJd222n%nqidZHna(kTJK_vDYktACvwg`BbOfv%6lYTtPsqqDtaHoIz~sULRqR|ov>mw0bwqd2 zv4d7))T1c!t}$^yF*bZq7(wAj-DS;A!gC&HP&X!iNB~`Liqy4Yazw~+QAnTsSooGU zJBvL7|7sYFqL0Ys6YO~me!7z1j6ya`C@qPWT+APmjpikB%V@k23Y1{n(55H#3`FFyvHD^1w6)3#88g;+8nvbEf~s< zE)EcEJS#nzQJJz-YW$0&t5@JR>lXScNVkL7D4B=>xqHAZl9kd>ZUynxX|rZgS(BjshOn zOvnL!aREXE(L+k@lQe<$3z8HwEkqiF(Ir z+7YWyJ?Kb~<~Db2Gq8_U=>)f-HQxDyrx!_mpA1HH}s& zV{wHLQuCzooqKI#BKj&+I{x(@3?r;d#Y{Kv+2+`>^(1YOAreyJZp!HT(A-BzA z)9x1DBS?*GJLaGq`}@HBHujM5{E}rko5%_j?T^9*T zoa|yz{0Y_z$4bbF-Jo~x(ZsN%LX? zR#%26D(M@aIUmg2H+GB_GFO-5u!7~$yqD>*rIpvH5MHgzI+cJ4sk~}YBU!zi;o||A zN=Y!QZoQE@Kq}0fAiUJ|ad!KjgYjhHjL2gd#MUh=NV4e`akyi#y%a3H|1)E>E-=u9 zzP6!@1!6|ig02F*>s=oW=-xuX#(u>ys4HD+S0~v*j&mZx_Sm0T$C=U#N_TtPJ=6;^ zHOJnJ@~u-#Pq{oOqa@fv?q#~yGaw6R{x90Jts<7eOk+ENj zJACDmRt#>o5$CPI;R^9!n;My*T?{#4`Lvfo+@(jQ+wDe+RK~u3xug}2-d0}?v2UK@ zyblcAyU6t4Lr-sd*v;BeM>*oXfS0_cTneaLdbb4~^T6d~1T7neb24%WkFs4kV5{6# znbtJa1~3n$Q+4I!R^!rLsW@kYZ08@Iw%&Bft3qSD@YzXmceu@S|G54<+~Ou}$+5uK zpgFzkjhuSE&Gd1kcUtLn7sQRToEU0BQ>c*Aw~te=*|2YBzL@K5m5a_!=B3OqVt z_v4n@`;}B{Jz<~-1?0W(`#n>Vl)`JAwu>+`kU&tGrjbL3dI&DTDX!()yV&TvvHCk{ zqYAtuF2fr%4?D5Mi>`=rG{SD2$n?Ob6!7KDZ zBPu_~Vv)I*FQw`GL3wYx#=Xe&Wn zyhMy8n6W{bdl|_?MItbcMr1FV`m{%MryE|H39CM8a^4x*zPnCR8_0q(}cE zIEe!@3Q&wgNffpjM+0m)Oc0<*>cX(=35w%|sL;1CLlO)bg8HF=QD6&q2o;?H6Ce6I zko!ds?8O91KU~rt=@W!K<2&JMx#&>A?20jp>OSvlg+mj+^Q%VlGeq>mJ8QcZ_Ip2i zB*!`$s0MSNO7z9PU`L`$9yJ*_CG@cxQbksbq^#Sz2`j}bQ>!8>ty5Ez>zTU;%z*uq z#c6Xu3gm=Tn+5*6B;fN!CSD#K_VB zthvYkBO%8sxc?fYS>uN)?8gY~Nq77OD;UO&$;VY>E$mq`)ay!5%!h`QmL+LL2fz?6 z&@qi{L$xe2gRsh!I3`tEsM;w>y{o=%?2J6*!eNT7D@&=t%#PHPwurkp zqzQ!r5KbCF4s*G{Go$fo#0 z5CABZ(kLwe0jSa|wbCrb(k<1}F6GiM_0llq(k`XaGNsZfol*g)PcW@g7cfWalui}3 z#2Y&`$xM?3{YTA^M+d~sKD{SC)zBT{$Jm3-$b>pMa!)f{gFvk`sm!+uK%2239jd4n zNjR#8yG1)y1{ZZRAY;+&Q^)kPnUADEP$J24sZJXWx+}8ITHLR@l*XF-PPP+I>kQBG zjKQC4iX+X*$z;zT`>H3XQYj@?Gc{H*Mb=|g)@5B*G(FR2b=GKwR%XT0X_eMVBv8CC zP|$oV^axE;b5KUzP0Es^as@L)|3bKgG1RE@g=7P}&U`m+1%~GdiSsZ90&}G7+S3z_ zpc+xm`a=jMN+76uh3wP=MJl5ieYQT75$*e%y>v#y%ux{o%pZN3F~S91ojJrzvu!le z=Tn3ATfo*M(g;9+2rz*vO<9#KfR(M*V)as%MOHSY)@eOfFjdx?=CQ|9vz&OA*y!$drl*ZZr-CWFwc>&&YI2Kf5K0UT7RmJ_SIqM@ZBUL}MFn|L-VAWk+(52%#w&OcC z-3FFf4CZ6lU0~0R;0NyG3)bM-b<=dr-E5Foep^CxgInKiGOFFp`>3HpF(d-4FGpS1 z%UdbkQjO;AHxAXwUw~dc-QNfa;(J|%5yjW9&C~-?pJ2SWPhDcw$kzWLL4~DJSZ!GA zqPF%0J)F2qahOJnwcl9$Q#!?u>FQYj#e-nF9wdn4IgV01w&rWb=4{sHJT_Ab9%R~O z*+Bkg*)3!f|IpxbHs^Fk=LI;~w&c`;Qp!^F%(2Z=vXSHkwd8!3FW2+}8Kz<0{b$IG z3*&cRd0!Vnvd z>LcC-9t&S4qMpfk4a8AY51xr=#_JN?Wxn*~iJd;W+%SXnK4YdIjqz4MndqtuJ@%x6 zW&rBP|Ay?ymTWxM(xdKNq{i%|?&ETPT^C?#yMF3;UgUR7TGAU$NR~~coa!^q>M9%v zeXeCqgU*`Ujv=!}$UHdf(stg%Fa@{f=-M{Qoq200mQiCA$>{`2GT^>u zp4_~nx96_g!}U(W76pQHKz3Ttbi0S?Ig`mI@A5WppjO>*R`1N#>_OI2DV1Wy67A!@ z#1k&NB%3wUPG+T|mULBDu6~oU%GZDjmiDqYKt19$*#gxDZ@=Jz&lmvK9u+4W}c_D*Wf z|8CRwEiiC}Y5JaTp1ibq%tx^JLdZK;m!RQs=UrRx716aOY6AQu~qY$^CWh5#S~ z{W1_OP>2*T^~+fBP5kja&~)TBX)~YelB^FpsbvjsZcxSRT>tPO*-?j;AM*tzXYQfG zo!?ol^GRUw`{MJU=H_N+-DYLrnsxSR5A^iDaYeuJYfo7nM`sft+N1T|)q8aB{~jR) zWe7^b>fh#59D)~qCI#5ueYZ(upL_F)a( zl(%*U&f{tiQ$V(HLAGoJ9^EX3VD*OG37=(wR@zDj@O!IT{)564e&H{!@Y%CUA`9F_ zw0F1R^0f9A-7ba{Sb=|kUV*o7NPcivOZb-Hy8(eP21D}@+X?K6W?k->#w1nAb>G7E zSCmd1kgM2YQIQa0ianp_WKHc@0fMn!N2*!M|o)N>6F*@oY#3p|HpZ7mU#{S z>~T1ZS9@}D*WJVRT21`FL&CQahQ&~r)p#HG42#Rbt!byXpRI}deW##6vHGh&b<$~l z$s{Tymh1J6s0>?2Am6-=kaLLF%)N8FT;6?=;bmGR_8^1EuW-RsNGkx~X-WZ>c^km<9a=*>3`JlEP!^`oSq;IuYvq{xg zU-hsrAfQNulZbGLd=!zX)atovK9G^h^qS0Kx7sdtk}-0};G?vhJpM7;_8=$0z!rS7UENw zu;B#_slZr#1*??^X_rSRHxbLPHS0BxU z6#0o%8l642bn=7=Y08j;gaUok^k*TcY!vaChxL@!tP#4NLC)f!UM9*7&`tIw=^WeR$>D7A%n)z)ZvHN_~uf_a~iBH`63xRKiUPGb-_qjWUVML}&I4D^vE>9Iioh8Y4-oDs6kW?Mt2 z4b;|aHX1k=jpF^N-%|In^afaM1em2)alAA{OwLI$|Ab9Xo|yzJ9ECDOP*l1}iULqh zQBs{%1`-@vDMf`|p66ACRDXi51z%Y5aRpaeT`>^i0wF56qKG8&r(314yyXy@bJk@Z z8yf~w;irNP7HX-5ooZ^T-wes>kndRdB7+O6IL?S7N;+bREz)|TuD!U}3v8PXd(cD# zRtad3E-uS#v#%0R9775bgwSmPUW{n*afT zyQx*(w%qoFU>XjJqs+)EqB1$LGu?GZ+o^9Z+6ln=~e&~5+pI;}u2*Hmq4%JnJ4w~)5 zmT$Fyqc)qU^R1S_iW#j1WUTR}Y9Z~h(YJIu$Apf#-cHN1uT1;w7sBi+Jg}DBi^ZP* z{v2I8(2LCAT~N2}yYr&4|LD$s{|`XF8AEu?NPr6vFgPnA&RcT;m*X7K z1_*YiaU644qAph-txya_*89e?a`&oixS(_zNd@?v2c?i1V0eBBhv;S%D#EnTcD1Xa zVW0-W8&zz@}U4}~Dfq4bUyyTF``hw0FUwj#nkCT=ZC zc{rbNva~om^}#jvNgLWGfCdVi*+kh*9JLD_h0zIL?DydIwWVzea{1F|GMSxVC^B)HrkhudQ zAfAIulsgLGf>JOlMePiptuEG~zp*150W@R~+X1b3y~mq|P!=Vf2~ltO0)d#^q~*+p zfL%VyiBHqrV45|;i(K-O4mjd6S))Q%q6}2CY$*+2YQss!ENU*yj4uW1&j9wYpF=Fb ztJI(mYo4?s)_ z<4!io)vqZ9>JQr^N;+?G1P4Tsn(H7D3eHQ&^Qw2fg=BAg3rR>8O*W7f>!oj%0mtk4 zkgo(xXd?k7#MQF&nGt`-vyCjhlCEFN~V(U__@Q)I4TcBMOqJuLWtx91E zVh>07!Kqd4@NoLt|F`bck3`G{bfJpjJU9TC0T$>|P5fJ%_JT6*=mHV6QUQ5hLWVUB zM%v~wn}tjOsiUz+Si`2({qgU*4`ghE5l~tG5=#S|Z9>k0*WbWHrIfF_YLy^EG z4p>>yPP4ZHICU9j6Xrnr;>wHMssqQTRqh~Q9FEc)0lc(j@z4uW2!vKPR^6v zC^fKf#bIf>|8UbjjP;v+sSabD`&0WZwNunB>|XA9-g@QFffKBWAjLUFyH%CbLORlb zcg4})(Qq1I4NFNYJj4XXoktN{WS)TlZa2+;gC2?4&6`>{)J zF}+#MV~^%pZ*Mkv)iV=r1@72p^>sRGarkhcjqcv~-UNe(TYy13bHIDSB{vYA=xb5r zwjLh?q*cZCl~4N8dW=M{@8&;ScepzCxc$J3a8`^P?=%wm`G zR#P{#^4(~xVf5xymlNF;e@ZR3fefnGzM=a!eOWPg-cqd zn6F-BAT?W1SG=~aFQ$Bl%MFgF+IhNF_s~ZkeBe)P8JYiJ*hTqDp2lORc0^X=T(&1F ziP2TEkZi`5KBs_gh!g>U)jOKw7$@dxvQ~W7!)myvQHEwzV7GL+V}BkJfdCb6`KN;D zU<{%l0G~5n>1KYNHZw~Fa%AUxlE7{I=4~{Vd>&|7rvwH{M0eXa1+0X3;@5Y6$AnEt z|6l+&K*+z%^iWY4Lon2CtTk%1*J_5xej8_mw19Sshkq(qa`rWb;{|`3Crz)$bs7g= zNHBBcb$&)gX%l5w4GIu&n1u?A)`c8la07se;5UAIr)#fB zejNpI5qEJ~h>Nb4g%sCXrk0B61X=Htfh(v~YFK%(<8?(Pbpcj|YZ!gdB?PFL5E_sH z90**&r8g|a4C$5%t4RA_!EgP|3G*wbxiD0L;KA=IEd%OZ*Tg>{ffjO&&!_6IcY z$p1Tmabp^VV4TPYvGj@kXMRA)EO9qdn)Qo!U>*k(ehi0*54nVfs7rt+g;Pj|dsT&6 zScRf?g_7rs^QSs`LwoahjQI#d@vwW#_-60tZVw@iWpx1u2pHq&hn6={RI-SsC6Rjd ze;_w~93qFgW_h=lh$6U4>WF(qNrlFEk;-R`CFy3DM`mz1kPSBuwpiyfJZ|8#-=lyT{Yj>H9$f9VsD zN0Lklm1P(Yed&Jycrk@1faDd8($aCWH(&y`j<)BA`E)puw0Tq+lzm7_me_&hDF2Q? z2}eqal+@R68_1D`S$X;wiI)*d?;wd)`Hjanj&Wo;XJozr2>&ocuURskWhGtEGKFsS$lc*YI!+oiDyjlXPmaiW`oI^m`Ih$h+4syc7T*2 z0{~v@q-XdQc*50{+lV(u`H6%EHBi)n^_g0QmXz&?Aq0t^bJ(A|C4+QHnc?P?@;HY2 zxfxq0GZEPr@K8_lp;oRkV=n@7gZP_hXbh!j5kz;E-DwvH!isq(Lznnc&6%8rwS;jg zmlo+ecG;2bhMjlmYOVs9IcgL+37+9;izhjt<++0L6@{I*4-wEoKY4zkIsZ^wH=)HL zPf8l3FB*l@_mf0)T2d-@IRt_)N+E!Upy8>33+jl5S*6@Co8-u%fhUJ&m@o99Z$>GY zDM*_nR9EZyyMRb{cj;iVZP? zmG&u`PoyHWNrp6-sF|vPqG*nkMQOAudaVMdshM{1c$Gi+40$@LEm4H2xQexzgdk?D zZ7HC9N2qZLT0gp1hnJm@I-c68e!a*IlbVXe$D`&ctO*LFSPG`EN&llg6LnnZ1fQp(B*{y}ToE++^e6f^0N|W#y zDWL{~>)NhycBvX0tS0KKZO5s}da;AYA|Rv&`eJAD_W=Abs^<8jvgvIohoONZwg<}} zBIKXIdA0!Cli(UN4Jwcj_H_Q*v`b63;@XJVbEX-(om4tz7@B;~WCxI=2+cB-_FATP z+Y5?zNiIMn|5B|+s9|@ZgtNq)Y6^=HX{g|OnsixX6SJs*IRBcAX`?OnVgwhbEa-dj z+KTh)ZJBG4D;so!rncNUjR;GVma4Y#nO_V50hz2drZbMSK7P z1V1lz%Lyuk+>J^uHB zg65kf7MdYznlq$~{q}85D!}p@y}*jS1N=JD3rk}RLNgm2fo)bKeLju7=B^D5%q^b+Wu;V&4`Nqw=pUfL8iieol!v$0| zUX*ddx7thvTgR)as&s5Xq8QAhxy#H7u}~|R0!6Ik*>43pw_kj}&pMPoiNp*`ynAND z+Chme^vbBKvYczl9-3AFpq?Qr#AFb;oLt268~@7QYR}HAtM7@p_$!G4Yp$c~zeW>{ z{$T;6lMxKy&x*`p+U&vy?9H{@&1~5!*4dYh{mw(I zVf&eDp1i>Itkm>-sBIdo{49eBy}yha(BB~z0Mfh5#?h`|0Z1E_iL8jR?24np%?ZoG zpnJm4Y|$5R8100?h4@4rJ<=h4*9N55^?QhX{jWuNnAq8az)~V0thaPZr8@ zo70b-4dF}&XJD*ox6&mUtk*|UNBs}+Jpa%1T+ftyx%8ZAe0zzk%ZP@X+IlOxR=iQP zdo&b4&`mw2h^af)*xbk!iJ?kKrkMfQ)sV*X6U~E7i+$+<9FPI7B2UOX z(qr4zBh9OSjnbR3%v;*nPrP;}9H8Vz(NRp*0X2-NeUdmW$g}uFHayvnsar9K%ZKU3 zbSnVTDh8RX$&p*RpN!qW?Zi&)x_*tQ4$aKSxQB`Th(JA$Mq3M85qE8Y0kAOPVcjgP z;KPO7#fPlD*vhmw4YhXbnNcoS58z8mPeycFn=l{o=@c$0;d-1kQUK%`}{9 zhrH`G$d-iB^(ErO4$#!QQQ|fG%I_h}wOvZsK;NeQBqiw{cZQ7?j z49Bx(q^a5q{(%6+zZC7@5upmSJ>hGP0d1ZEEeq!vP~ot!0U7>5Z!Xw~ovA0f*05y3 z-eJPMD&H)qX;u_q$kN&%nF6onA$LKiYINrx=+qL=m%qh;H)u576 z%*DL@<0hEhJX_@LEhET1Uqa60sDj`6ZLR#>-~O%9fe6ZMTIK2;!d%XdQ~B3)`K}Xv z%I{DFwcWCF9_P#c=53Me&Hm<0vjGbbX9{}GBD>ywUdFuct8!-8MT+LcJ#UIW0n$zB zb$srSKF4SH!8v{Ej=dRn4*%IBOp&qn;ZI$g#dvP-j_NMzH2-T>O3suLeeXGIDHaj4 zoP7nmunRd$<-1PLv3SppOyW<>$cad3P8-Ha)pX{Ht{%U)0I&da1m`3_M{SPsCa?0$ zuJX{%G+qsPBWdlktBjybxqt4=cWW{P3M)vY5CrhF^seY0;OIrK?&xmx>AnC<)9&=n-P&@T8Dj?hAy-o766bdSgW{>Rtc*u@QYUcXLg z+w3i?*D;RtnQ!ix-~Y^tuj&6C)t)cfw%X0ae@BK)B^qbEpqfcU2?yrejepVHY%J4_6+PjuW!P^nQk+`pw? zDiZ5JAfY^jMB@>eR4$oK=My>zOs4|t6`R#=wScX+`@Mq4WwIEZR)@W9_xo+u7U*@m z9iG?6_xUWpjm~6{u+Z>u&?<0)u@PhA@iD`(akA2~Qj(z$4v+H_lv8f5?&|Io)ibax zD>W67aLrWC0RKthKp|9*AfeaycLA6on0VM&7#ZTYIJuWVQ8#pSnvIms^!hBDE}F~@ zx;tEaoE)&454`+yz$rKHG&Wu^(X~ERpdw#?Vt=Rz3SdBi0|!!>GJwmrR7d}D#7%1JL*Lt337QsKhSL}#wO=PbOPhuk2=?Dn?_l7og@{8o!; zQv_R^F?ZgaE3@TJZ>ROPd2>YE35$lVt_SUR@wEfA+q)6)Ho5rrwU2;Tv19r32^y4w z9e;k#74`3@>6ts-R2i?c%1T2dJTFj7DYXV!i%k>RCLn+|+v4z!z@Yd#P$X=8`|r9C zxod#L5ukGRXS#Peco!oUF_KZ0u4fBSj;PLvRXc5R*0;i4e9pWs5Q& z7yr!Us4Tzq+^r`s`s7bL{sa)N#Nr?Vpo$ftbCJFw=%&Cawdi*#c|V4 zIVq0`6)$9Rthl@!)r$6{ypq*cAG2?>QUu-dH(G54u}4|#sSd=^G(0dJ7Z_v{gJLa_ z6Egr`t!K~jW+a2<$XT)jMs6ss9ugp}QNoe}lN zaF#5Dq(U^f9kN$X{oRxabIIH`+ghv1X<2;`bCqF+vrN=r%Dy~x;)pHYFJ5X$ z-S=Nmj>_{6lT_QFf=o&#EMdd2A`;qc>3LZvjrQ~_od8{~mR_XvRTG?WefA(+CI3Sq zz^X;jHQL=4r=#w>;0P_0L~Qfrc+8Hiz7td>3=k!wKPbjARaN(KKmjTnKASQ7B%77( zTQ7E7OzTD!u`C~{uDazwL;ePWOXmC=@Wb*UcLHAuH)g$vKAV|ZzV3jz@ePL)4quz7 z`Fq^Zwc0% zQd9*)xjm;2m=f*e3xXByfh+#n_ge$Cz28cGx;^ha`aYTDl#53Ofe3_QIi^=4j~V^N zHG*B9QKvPZaak{}p!1(`^Cc=4kv`gV{=uJ)e51+LJYHuaCk+UD3G7DhZvTU%BnXXb z(Q6Wy6u~U9O~6^~K%VkA=mfR3t$7rbUaSY7fUQiP_v9*YB!^lSf(ZQ|w zoezB<+gK_Jg@Ok7(1)?pSIB~Qt;mt(ArYvb&nf_((e3Ya{)^%i{nrs**lB@aV;!1u zW+=KLP=R4QmDP541PF$SgkK__%ATPB4Q#N3aZJL))&{2M-4S|dOoJXLmPK6IO^lr) z+6$E?jC@#Ndkw%4kbaWB9g2{Tbm-v^f9ONtc+G@$W0D?4q<|&*>?5K3pLF_HN-64( z7*4Swg?xn(QrSj@fs|#Z9B2!<-Ex88!%74$c$l)8O^(4d!UxlsAOB7k6NJ^<&L5GO z!VsM)g1FqnAo)Q`YqD!FsL`ecP~a^0;c!=vG-kvm`MMw)XoA199tx)i4hMWPTup=n z(N0N4Q`+;C2zUd9m=+T){&QP3dXm5P70X#xr;JDVM$1syP{XaUA8-`rMCmwANapd5 zZyTo{Gie5B-tKP1$k82KPyo;^M;VCR=1NK8C;0s(oe~=(^azClo7%L1-Q0_y0F%uB zU=jeHgy#nGAj(u;6N>drYX5-IPa9nl4Ia6gcP^^TgN~z?NZ6W@oN`i8b``AS>Ls}* zT2?GE&!@=Ls9H;yN#U`fYF0IqT)*K>xR_w2E7j@KOawU{TK}Y~c$|g_d}vNecFPae zC{tV6;x#3H5@^T7 zbZ};str^j})?;E6w;2^|E71v5-Wv3y8NzE`QKLxWP_nRIEFx~>N~XgaqF%4F>)aIE z(>&Ht4?_JUQNgiVhkn<8;2q^s$4l8*LY1zirI>a1m|pefA**JbR9E>)U;A3nJqw$y zY%9uE+x|ChRb8fA&-=Vi)r>&GAuhdoh*RS(cefFCZV;h6lEN1DJQPu%gl$r@cMjF7 zlNE1f$xC9&CP0O_qOLSSie4oC#ar!sZz!BFU;EzpivO@Z%zkBSfZ76>!$B7C+nPI5 zjv9H!Hd635Cr7ykV;DuiZEkQVEG^*x(WcYA&KBDXOXg~~euDvPh(%o9LXH^CNlWw6 z;I%-ihKa?2WSi!+wOr56vK<_$@1C97h5(Qm$1aHB1@+5gVfNR*K>lK)ySzHN`S!>v zmU5B@#Zt;0H@fUYazS%Tw;#^z$~PrZ&D0}3BAemEVa~u;%baGVo)|`$MJk%x9L=Am zlT(up?p`#sPDTzYrg5Is0QMY>3gkH&Pr(2LmfdV-KfBq*l6JFM&044MQpe>Y8!)+T zXvX+BGPf0Sr*A!EasSpb=%zG*QVM}?_7!J;D*r8^UW(jvu9%>~a?hBp4d^v9r_}-f zcX?^e?^;7s*ZJ;sIEmO(8p?$L7?^;>DL!$FFMAKnp7yk{$O(xDY(l!A=M+(wKY!DA z(D@Y~naOqwZ#N9w2VrzzM+iwSu~?aq_9sB2x^5+FvtHed2gqWP@TL(Ka)@|tu2#r& zH+Z+&0+;%$2QI}kAC=!lp0cN)$|=2cMismU#42unKMJHN9tjni}kS7$lJ z6+mBqCp%ew`f-T3>BGh7-DN}_^;4g9)&E(4^@4|(m2)-H<#!$V@z4|RRKa`4-vI5h zOB({-A-l8>v-XYmVwxmZ{^0M6UzL%|;LHczEvSDy!yi5hKMrAAY_7POyH~C{kF?E< zZtn5mwQt0Mvae|;XV6=@L|J|_*F)KQ`@_1#`8P3{n{NA7UhL7t4f1%c{CJ`6%+Bsm zEDzA`?23i=7SQ-2aN63A?hudx4iE#M%@?FEd~~b=P7VZ{YWftQ<*=^_60eTVO;-l6 z@y3nzGG@Svr2K@9p!QCZqQp}!Y29)R>q1cV?hgs^&zg=<@NUfK zT5tv%k332)D~3T2F3=bbF!-iP0{^?v0pUw@v?BMikPJN?%>-4H$FaPD@{ zP%4rTX)I9*sc!X}&<&#y{4z#*?(7rqgzpm26d~XN5hVkiZ2>aQ3%$@O_UsjBaoPCo z@Vds24>x57b*sPx&8`_lng!A;|Ga@fbMOU z$cG4-5EHwx2)*(30&RuXqama)j7AXvP$&2F$T5p*W5 zCxbF5`!4ma4HtJ2DRog7NsuXhkv@RY{Dx5~i7_J4@d=;F`+5%ZctK5`5l9rVBdv0o zD(R*~4PTycX*jSbQ&J_{QtFiQDot{#Fa`qAPAKmZFY_`2hk+l{aBKn+Faa_!`D`!+ zQZQYh`X0a_pHlm1#T+q8w-iztMX@C1k}}D0ix|-&L8nHdMhEMr_Rb{KMo&GYifbyQ z&G<<*Ra4~tku6ygHvhe`B{z}(LX!q9Zz|D^{PaLCds8S?K-z$_ScLN~KTzNV?I@LU zAP;jnl@rjIvIL!S383<#=#MHljWRtk3VF*YCUQFgpxEL-XYPU0>MWL46Ft*YJ=2pD zDA5dG(=Fq3C6jQ>NG~pDvL?4f0QKMm?9n&>6F_H?FOkwO1&28elQ0D`K^IglS*swI zC)^s+CV`ADv$JVp>IDNahrSc)_JU?Sue1&&9M+RWOO!S52PIGMCFN89T9m1f&^EvG zK5wH0YV-t7U^p=VICoS>dz43i)JKC9NQbmXbJR$S6iJCwNrRI~k96&fQ!SG-DW~*6 zo3l!bvKL<^LjPyfLNnADwUa-*)1xF(oW#TL#_~h%Q!LxSMB8*d`@l7+lttl_8&xzM z-xBt+hfA+&1p3rZYt#ZZjySt;NdweS_tHv3tUwQ^FdNlT6%YL1;Yi{Z{;F6yf``9(%!^fqnJO3m7VHB% zf%RKqa7{(QJU5HLP_^=AFiukuT;HlxT@_vf^Bv!FPGhxRX%kO%aaX6QSADfz#b{V< zG+=EMNB@&_SqJq{4|PiWQEXuH4JY+sBUNG_bz;@>VFF-W%3wVj=c$T~=`s zmqDEZQ`7cet1?{Q0thk3ax;hhG=(D^CrC_+6VFjH3kpQPbZXhwUf~vXXA^3}kzf1O zXa9dSZ{?+LXLoP?mIY3LX>-?UcQg{~utwXHL zaZ>Dj;=mSRcT+RBH0dvRr}lJdHFdw2PGj*S*zu`i_jYr1Np&}Ap_NIGP-=yDdxuwsK__cLID{wn z%@CL(9f^luN{9OpYKVE`Zrs99$raEBT7G_;v$TSy{Lz_t9-% zIEL+4lxLU?vsRMZIOo8Q%d%3Fdf1gi&&AGTk5S>DgzF}Gq>nY%Q=CPKofwf9xh-E5 zm~mAmFY}boA01JnZMw?ON6TlKe$tC^bD_?U6?@peR8veF(5*spMz=*F3K z#I!f|c#h%fLN`fFz3C{l0V{>XCp`C;6SX)M6$qefG^PXAlKt})Q4x%RcbWl`n$Os$1rnQIIz}tHU(*bgHO zfPKeqq-BD!7d_w@q-$myQ<|mqS*sWMp7qa|g}OQhTC8V!tjpSqy*Qx>G}{`Qt+h3O zvsI$O8aL5Zs*hSH!g*KFHY^s{f9)tlqb?KMjb2gOd*fE6W3{Cd+ok2&YJ}CX%lfe) zTe2gYvi(*cmGXlA5~8cLajn_2`PW++`=d6tx_TI#XL(Eyii@OX*Z(3J8L8J+YjvAU zPO*o%t8F`Gz@c*Rv`1r-Pch5l#S$I(<&tfPWZt zwKBeOBgY1ArLw!Qar?HndcaLgO>@(=J;t{ee7qanPyLN>3+K2ee7)N+uJ`-E0?MP4 z8n4F|!`s!Q?~l8?d&Ix@R@>5iuhX^B*TEV5yk8u~DcdL`yu#hOvtPEoT>!p6{Ij_E zn|;{6r}x0AI4%p|s%tx|1w6M2e922mt|PNTcfrM5e9EibvHuHHVJKY7i<=;mo5!s) zJw_GCk=iW2Ja(;{#M9iDPrSt6(#(T4#^YS3<($sz+`Pw%QMr7Edzz;ccgNp+>oj`9 zTKUI={6mRcRe9XMONYb@{9YY>K8uiJI1|vrfyyu4%IVw#q10rz+|wQU!Ye({W0{ue zo4z-^kVXBk-L~CSs?D_<$zR=fHCVeTeQ5So)jz?`F&&_Hy|TwDA0GSDYy3(<+aPkC zWTV`o&)GOM=DA(LGLikW(oCgXJ=U2V+pm4e+45QLR@ym%*E2ob@7x2)8@%)!-P=0U zzdeuPjJcQEkas3!5#4;H85m^N+SR<)_Z{o5U5eZA6aRXa-SxoN3*O)lp1kpV7|?sO zD?DP^J0Smj;MWTs={?aWo*F+L9vUffgu;5VHC>YQ5B z9pph;$b=Ka~xR~wMO~sz*TeGVkoI}T5<=39vbqwh{9cG(;lxP0t zyY#~&`JFM`>Crsv&;9H9J<0t(=HE2vf9z=-LE&?6cb#4GditSm)q_)JT9Np-|4!WJQl3` z{`5$o;2>e4;b944qT(WBK|*X>nm)m z>@01q?Jce>UoNk$K*8^>a4!I!@iB58Z2uYA6~yy1X{q#dwRM(OHrF*bwP`mNIQF=> z*!THZm|bmhyZbwy3$uK$r~N&CzW)AYWq&1=eFF8BBBKBux&f+I5o_r1VZ&i(B=(^Q zv0^QY8ZTZ92F#)>0>$8wEQtps8*@~ku?$yE8%%OBXUcqOlP1oZF1sN#!_FkopmgLB ztvB!}(xgcLF}0-hDW{%1$Dmrs@haA=6D0=I>J{bJtX*RXEo-*SvQbrvsJ!V`C){mu zz0qyEbFSRHIqUNMCX&|Kz=4fIEo|5@(@gy!9ByoA>O4^jt%j{^Sy$K0t-$)?x)n3$ z%dtXhZbeiuYLc%a>utT)HC$)0@&CSs{aQ9|+PeE9_$wqqszg(-Ua_?G?Al?G9vS>Qven$A@~*wxwt46X%iq4so;*6=%fQn=ma94b{GTR7 z;P1abfQ;#vpEY_YFxhn5DHv91-F37eglT+4S{|p>ci}7PX}DW^9d3x1hq;BQo;7I6 zx5kAj(t!(c1G;D&0|3hS9|JGiSeJARMYto6J^F|XcC4s4q#K*T_a2fTqL(C-c73Dd zh)s@HU=tOZ=pB(+qGgMXT|PIX17Hg1<%^^ssN;`mg65-xYVsn?SRrXSWFM?uA?1>J zM#-U`P|hYIpL_CVrG<0WXa7r!WGZ@8j5LC|D4D%26o8wRT6*brZDMJtibM^uQcFkr z87Q8mlIWzVf+|U+D1>rKCtxl>I%_@&fGH-eHWDx?roH<5D|er97Nn4`3RS3uqA*|q zG|Vy!ihHE?sVcRj{weLW8v2E79NZdv5?P)mkSn>)jnL?={*~+Eu9N}$F1(g%x`w9T zy2GtH=z!`hOQ>2~Ex@Oy`Y*Nv;~9#p-rB2-AAfZl*}4%65hlgBMvPy(3C8=NM;`&R zG04J-3}l-`;!CKgqU&gY zMK_l*njv$QS%XhQJ^!!MY;H&2&^@O-Wf&{BBs17JTgxif1dAOu+VCNqwbpDWoTAT4 zD{5=Sb@x}J+{H0X>AM|w<*`RoPc&iGg^w(jcy717_||0mI}X1rM|1YgFjt=TwV6|H zd0vituC`Q+zpZ$(^WO05k$1cd*8fj`&D)mknCHRJS&4v!Nu7)aXu_i)aDgs#Q=F)#o!t@Pd=k`P{o_rG+niVuWPaBM#=Tid4K}5G}aFRbg?9uc=@c zt@yzZj?sD0Bcl-!z(jq>$ZpqD;TrSdL@3g0D^jE*D+G7P1>vxd?*pO_|Cq)24bqE- z6k{3@sYv+5(SQ|DVkH_WkU8S9j(e=+?5dW?P5QBuTI^#OKN+Y{dNN69JS7OrXvQ&0 zWtHZTq;nv-fHfLW5tPIvE+2A9HFu%1Lr zq$|x4hGo{$Fp~%;I0Fhzff6*EkGSP4q$VsqnzNz6f@iek`A$JnbfOits6=CS%#E%S z9EvJvK?(8Cehw6*COs)YQ;Jf8y5LNQ(<4kXLpWkNw5I9gXh$JB&y4D{rycaEI&pf> zRIc)j&f6!MqM1#8uC$~g9qBgPrH}yt&Tm$&Dps#bQ=4*?fjb3iFb#>(u=13wW`$N+ zhu6`3w#7v%C8<;6+EjuH;;nPFt4dosRjOt+a59bBOx1wSt`Zh2M+GZlznWIXCiaVr zUH>d7gKCQCF;1V5lIv69s@Y9o_O6~Sfm9U|SiubrCB)ilVN+`zF`RX*kiF;+U7N+& zMpUe=y=`Num>;};wzs_ft#5;?*QsixxTYPfYLlB4W<>V2xy7w>qx;;)PFJ!*Rn~4( znpvi9x3ky`Emi47IKc{*wBJ)MdaJ`)>9QBPvTd(z-z#7ERu`%M3@&h$8eH$n6~6%; zZqs0A++n~8x#>OdAI8gu`ZAcl4sI|H*$LrMN%*?&!B20y%es>07rf#ft<(tY*GwX~ z#GUzAo!qP9`O=of4~FfFT`c1cI>x{GwK0xw3{U_MU}Op{uZU4=VzDq3j|#rSg#VLV z;Rq|a$xDtgg@JnA8do#J-1RPas~R{vhFBO{Ch?F-rqwS)5g33IER<)g<}|Z;&2Db< zlSgsi9Q&8QP}MJ;zX@OioHvs(95I-~44N^=)W9$#^F*AiXgDvr(Tsj{b8}h{)6IFm zb=_=s>73=Mjdr1&2CSh|JL0}-wa5wVm~+Vw-LGahyc-RbTw5B)dERrk75rn52o~JoeyO<24gc^mn;_k2 zU@o-dt?-1qJK>*`^0cQ>7+hmo!xgu4Q#U=+GjrL?zvi;P{~hqZk~_%8j$JSeeg*$b<1PE@R=|ADfGTM#XXB}eS`aHCl|RTUn=m;(D4@q7sg>3PV=QJz3J4oaD^jo zia=NsDcZI6#WOvEw`Lj0pdN67&5g-klN`{)D!S28EC7UK&gE{GIoz2pcc%|5>v+bw z#q*AHY4SMi1?IS?gAR1TkKEM3dZD+aJ@>`ao$-#B`ok6f zWe@p~yWR1=W@LdRPc|XL8h4n#z4fbqJnYkE^UCiX=Y0ou**`yOA^)4$r)?o$B)1Kv7bHd z(_edC@3qjtAH3i9y>HbP-23!8KK_fZ|KzW``TY0)q>T&Z)!RUDSe+ph{B^|k4IBM6 zfc>c&`8n8~z?k_}VE$d8FrnZ2-JR{ZR03Lu{2f@qIiN=T9|oS_{;8D-LR{1_p9><= z^AVu*VHyH{AdwXw?=_Us85{&!pbr|~54v9Ku^9k991)0t2CCl%W*#}&Aou{B*u755 z^&ARTVG3Fy5w4&X7U4R@pbXL#)r!r_Vm zA||4h!*Nl0(4id)7als|6P}?0p#<9nVkV|yl=vYkM%y78;wvTMC#KY{Ac!dHVX+-p zm#v&7#upp*A~33=FiMszDq?Q|7~kAtb&w+B9ULnTBQ&-mPd%e6!s07RS~1SzG&N$S zoL!kgVmZy07j~0=*1&1f+SGR&*Irr)7h8j$5)9;RXP zrD7VT)XfrHP2YaSrC>IrcGTa~AZAtKWoBBXW_G4pZb4Gs7+_8&X`Vp=9NqnW=3=5{ zVq#`luBHdEC1cJetN7(bx=U#a<^#fsYxX83djF<-sV0=QCA8QkR|;fq&gE&Apl>#3 zYd$A!MyG79)ky|qfFma z5@&eEr-R8SYtrX^PN#J4r!49fa;j$;Zs(eSg!7=&a^@mEQlTIhv-8;=kutDMu}te1m^=tIt^ zjizFLs*whcriv1ofFcT8`NWE%o`{y{hZd6NpL)fgo&#Za1Z$xWOZ-lz9_p+XDy>FnqLu)@q!}>9RhnvVN+azM?(wYOjXfuL`KJR+bltMHBc=thS7y zj;mCX>pMYfebQ+sSnIW3hXbY>us#H8EY}ghs;ow9r)D9(N-Mfft3HM*!1Al8Mw~6> zX}dbA8jRr4#p`gGgu-a(Ie7uHPXFw!GHY#BEXMllD?-5`uxqbcL;<|(l{Rc}a7+!U zkNV`R#j0$-w(P#LtdqK|%nEF5I-ekTY@_lUcZw{-0x5VXU;)_cy{@dS7H!5JEz#=F z7~HJE9;~k_*lD7wxv7Y&E`ZQZ4$WpQ%(kG`CT+}WEUpHu*p?^>HLcTPYpW{kqe29c zY$VHFEn9dk($1~izHHZy&jFb2*?t(uMJ)*$+MMVT0>G`^&TO;Rt+N*H*M{xgHZFC)n2Xbf-drkZh7YJI+gB!_U?ztRMhTf z<}M7PxUTUsui=g^_u9nqb}#sHt?cTq_L2?nLT~(K74_E3!mP-aI3xCU?)NJ1=hCnJ zhHp-tFZyoYmvP&Cv~Mzq%gG9@5R~us9C}J(wgi2elY%ut@D~G`gY#;^eF%X&FFl}`VjB~BkdWbik9Ka_%1LE-|!Xxs}*B07gKTb zEin_D;pjav)xK2 za^R6zaH!In8s!_W@*{_?CM&@xlky=8?d1^KvrhBOnL!BnOEpcFHB6aW7YM99uIs@A5Bib0`n8-WIYQhck+x2I<_cGN1D@ zqq8T6vNKC?vAy#hm+>@nb2jU;J!|tl*E2nDayvt>?hUh9l(R3Zb2=CF5f@@V`*Z)! z^2(938J`k0@BcGCOSC@cGeuu?Hxp4ppDyL*-r)#M6dyBV5;RF0^zu%$5`VKh^J)KP zQb5mhL{~IQXEa97v_;GGI%o1m@9p&NrbiR=y*#u7mvkc&wMpM}5$!ZjtF99x^05eY zD&urbOEpz%@l;>+z&@@@OR(Se?G8KjG0ZboA2nGU^%lqQA}jUT*53m*HBgrXOpoYUbkyc_Z;9kV^9lrROhu_U$bOa z_GDl7J}36avNf4q6<9-c!dSLV8#Za<^<9hbRzq)3<6dKLiVDZIXk)f%+eKCEnkv}# zt|jqk%l~$4@AH{NZWu2%h_jB)>0hBgb->qh!=?<>1 z9UoZ$A9rLYcW*mxZSPfcL$`Au0D3EcbMIA_hPQizcUD6lYg~7guC_w1ty7EjR4X^D zHTSMfRe7hkfGdCl5cqlzK!;NIVe58A(|3=Wov%^EYwP!W&-HARO)B6vZr}C*oHv2@ z2YP2XeA6|9SNMpRDG8Tw2uiPU3$%rwHGk{WcxyO^*SJDBH-Ln=LM&@h&u`-Hb{aYO z$8JWD-wFSdH^KdqJ#RfFZ=n5gtiyMavhwv^MP+{ySQ`vxR*NsXNusZX<-`rdL;Y3 zR(qvm4z5Q#uJ8J&4?w*8daa8OVyv|Vk#lO6LWV{8B zgo}rQ$6sp1=RET*{FpPm$s@SV2R+MM*l!OF%)|V-?Sj8?yjtA+$E$qu;yKClyvg^v zzBBWMgO1A|J=ezpq$X6c8+&9IJgpDC(C4qumwei*{mGlX+gJ5aY5k;hJ>A1Wi*g=! zix1fk@xI$SOj3Q%GyK$#{MBdu*-ORN*L~eb(N6>Q*cb23AHL+5xDrr3$=iFqKm6No z{^Vo$+$%olqr3H`X5%D!_8LCCW1*uDe&*+T>!*JGE;#C=J%WIK=x0ir(*M+vQ#<7E zJr#65=WF!Zvpw-kJE{Xe@&|u9B);t*z34BrgqObWgNn8HeL+k8&ttxwPdDt-X!46H z^Edy?i+vMay(M~C2NzxFQ#@2|i7r+WB{zr}w&|8CXsH+>od0Yo??x)4TW zBDp*^nNOut3Y}7`R_rud-FBs3a5Fp>m&s>y8okc8*>Cl6Fb~x2^m`tk*Z24S0XV_9 z!9v1A!$ibI#YVS^n0>}&08tWCZ@KmR`Q@V|2IZS-^X zboO=kcKCPsc=~xtZ#!1~eg1y`|6sNR4kUOhSAkl=C>>MQkRdU9^(025SP9Zaiy5(% zBR8%iN04<4Xe5czWJ#1KRi#w%($TRT3QP5Kms4R|i5269R%TB@xDP*HC3;`j`j|w}4(87p36cI!+Is=fzCkjMx z#T6HH5f;xrbI?I%PK@wI8*{XR0Q`FNF{C?m6cR@wb5uycA%#?tMJC64@x=ybL@vo9 zt36#XCl~xo#`dHfVoW&W6q7z4UDGl;>*UmP&OJ** zlFy3%ICIe7(nJr*Hr*@2OGdwJ^igWY$@0#8Ahpy{O#d&{bj~sf^)x3#&-08uH{a{? zR8#?#4Z6a9BjpcG+N?m9x`a zqfKj9w)XSYRAHUf7T8l+o%Gw-wiP#9a>tDZT4~c|h1#{4NK`IGiz$~`dNT#j*l$Iu zci(sH<@evF&{cQfQ`qfky=zCC*W860P9op?U}Jb$4n1pr;ee^e&ant8(O9 zpAGid)1`LJ(4$v=`rb(^x$fW>-@SV3yXXGuJDT?mciL|^zx+Ns5ifo7+CWyyfr8(;$iSTRBI4}BA)UM|d~>1=KZQj&YEh22ApjQ{Cq_IDk&JrGV;}dpKs2s#jv_126r+g2Lq;)? zi-e;U9oa}o4pNOJ%z_;kw?{u_5|f+MWGC-*#z0O|l!8n{8{LQ#Lo)J_8q{GcNhwJq z0Md^@q-7Cn8AK!A5|^IjB`1+(z*hz{Sf)G+6cwpTNUk!O%Y>yc2T{i}_EMU@e2vKv zh@WX@lbWuvV>CwT%EnR6%UwTuSB9WaBbOZAcDNvw>bfiKZYDstM zx);t=sTS?!JUznHrA8I18S17+IoxrRIlZoR z#kfn;hD5cuJ;#0`DAdpSA>-S!4Aa{IQ zmFs#R>sq#m z7rh3yD>@O}1^}ZYto@}ie{C!h7UuZ6p1pC813O;m1~|nzVX=A-s?Gx^Sjle7ZHGVX zPS=KECb**h(=UMky)T=ACsR8|K z1RwdugvL=r9g68&Us~BD2J@S-8EKXRcFGh^cD1d2ZERop(U1o5m3Pg%F{4`C<0kf= z5&dTURy)^w#Hog3{cdJwTi)}w_PTBSZE^J)*!%YNsmWdLcQo|T6Qy^-4W8|wX}Z%Z zl{C8#{&0v#T>s(|e|Y31ORmX`ir@d{xT-~W?u1iZmtqz9v>EMgco(+hD`)w{?R`;v z6G`9w-nhR7eRF_sg>nE-K(N0kRdmaJ{&P?_TFZ8u@Un|eZJ;Mz=}TuiYQ+)rF$t>U z|HgUF4TtCrH~r~I*ZR<3{c>ruoakLAJGQ}o&*F}0<~3(K&8JQULnHLEWv6@H?LKx+ z8{Oe}zkA>P{`az5+*YunddI8&_9`ik?>%2U;2r;Xub;i~kf(g*yUumY<2VpqS-acY zj^x5y{Rv2zyX8%PddOQB^{Z!n>krPrG~Y_}v-kL%I??J)Pu}&r=RMH5vzSNF4Daq|~U`|Uq}K$f2ParQplso($L3#?@8qwQO{{{ukFJHW`}wmHZ@ z2h=|V96rU1KnkQldAdB9sz43gKn^TBp;NbSIG_&Pzv9!QPg^+QLq0tEt)n}+#=Afh z96)GGIuty;%*(sds)`^003rDV1yBGm04x9i0000400IC8|Gq@*Uzu9TgJuS%|lYyah z^PTke_4xPl`TUWLnEMDRs^qB6_q->S=3#eNIM+ZEBD~4xVo^IEA;f4yW z-z`6Xs+HRmu2Hza1RI_^c&}e&fq|w?%tf%=#kVqb*vz@J=g*)+iylq7wCU5RJrDB0 zdIM_@8&b<2?V7{u+pRsA&VbN?@8330v1ABsWS8M3OCCp_64~$P(4+Sz6P@~REg3;- z_sD&__wL|9lKD7Zxk+y(ldC*FzP6gekVht|12~oJ~`wER90yvl~QK8 z1UsS{wZjn zzj;eZV?$Z3P0E@gubQ!Wyfrv)1~djyd9L>#kK{dgBH0`AB4x#13n0vB(~)Y_iNQ>uj?~w$LS)(LQS} zu|Z}#ZI{mSM&_a5x)<)a{t|8BbMwmYdPdFmOjyz~+}=)Lv=YVN+}E{ZO` z0OuPfr2(}1;;p;VsvoBdD(G;-sE!)(#H4j5TU-fd(j( zY@>d%j**f6y$>(rzoX&&q;{C!&T<$}iI5_UrVbj;SxtY#fUB1m;a@o0;E|50nx>>fox+Of`E>l$7D;KRWDwJU+UI~wnL*SrcEMO7GNTdCqEF#;hk zRjRU}2uFCar1VXM+pAvm40a+D8K6uN5TD~_*ghKCP-Z$?pXWYDL!t#ueU+nDvM|9W z{gI1*Cn*kdYLYty9`JtxG@uj*I5zw-O;J~@pV-RwEiLZP4SFM?tTZ@0<0TJO3i_a_ zEGEV_x{*V@svGx0HY2Qf|8I^bq@%7h0y7@g@Q*h%-^@5iB^|OakeB;m=X97XKorq% z`SZyBmPiZmAaNR{@ZS?RIkhHIv4ETeB>>g6z*08QicTEZBv(nWO1;r;8JwH((%3dr z&GLhDOTjC5$xG=qZ-si~9x%fQz3;tE#mSfHPwzW)}UCv5|++ndg z;4Lq*l9DS)R5DI^$<(!Jl%a%^CqFsGcIFE>@HC|V4rE>ZN(wF^h@+qwQ=6v%N>mVH5U5DiXHu2Q z)PxQbco0oWCc{@vidxl~&cxau5y>lT;(43xd^Q+t)4pB6h_u>#~hQ_DzMqldbhJS46SW! zQ%G)g)ma@7|7}=z1}(C3hqb;*3jSyt9D1=;wr;ga6L%X^LG|~)0Os$1ef!(6fs#kV z-AHkv%c|x&n7Kn;8>raG;0jxojtt!pb(_kR?BOcIdpj6*rQ*Qvni#y}1M5~l+P)T) zR(;r%8E=qzwHIlib_teOE!vvkOzQT>0tT{>)yX3Q8~LUOJhGW|V&PUj_{mVtpb48y zWm*Zgppx|PnfX;6;qWIDP%hk|9PBefeSYQIvGp2d1Un;g-W$FR$&Me~81(qw_N_*N@ z{h1!e|2C{651X3OzI-UkQ0A`&cGt6Gj&-v#y4A{QRIH0*O`AQM=Ayax%RGweR7{5H zjU)xrg=_VEOp4#_(D%_pPI9yrO>IUi`P%2?ZKK;`20DKFVykn&x5xe1PD8bYSvGfy zs=MkJ=I_c+^$c%E%vH8%-sAAbWD9R62nE<9){%6?#rf;GJM)P z1un#^4Ml8Q{5=QW_I0{V?zrT^Eg+AAeaHZ(5GSKL_dvuQ|+>qHvM^nd%PT@tzQkXg+AY z>n;Vm>KP?=Yah8T1!MVfU*Ze{zzG}fH~?YZ4)+VgQT88>`IOBabJ<^o*sNBzMB%B- zzB|9OKcpN7G%#LmI*y~#gdgOO#Z5ogw#aw3yHvs-_3_ha>I+wR`p*jO)61##sONgu zR}WtV!;Wf2zPRqIfO6X7VN3?Wc6h;i1o(TX27JOs72alP!^eO&W-?mjePH%{{}lL4 z(MM~mMqWbKIEkb={3d;or+7!_T6dy(PKRtJSZWj?K=WlapeJaCBXJaGdhzFeuCscL zCU#!8da#Fb;_wc(_a*{(1qJwAB342s*MOT>e8RMH3UxBc2W(_uHGmdFJO^vf$2l3u z0p_(h*YrNww0(>BYvBSYO80ccc6=!~Z008>oR>vCCWAI8e{@)f6IXxv*JpTmN}cp| zdbn&q*neyngl;l~L|BB!6?f+*B1@=%47h+Z*I`iz9^BS>WDtTCNIr{)Z=2YO7sxZn zM|~b>eTuhuqnLgDrW=yi0^PS~gQixQw}u3la0gIFPJvP_@rvJse(+a%{}5Dzz$jNZ zXmS0eaZ|`~4G4+KXhA=y4nZh7hNuUK=vYj6fS8huEw@m!h>h3?YTnpA!)J4A5`oX5 zbD4OB*P=ctV>0!2TAXNakEe=~*NWu=ewbH&X^4*xsE>{4TEiuW!Dxp9IfuceQh7*r zWY>oX$!KO*FboA^DljUqRVjTJ|77m3x_hz(_p!Nf19)F$SLj_2r!TD5A( zCt7KCUbmqELQ{B{IC&FLiMmLI4&Y%(QGT&_f;wqkN^_GK*F~`fkV8g;1eugOScR%* zU~+hnztw|X29cQ7M$LE*a-w?{xqBGNOCzR;%BYPGnQpddM;z&d|G=ap<4BYO6f-9I zmXAYk)VGDVwmH_4UbdD)TsV_KLU}tSPP0I8>xrdf%kWAT&ONp7Tw-A8{mJi@`Ajv8dIS1@=m1*}=U73-f>40TfT~1Yek=TvP zh+!*Oh7t&9Zh4ZN*qffXimT{z^){Cpm3_duig>AkDG8H%2OFDF0j`Kvp(%@qNu4m~ zd6^Z1^b;(;7$#5@kV@$&!}wbgS36)=7Xx52s73+ECPt`P4p~VDr74Z8xlgURZab-s zs>X<4iARA$cMpXXdNq{HhAt%um$KH9pwlyjM|}i`T24IN}Jz^q^?O7ELDZLNRqtycbo-dK189nFXr8il2HYlM(UAYi9G;nJpxLOPHGGPqH_r9e5n{jVfv*T*enf-iWvo=qj~|A z*B#*^iG``AAS!-oh=NNOZlU*Z;F&IvDG#@btGSA+|Gc^`k-4Jcd6_f{qj4In!&-KM z=zB_vH+Ary6!|93dJd|2sMKnFN!pQQ>2k{ILVQ*xujm`GahzN_uD)4{UY_aoT5lMS^AQ9hN}K(3TJAas~VzXsb~M24y4u%)Mi zQk%K9)uNf1r-arhSvya_K(Qx#A0ewD(TcH4z^e+dm0WqYAIqrysf7GFmarPBSr$<% z8))Ck2fJBj+~gy5`?6l8g|?xRhG$4z>ZQb1Z>!YdyuGL0O1&Sq-!^-y9TW5qpu6Q)!Md=8fx)tOx37D zwt1UNOM=`ruC0hJE{m=TT(0A)IXA14|IPciZbl`6JFkF1M3MV4xj0gf8;f&?tw5;@ zszWZlSi`gk@t@Z?n){Ku*D$DIk(&5Rt=f8|3HZNH z$fjP)ZbB@uRA?rrT9kU{iNR|x2~51tVrJO1h4;!Do(gC33MPP=yGv@c zA1NbldxVY5jnkW5Sfp@F1g^;GNQ8oQQ{0I#+ldqGz*)M*S^CL7G_ML^!~DB>OGCN* zYQ|-}wu@S7X}kq$bmB3v&XA}T6|{gvv(Pg z!Lm5XW_mI&x29?O%GJ3ejQcdTij;30&~QA^acqZqT*r9=K?hwZ?F)sW_`ZpXqs;7B zNE=cdLKF`n%^IDJLCmL0;kAwY$O6o4$izqeoF>BCO?RA!eQ>u6yu1~hvlD7-4$1+C z+n_fss)!fQseHz!wxPUG!yUYNWSnKjG|SJ%XLV)5TeHh5%*HKjWaZhg*kJ&nXw3a< zX?|K^Q^3Rdw9#OV!!Adb{~{gBMykz{3ccM-rhn)sGO7TZP|nWBP33x}H5cJc~#?*Fo$4*_{UGvM7eSb@xHML-vd_2RllB2U% z1oevqq;1+^?TBUl#b}MK{yVa`Tiorz%K1}CNt-*CtQA@%rE5+5ryS&RSg44(*^3 ztk)kv~W$J*RI=LSJu74@6_cYZneLYaVzaxgS@Qv21eRl);Extht{==PaeS`Zw2>snmfY4Rk6yQgWyxG^Y z7toH;^yFuX*tzS8;o_0QV=2v)gE5Za*E6BMB-pBf2M#1;9cpA ztjuRA-be)w|AcrDPY~zny`D*|+TIw%%FWSAp_ud?Htt45FhqI^1b~Kq=s*t7(0-gw z{-6+Cm+i`+&b!Tn&F$69suTXocK)~ub>sGJlqe(SsvfoKzV2gA=FC3X1iNalc_Qrz zBDSmMf2ss+?gtZa2E2~Q!oJ>0(XN^|Oqfn{NBr#?s$C7_W1-9H%KjU1ZRkI)iHXj@ zp;d6nTb!ee8{rO{V2s3^j)wQx*ZVBbl03qhusXY1>a8wqvJ>+(zpz9G3Q!R1W|`#t zx$rd1OFI0Fx(@IGkH4V#D}7w=#18E7%^&98ttB03kL1`2iM0If<1(G~8K3o%$G6Q} z#{<^%ig?OyjY5AQaAgZr1BI)9JG9PVfC z+03{E{k{qRPV_}Tjg#f`Wlt4J&-9H?@K0z-O`4ep!GbZmb^bo_;QZ`c&&lKZ`3gL$ z*$(y|U$3C}pKK4p5Z?Ox-0f(d=PN(LqTao6pL%vB^SVXddEfi;J_Vh9&q@B<4()`E z^>Md{_=&%oJ+A>AR_8z8^aBqfD76g5@!HC+N`2;E3FICD@dp>5v%zZ+;XmY{oH9L= z#UOvLL~HM;9N(Qj2}p_Y-y#44+vhc>RKH(Iv2ySrbyiIyiixWUiOy+1tG*Y7;wH8|ks<>S{l z>gM)e^=Er?1&lR^jVyxv3?>A~OCdsm|0p;pHnG}-YS1Q7&=4(5M~)voh6FiMWJ!}G zPo~ULsSG0nYa9-ulZU3BJ*9Bw6vPPABLhS#^%OclXHla^ktS6tDi>2;fSSte{}gps zRa6fY0Mjar>zQI@m|aYUBpS-;J=E2*S zFD4&746P;}SeUUKS%?`HR_w~yFk)rPP&5|7!JH{)KZiysnlxzBrKxxst#UPMkuA~O zDY9xN&Ds_2%qpdOsiL8dB(1gd`%-V?$B`$uwRGWR$j_lSrV14@?}u@SS@zns*!Hg# zoq^}6p%oo?zr13JFX-&iwVrg!YD)~!m{R? z_tR9#*wm|_h5C4tKq8eitU(6xv~$7e;Cx`pi44e0v+v^Q0=3h& zJTxUmrw}pGM^8$oG6U)eGd(#rO0XsuIcr6O}g zNXPQ5)6RzU^g}v6-!SGo5&;cX*kO;+lA86bWMUU9UPtGp*WPykW4Cp=lc(Gb1T zf%f9+paQd~tyV8<^Ra2B3%Ko;o&m?zxRzKaI~PHAc@3IeLO>SQ4Mw2`jiZ*SPR;77 zvBp>HEn!Q;nI&dvvBeXOi3317S($?t9~ahj`eM`DoWWHW?^)C3Tj_;F-*s2P12=$= zdbe(!Bn??l)OBjIYJ}i^XIqTl!z+|A)HFw}vF&N#ewgm#|B-j~VxpUvx!9E>jap@9SrSw7$tyQ=XBF412M^DkAKEIlN#8hranTj}Xk1|A zL&9OrhMua$Zg#ALT>%jojh5JMd{Fq*V(P#p>_tEpFTxnn=l~Oo8D)44b6D~~7{a40 zuVRr2oj}}hD|3-3stO%;j{p zH*hfsgQ<|y3O^S#qeYK?c5{REQdgolC6It!OrWE_xP{6z;$JE=V!>ke zXlO}r0uG1dp%KTDiqAMw<}RX(PEdk|$Gv#7;B;F6%S z43R-uH^u?*vYh>b$Qp@3%<+WcAi-0PsjLwu9Nj2x()8#@(MGXp`mtkC*&n;;_dD(t z@;l%}mJGjHvNJ*wcN`L*Sj<>~cG{&K__-OH|I)NRP7cw2c%xA^ONmWx_EU=coF1`W z2{nWkl&Vwph_b#QDX(w}QTDu2+04e(WFFI^$-$`G%z&fffRwFm#V2)6NsCm@b$(Es z;}i2I&cC^|hA{N&OcN(Y1*B7ISm+@Yb}AS9qz@i>$xRY@=00tr)Tt*`;tG=nKg_x? zjofTySgjgb(Sm3iqm{x00bA1v+_i$jY$;$Vs*yp~kw;e0C?4IKTix1fq)MG;K97p7 zajx{QZGmZKh|9ZBZX{aQGISz zYnfj3npUb>u$}=_OEo)45RqSvXdF*U|6Aga&zT?G?SDo1*6t#eyT4r*jWl(U?>$2; z(0$}yQwvM>$ZM8;#fYWeBHb30gTouI@O?CDqsnUmX5vD#2A|ZH1FoOmK8n0CsW+1>9Q`8~JzOr7J^c zxZtvI@1+bru9waEvebx2z0bAqV9UpuHCeZE>}a!_?|~2&YmA%wgx)>ndY}Y_7sil5 zE*4|74+*0N$A!-E4jaYXNu5`csLc|0?mMK5?iZ~~76wOE_he1;ajbFO?$YjT|Aa}7^t@ShLENU?{4>Q&J+NK(?CYdn7O&VO9ZQ>3 z(L*m=mMST2XdQjV`2M&ELFV$6e&G^rXIf%N){&ASp_L|Yn%v2Y@_?O+?iepMGG`9w zsmt@=<2Ef>7750+S;pu}Qm{pMpy%fDYKh_vF}!ZAvY#tg1^n=C{fCta!ylosoNE#?n>C=QCozs{c3Oo-tAJ6 zX*u3cD%lp?=DKmLh_ytw2e)qRQ~t21;lT|xic1}Ajzv8obM`I&kX;5f8Jg=Bx9R~^ z6YXh_-L);nc9a+)a^ei(|Fr&9`P*4_?k4v6;C0v78&f^xYyY)+JNM?F#SH6bUlr`G zP6l?>r&l*`gD~3bUH3dJZOMf9a@=KNA5WC-t@fdm zTvc#){mPSxPyNAM*tMVXurJUBWDMT8s>V{5>HYUt2OS$h$Jct0S#(i(vZo>*Dq;S4 zV#?OOik5Y8sR`VMrqjI*S^t}jJ0E`8nRen=Gkrr*W9>;M1LWIQy-?iS`q$6@N0^s# z<&Ay&h410CGi?@IjD^*-=iaKs%Z879cu#>|hZ(L<`4X>Z+HC1cEYB{+#Bh(ZF0j|^ zZUe0X@D%B3LhtiL|4{r$Z#^g?1Ve+3mh1FDjw>c+7EJo(ie&PKvfs%NH+iEZmjYIAn4%BS#3Kt^q ztj7wtXBq+v8qmzI6bT7g?GV{wYbu2h#ZW-xX_J75bL#1co)2gUZt76UieygVsA6=e z%fzWD(|8UlXk3vAExZv%Ujwolciy*3zb0{%e+QimwCeETy2RG~G?#H_Xq!=yB z;I{CJybl`1@Sw=B+2rvIK}1s25dEa>4V}Oq>#OXCZHetI97PS@Fh75Ee`D75nkg;!xa$A zA^(unW=~h_?c0jM2zJj>tbyjtN#l(02m$5q!m=aTP9;Y2a;g!Du5n--@Em`}%|Of@ z@B#V||BM_#u?t1NVU3GFbE^WuJT(WngtXHkM$a1KEz1}p8EEb}@6=@+3A zg_34By>lx0;qNhn_vkE%Kt#DTJNs}uH**64F;^04 zJPWZrnGxp{tt{;k8ei?yY*b7{BoE+IJ}XMISc?tpfw4A+>5LRW{W4I+vEhI-sLF9~ zl&J8=Z&4pKK@k)}Ayq5#5&zVUJM&6KlQA?= zg_fE@%usVib7;8YbV~$gB>Rm%Y4bkk00OJ48;k8vWAY{cP%5*K_N1p!;YSsd|8rR> z3^Cd0Le=Rxue2zmDe%|=RI!y0>1;@&XYRN)MU9~nJZ=cDAx$O08QC<;$O%oitW7ua zWHw|9i>NIF)QI*&F6Yv*`qDSeQBeQWUnRp|krjRja~>NtQXzFg5jJ5LwqY4oVHL2< z&d*X6lLd#ehM>eHq^VP}wPU3M6izf_d?#e(ins{TD_Yb%GXN1?H9gyLUE!4@qp{vH zhAQd3Hts9CsxNNV-~6;MnGSRG0hIQ_8;NQdgSB@- z5pX9^Ni$GEl9pKwldzifSsgd@q!o>H%UUboaxoW8E|Y7876ZGJSoffjRFrJpLSVaMKh+L4C?__zPcFD?X+dmb|1?L~^( zQP=m#S0W$ywM2t%op49*#BK=!Q{o|%)TBhY4`@r(Z%3`j1hjCMHi)+-DEU#yp!GDW zv_i>ZgGK0SEti71lxId(RI9ft$dQA?w6((3LBIDr%3w`RIO|f&aAMVT1!oGLa5jzZ zEj0yp;de#!^Jl?@Sb6tr?U*K)GzWPzLXX#YV@*;Q_#Pd&fs1(WBG|1a7>Y$y9wrxJ zNH$c9Y9l=K*&cE&EcuI*EQ}2)=Z5+NGnm3uaCRyHQlo1zEUCTh0t8RB9U|#=Me~XhP zaTln3OUi_ka=h| zvzl7hq1Cf(-8PF3+K{RN*dh@(M7g5(x1#klo@$p4{%Dphk8i7tosrZN`8DEd*`#kd zVlVZXsd?M-d93$Ya(|f!DF~p|+L5E$m|wwN4G@A)|4*ip5DX0(R#_8x=W7UXb8!Dw zsnNto1$snZxfEg2t*_~54U~uPDS5ZLconszF)jrOd5p=Dn&&N`2JloaW31C}V*5FH zfO)M+JG2qoZ$mmda4rckl#w{&=Sp}rS+=#+b*>9gJ!JEuk6H_?5%G9!ly$rD>Nl~A zMtF6apN1_al^VHKOsf-^q^XppEmTqg>#|=}lkFP2wcGDRCK7tAvh_J~k>#wXp|s5# zA#-?nE}%WjLarZ4C|_2G-rG&9nYAa1V2x7S=UO@W2iNoyDvP$F?_={|2)Ha!f8EwA6>fIO#66Z`n<_)rb}Eo zg0{u;>9twYnNz%e`BP!F!Cw`=yCHdH4&(*_=6KlBBFx z4%`*OQ%i}zyd7uCN@n}6&um-mAlX~mTz=!&_`Sv}4LG__0C#)nJqdX9YC z7KL54*4Ng;-ty140LUG^0bQ8Ln_4ZW|5#+zyQY;pP13nogZ$Cu;JLd@k-9vlyqwP! zcul!{+qu2ly}in-oyNvI#KU{XOC5q%J!00p)!AJql>I5VcD*rm*3Zb!zbu>#dEZ!; z&kc~p+%gFqFvd`b3i%Zd^dUwh5QA~}rjxy+p&eOO`)MWJ(>wiR6sW!_KI1E1<2fGG zjTo%S9cuLv2{5D|q+pfnM%CMWPj8ctn1Gj#pB^W2aht=lhzcDGp#^xIGi zo&qazK=UUsj(vxr64ssFqVrgI7k5&_ed94a+j9})r5@a|KI*YP%*P#BKAf}1c%91H zmri62icrOaWZhYQ?PIIu*U>X#|ElKdoyN7BBlYge!&3odcy65>-2+|FLkaCQSMcjt z&tEXuPVT!ebi@HZZXYP+4N<1q-tDiqxWThc zWuDd@?Az;Y;x${=t1d^=_^Ry(oCo)vo#X8x+SRG@$r~Ln`;zdaJ?pi;+Fdu0>pS_2 zpY^Go`IleJfgzDFA4l?i##wYqJtMMHN^aDC^u2#MJo+V(3#i?j=BM5Do5sF(F8Hj6 zna@#I_3s>wm2VHGn$0Mu2Org2x#^7-2p$q4rD!}Nk&`7e(tJ9j(5cjMFkG|Pt+vbk zdc$98r6~rZ&*`>)g ztNRPc4j{pFjmRK+axftiY0aiJb0)>nB5Q`kV9a{;j~nyQw0S%e*z6EbSTlHw1(c(heRo_qfVPX68wl{5I(Op;mh)C{?k{t555UW-*FaKyaHB>A_;eTG!hZKj z6`U9`KAH$0Gel(dA!Uv=9ZjyR+2iKUoVnydyK*$?G{L6)a>Da+1aYlE73IXUmeJa_ zZ{r@E+c0m!_9QXyu{#hRs-De?*;n+eI2}q!V3ov15nWdg~ zF7c?$7df6)|KI`Sc{ARAw#0;6ex&230keS*ZQdop-^fW>#K;l z2BSOXA@k@2Hr{Ayj$mP$shYF?FxFac zEi;x1*ZJ$N><%lWY9J%X@Wg9V43RucW*oOlfg}<31m@z2;{4i^*xOy4E7Y6 zww=+#W}vC5?YHA@dyv#fn4P6!X)L$!e+=oDB4`8X0Kk~YwLB}HK<_-Z(gR*S^(+e| z|9atG4_*s$^onbQl2YFlxpe44bGIr|+|0?<= z@^W)LBEHIFo=Q0hkli{UA^%587{cc;M9dO5w5Tu!aYLg7-qKEDMe*oG^2l%AZ7J~88bV&)1QXNl?uydg5GzulG2hQM%7F~K}k@7|Nah#P#tJM zo7$vu9yDX2Fcw!Wb&BrJ7DOv|2?wjW#B`GOLRt=hn zToL}DX0uS)>vI|-oLa3m#GazFus}Vmi{L5Lgpu@?atc{Af;H8qf~Ko_i7M|Zwb`yp z=d+rXP*&AQvdfMoP!c7p-vFr7w8C~5Z_N;51xZ^}8g8~TBWWYmXD*2OkP79()X>rr zzh;RKf#sOvdD@~4aiURGj4kAJPifsP-L|&4rIS2eNz}S7Qb9>o>Qax2Bj4>4wM_Ja zRh=4I^^ygvrA?M7jJ4VIsMeS*JgY`yd(5_;6NZY_Zbuz=sg!Dy|Cag8id}!{C@c&T zyAdX(T7l#2_*;BRu)XUYEsFtKmd~c=y`qh=eCPY7 z`WDn$t5u(&?8ChA%E`;R`iHfPy**3GG%>b1%b!;rRQM6dc`USFw`O9ZPaaplN z{;4|)J-`x!Ww=mYaDth92s1Q?XXHqnfQR5oVa{q)Ph(3bAi8CZemKy94E3tUOlG<& z8O>)x*g_ah89Kii#`|({i$yCB&zfb{z_YVfw|ZGUuO@q?|Mc^a*Q&j~IkRON4HuPM z3+C;DILypOr+c;i1|r_85o@5hih-wiwQz10Nk zs_J^hnVm7{WGrh`)fnM(+p1L=KGC3P94H(&w8T49sH!O^x|IBCDUGL(%+Wk&6zXu=i_C6Ky zQZ@L{3BCoTCyQ^GP%j;-CDw-jx$ndV^f@*kbHYKqo7(7ev`G*4;`qhzB5d@v#Y=Lm ztOKmn9VLbLy4>T5s;x0Rz3HjG;lByfxwPFU7yfJ-&76U^2r!#al!Oe z$TDC8^yQeKTwaJ|UhnM}I((ljSlJ3fi4vh;{{aeH=taX(xY|M?7;#0=;0XimG!abk zo%>}PB-ov&InS}l;KQ9>@Rghc5}#&Z+YTb1gvgDxbhr6)1M$giy*Qhy+yu zT%VNS_GQnw5f&PDUktk7>A>6xsv#VH8VL3V>_DN=_~DG~$Q_hfC`}l=vB&>#Ln3Yo zlw{o*&f+sVBk)P$P*tKPGDrksMv8Ib|0iM=4mDrVtx2u@+bEJoj~rg~;YF%IqbisT zE7IaI>eV`WonggdJ<20R9gYg(;yG#F9I8`4PDK_l)&pu2OWBjph*blsl+a+Jf^~uD z&5%gR$_3aR0+pJPvES7lAniriG+c8W8z`p;&EUWkmG1% zhCy-GKdIuLsbNNHq(9!{3~HGz_9Rd0(V84qWs!}Iz*1NE$fFh4@SWW}fl?#RTT_Tt z4Y8m-M&mL~<#Z_q6H0~^gya*-)fAvz?Id3~ir^I*od|-X7JOsTN#FEIL#UXeo$pa z?%q?>qE4=1nCT@UI*Ug_1|>QrZ&ll5mZV9#q$i>!D4yd=s^m+?j7qTVv#aetuUu|j(A*5I1IK%GPq=(#52KeTu z86qPhr=fM{cXsB?`A8j}p@sn_0v4bVI7DMHCwTo5CHWR)+Fzty#p|ul?ZApfepoB1 z8-Zfwkp(9*#!E=K+j!pF|6G+PYWiX(nkQYNWWjA?C$?m4?q~+tryWF^@8o6}Lf?p( z3~@$iMMWHJA!#ZcAB1rqK0@h6rPFop!tsenKprJ@1m4hqWkEV8TA)QrJ%>r;pl()* zBp4@(ZjWp^oYNb22;t@6}J)i+J4i5j=A%ljUDSGNVEv1#PCRk1oa)SXY1YR@oa zWWp$3aYWFRshltxdXWS~_8901%ywqLo4z8VsOx7E+1J$#X+qYQ2CGP(>YjR>d7)Rb z(q~Op;l)}A<9wrf{3soK=&d20?xd-I-cf+|i;1pZOC9KeIw|H|YUgd9GuU7(lxtyf z%^?13x(=l-c3fSBXuRer$>H09HQE@wgav_uAaET$LdEw)&(7YKym{-q0pWJln^&f4 zSVAl|0@txF>asek*m5J;I$n@`Y==H+?+odD;#Y3;=5Kx;H+5^>o3RY0ppr(*LtlKnrGP-)ash;c_pj=D%2PJsI0?L}J_j!dL5v)+ILb&cED}bv}=8jeluHdRJFI1lgC$8e^sgiY;_Tne^x~uE8 zhO^0#|C>se`Hsl?%JBLgryhx}y*+IG%BX6BQU3BT>k4X04&7VM8SIXueO_pK32}9H z)*LAC$yi*HR&Ti`V7T$F-EP3T?PP-PLcTUb4a4iG5@xQt+T`e>9{K`n^5^1mX+fr4 z$srY=Y9b+kFC^OPzz!z($s+sC@cOPm=VB)dmr~X;>=tb;{o-kF6`l_>>&ULx*gmpb zHY)(%E=)peX))N^j;*G6mB~U61IuYJ?U5TCOi4vB=8A%BSWg*TP>EnEDu18toR5P_ zfN||#2v71hd4;O_pg(<=Wyplow(#Y`at!zJF}H8QzVPX6Teq2TlpU=OcWuKm@@xXh z|Hfu4pa$y2#%`hh$j4R)jk#`UJuxW5uqi$1HG=6F=k6+3KwW}yeLN)j#CVMe7dGW&4~v}~3Z^5}XgB1^Ni zE%NE!=n?~PO*S%YTC?t4GC0QxA7nBu!|lmFw4|o9ENe8o7Bhb>g7xr_1raWz_T(#b z@X!$mOXCB&h=xC-PngEB3G=J;*57@kL-DqX@cx-ZAM*-uF+I2IGhcKLC$dOVvqw`N z#zIxw4ly?u@fOb6NuzY+QD;p{Knlt2YhAAniSegKF_#c?PT*=(zq4n6YE)~%|3(s< zi`-CS?vM>#;!>C0A08xAcCQFmXd53iqnH~iHFP~=_EAtZGOqxH`W_+QoQ7sJSDWsh zuBID5=Scsrj_hyro~0y9Q}a>RmAdF6)p&dk63;L0+fTW+U@eLtTS4GwE6r zX?L|@lVlHr^|RvWpI-AgYwU`7bL`GGuhDeLjFk)eLX*^Q7<^++wFHYD-FujP|P{W&b0^ zMa_V2^q4n>bD0N4G<{d(VgoG{BG(xdjk83J`e%2>FtmfD5F! zQGK*hE?Ba{Hc3}`Y+Lev^Y`1{#*vajC6CaOz=Oq%ITwObt7f3&1nw;tf<7LsbR?t- zd2!=%?>w&z6~lQn{0@~e@mXNSx zHy$InpjeWP_i<;t^*E$A4*z(lQ-uo@K&h9y37~qCBl)T~01CMJtHV00D>4eD7Qq7YdZVtLS4(;j-OhO$9ugxCITz~stfs%w|cBw{KZ@RkW2imOMC(3 zx)`S929h|P4_sKtCfWwOu+x`+>ux9vJDHnx1`+D4!Hs)?1zkLzC0&wP#q=?Vcc&B6 z1$%p)W5@5(WjugoeH*u*Z7?OciJSeog|LsHV*z{{`kMLM$Q?46W;X*x?ju3RiVuQ9 zQ}0FAgTRifl3u$38$iL!FvbP;49>fUex>$`x@kIj$4~r_vwGjR`v2eiec=DS#-F<0 zcf4y`d6hHR+%x_GgN7CEZ_1;zlNzyv&D2>muF82CaHF$c#rNhWp3m@H^T?E|v##8*NyE@=ge3GmG`m?{@ z7rx;$8h_gNJvS!(%6>?zrOHnT+9Ep;1cXCk;9+9l?R>1L>$MVeIG_q# zUL19PWm`sG1XQjLzRv!$mNsCHEP5yBG@{U)&54r~X;3*{oIZsb6{@#z&6-NZnw7`_ zNL_^}1q((iPo8ny!1+onOd7Ih?5wfF)JzyZ+JYTmwf{$9B2LX5b=~u*@*1|( zj?qO>8jV)D*y7Bt1c zohB5D7qPU$Fhi#xTjW9wKkUMQDy{%=tTdXMOR2B!a6^c*?0CyXI)>@!~gtPq`Fpmf}$KneyR-^)rj%N7~m$5 zO|XCZsz$Wj!eMVWBH8o>&e_Jvh8=X8!?QX+>`^jEKT%O+l%3dkV1@1mGW4Jo_G*Qp zKS`_0O^GIz^t_EiiQqkzHofmZ00j6n)cr!e$x~266_wNhZDQ3yD4TqhOF`SBbvO%` zLaRp)OXO85bJ#mz*Iqvq5xNS7CH9W6;yR1L&6X-+jko5n_P zddrMWYcsNw(Rs!rO@prZ4ENi^GOmtYSN~E-)F4AwAX(%Tbooc!<;p{^W%XWOPoe}g zoo^+O0#wyh0sV`Krkp+9nZN>E@c6)4n?#z{L1ByaLA8*D`l)W!h#Ew$x|*PB5NYH( z#V^XzX~ks$8pB%I5))@?9ueG^P>lm?1QQ91)py{w&Lx!NC6nIw$p?jFH(LbDY@3!i zy%YvayODVlHd`AcSWUAHKA6(9%M9l7^TSzQh+&Jg`32({L)zdE& zAP*#|ZC3k(-2Yag7-yKtGNCna%od*({=~AaR)S=>N z%@ac{L&hCLS;mnNa@*Tt)~YUK2L{|c!mCe z!@mlJ<8ez!muK))mqygkq7j>iL-R$7Goh?O!+GfAPIJAhk)k!*7{(UIIh)it#yA{= zDZ*5`&;t4sUYJ6u=_~dp?9FOwuhbV-U=a)noKrfM6h#yw9 znG5h4s&a-+uG(apNB{h0HGQWSZQoyLxV)>k^BrC zD_}R;he{HXhMR^b$pA{#mNTT~Eagp8iAtFY)<>)aV@$1~5JZh^bWISOI_k00qZ-vu zPi@|H5S0#w<}jJEquo_o22HK1s)=6BW;d5u6DIbjx@+yKcX_u~^7`((<2`SA#Y$Fi z4$X>@`72Sz@LNX3_*u$-(sJX~wN}|xQfd_tIAo(?`7teGq0^68} z(4_^1G>kxOt%8{f>Wxh(bc96(AZTNU;=>Q_;Au+%;o*|`c9@>E%Qk=oRq(9#w91R1 z`^3az-)LHtuXb&@Bxkuc)fzYb^oxizjOto@@Cp@ZU62O?*+8$u$bB;Il;H-|3K{wd zL~@q%-v1V<=%&(5MLchK!D$~ccbnVa{xZ1jGdyWtSDWPyYZBc|-t@9Lt$zk|8vMz> zcn2Du-aEqKEkgE=n%Z4ET1%AIAG^4wwD`;o}8Va3tOh+-*OH%+4d=a$- zI4$g8r$7U?mUu^fS*~CV(l((!#xh&%NgZRV zJ-g>#_gVX0@2Rc5dJ>d0zVlrs8r3_nI;W(-k>+7UFa24~UUViH>tC|Otq>3Q1Tfi_ zL;ov*bXPI3%zkC6zbYhk{(_KBN*yk@x@c;w%408 z#sw%3WP|SS*`o-;9E3g&hUVaHBg`VF7Y=9el8NP?<;Hy)(J=e0} zWMUrTOqa6a@|aNoGRf<-0<1g$G@8Rus+W_oHUW-RgE4ZTIlQ|RNsBpJlPwf0j+cXo z4a5T0*|<)dGq?b#?ASftLz;k#vdd_=XY)HIbGC-io2Y}jETg(1+P*NOK4=mU>;E%5 z?gBgPBO)E#!6Q7v=!>E*+mFE;KdG^;r+Gp>8zVcKp8tA6^Ff;VBZ~UNulV7T1z9X# zNs6MtB)UN+z|*||#F2pmipV=af+(2;bgnbZJVKK}CNme)ySWOqE13Jkf0-~?fE?Ge z3ksAyIOI6-5r)-hx`uE=1(d`Yge+)Nnp!z7R-iQGa}VXyG9%nRB^;jWJ2&+D#8PBF zvI{rt6Nxb+qOF5P=!?ZKd&O7ewpyG;aSMq!%EI|Vx$~=N5OZooNnAiG3zTI<0tDo+jtfU)(#A*Jg9IYPbpJ%bD8x0K zGbIYND<0FWQ9Cg%(y?2UlM^z)4-+z68IBVS52tH2lp7%jI}E%`xFk~|K6R-5z$W)xZ#bnIJ9J}s=Oqq~OlK<2?1W?J9WFuF) z%qJWXC?m#kDxZQnM)-27MQHU*Qz@OMZYwRwqx3k0@O)CqfmZe2iL@vGMv!w6na$2Q>dZ9&Z`u}K>xIhj|!;?l&ujnsR>BL z0-QsBT!9s7gqeel8Cr&2_=o7B%`KD!L1Maz6R=zx4#<+9=IkNi*}(u6#VI01#6;6H zUA~SS!v17avJ*o8Y)s@iP=8|=2<4T|B$Ll{oX~u~(LBbjiNw`;#${kj7`?B6ID-E& zi=<%+W0+7uBZ3&LP0_la5(7shT~Xd#0pCnIOBK#_G(8<{M#PC_@v0I(?uq7)|)AZ3dJnfZl^`1Uc7(W%ws3q6H!cYvoMlra6LKTEV z#YQE-Q|56>#3QZc83IZ*M<$_{UlPG!<5W6?1|wlI+Z5J4B!>g*C(aYuZFxtngjt0R z)rT6=gP8#8~4#C+DXTQjyo-JhkRpKV?4D%zNQLa%8HrOi`g2^P6~SY!NCsU=}wq0OxAFhkt{ z)jZb+5MFU5zh>)&V1?A)N46}t%0oy{R+7j=zSj}D7 zhznr3%pp1PAer-ox0E!|!6(n_fEhgnHYFvQVLoWDK-vNNj5CHUten`YyipjVTndmt z1uk)YD&d}fP@ zYW;HKbdq9#(@k3?qV zM@H@p(NjZC7JRzh(3I}bd=LWg4+JOxBqRy?K#7_$iSMqNZsTq-%k8%IDgx-YgqG-$ zPMqTf+bzgpcdal6(q(|y*yH7jyRzECTQXh#=whmG%xJD!E=M!T=?p|Ji6!42V(C5G zg5Tv$T5Z%G)6TQqSCL(seGTv`cs=Z|A;h(*Fdstk!BfBA@C0YXh)u&&@vW2Ja*HZts?l`1tN6U-BeJa`1M( z=&Qa)t89(tq!DZAFK|RsrWlECnzo(j{KoQ$_N7?);WEcin4@J&F7VARX}oLm;sgZx zb)|}krL4hP>FqF>7MoBSMxL;Q30c{5T8cxc*Df+yq-ZkO4sw+>s1b)SO51Hw5zsPi z@l0oGbB^%}o<7#CaXG_ryF=vVW^SzcV6A2$x-$zAZv^Z92v7{OD4!1|xAj}c^<3BW zUFY>)XY!K3_3Q(Q^O2E+O`28HRp`OtNnntM%%P1{Z97xIhRiz%Hx!q3Rlk+=f0)O< zCFuj_9A9qkum3qxTT1nRuoV%qxFpU^^NEFj`w(5M+qYg z_U99T6NzogeuKTfcRScqOxy&>hW4h>a>;J?muCtTCK(wGqhUySxb;10$M$*CY%-pk z^3y=P6?fPUfDq`82fYYX-=yDLAx~~~FHHkYXjx0rRb7UbMvpt{279@H^u*r0Y(7k} zE>m(7__cR&PWSY;M|gvu-~u3ahS$Foskd?GQ$Gue><;g%i}H^b{J|%D!XKhs@9vcl z0C#pI*8iR?zhUn|fp0JoG>3F^1Az8(1$qEfvW6>cM_u-62~pa9oFd@)Mx|R*)dpGP zz=0(Cls*qDwu4%dLvshZbccF?z@LlOS5c-o#8|NH?R`Y}@P-clmEVcZjxOV-$lN}0 zwuk;Sb^A^qGEqN@YcXL^hXFCR6gjYO{~jFm(y1QW}aQ zicU)|k7VXB2pFL0x;-AqSMd3xJqmRscn~-^r-107n6_Alc$g>{A=y|-s5m*H8Bwtr zf&VuaDmvE0r#GQck+Fd~xx@vVWyOf5MPYl3o2$#qw$!HH~PA}~%4KyK1tqP|`*ZvGA*FF#LTZ-0+ZKi+=6-`>tHU;%*y z2O2DhFrdPP3yoocX&Ye7a`uqtrs8E>dcE0TCGMzdC#8&>CvH%ic^M_jB^hXO}O!^y$>^`7`(ryY__Kw;yZ>KD_t>Op}~Ej~=7u zi6YaxzgU;#g!d=!8gPLwss8;k`_sI6)J#JS6(CLp4v3&oN*yR*PW>gtPBTtfGe9a& zNQDN4OblR`HBpH63x#)SXrW*lUZ|UfO@J6n7+;X_)>#`4aDgZnB(qmFW@RWx1`zF7 z+K#{}Cd?vE;b!DwN+!10W5J=)L|d3m8Omw$RrLys&!kcaRk30Y^)Ms^q4oc`e{bYxzcLgcRD5Kqt#}IZ* ztw&#_6{U1(N0w50(g7rSlPRYcrL=?q=)k1ke>Mfk;D7$b#Gi96p+hUAk2)IAbF#+z zDo^RO^wX>W%?hDXDwzn}n!~8XELqIf;-wa4sfeKiozNmb9&I6wpi zy=@puTPxrw2e*GX6#!qy(v}D#iV@P}wvgd_?_rVc+ix~nG1eiK>0Tz~D4n@7B#Y)Q zp}+=R#`a}vT!=|##br_~%Y?RhQ^y1!!u9Nm;{GMDotrG+2{##Ffd3pa#|^iNJIwv5 z6m&Kh%5%@sB?>gIw+{W?(6+MU!b6)b9jd5#IL-7&@yQ%Ddwm!n0Iyo}S1UW&U02S| z@q|6L&wiF&PM={P%4o1T|EW%M5zfKpHJg|?q7}(ZCB?VXrf3D1$SPy40@QXJn%`3? zQ0`VujG=A{Z=f^_kiE$}SdjeYMJMNwBnC*pg(T^3Vx(ik$lcCRU@%)kgrdQf&|=j{ zg&-dQC&#*C)Z9;ifzptsNn!?}dr}IUPX@&Xa{QsQSVBZYB016Oo`$(Hl zxCXbhAq`n`fQsj~;s|G9$PcIah2wq$h-)#B3aBWHoamOJIq*(ej?jPxR=_YjT%ZVV zxtdvylouau4mX*r*YfTay29kJbf?4DzZmlfCkc#BP}vBT)a5!aSV)7waaal?_6RXa zQDaoB*c8g!l)i4z*Kh{q^f{epJgw;JcDsqw2DZl}_1~!{%2XPIQAaC-PtiLfUS{L*U zi7Y9jWpw0ln*0_ptdR=Dtw>?!!KCG)aG`!726P;>;r|UWL5#x4@^ed?p)a3^OHGAI z6m#(cTf87f4?2+!1DK3ys>r+FZ48>!6isBd5kyTGXN=7wWAnC&#wVpQPt{w6IP{Ug zb24(BKNH{h&S$=N;%a?(T!{MwxwQB-4WA(K)WQU500^m*fc;>nLFH(w1G)-(rYhgg z=Jp3`p$J><%B1qH(jw)}DTEnJp(?2m2M{|yP`I*o8Sq*AFY0;KRt9BUQZIv!}diC&U#g(I0*KxBqaezbwK452VaR5E7$4GEmA zsK}P}p|)tic71u)OE=fiU)oEDx$JBWXS>TAg7CFI1>$E8A(>~0l5(PW(Ylhkyk$zE zvWS>!G_QCIF;SqZ!m(dguUlPjg43(q?Z$S+C|0ae&#cF>&RP$PUULu@uJcSEd+AEo z1)2x1?$IZu^h)1}__HMMGZlo=s=c+5ExiF8@b?nyt6nj*vCK&aA0x=b%~nBhQSs6% zALz`BvZ=N2dSEdu9J(lQ0=3DtTTACf#Q*T(#xpqm#Qj@!pQ^NuK$J z;WX9bP+Xu#-QYKAy`%`O2&Ya+eobqL4Rh?0Gv-l-^0AGL46-6ei~~LQxg!ZzYoFQ7 zk4p1Wl=f|t$hoCS2R9*?27vsOuiQSbdOH=nH98KqJj+v7C_@ke5rlvVmE~LFM3ZlP zWeZ{^4>2`HOiD_($PA;l)<=~QhuT%tw{;3)sBgLw9HKbzM9LSb>6LcW0mjMVodiWv zngVBNx$V))gow%FRMm`-ivQH*E>>wy#+!qlo1(|Lvae|Yt5a^b2cY|Q5J!uS99N>b7f^#JqLVZMrN%ybZRDa zB1nRJgmknQNZW>jKN4=1k~M-wbshLK&sKZ}vNd@2H3U=uhLM1jBmy9Ce>7DAKX`v) zcUf4Gc$1|qgTq0Rw*OH|Sa}3@chkUHI?`!8bvl2yLp=C!gGYETHe)0J0%M4Oqqb^^ zw{h8!fZq@qrT_|Hz=Ftic52dUT%>@*<7>&IVrW!s4(Nw{cx>bJWHR`3tOt8?b%?RI zKC`!idj&|2xO9G`d!JA#FSua8$5z7^g9arw1=NYq_6{`&AUBAE?*>5F)^%-H2O?m9 zB+zw02!ugsfVKc*26uTKM16*)L9Rt_vvV4n7eoJchUV8m8)bw_!b=U8YBTg0Xee8Y zH-|E3aMULw>5y4 zgN_w%w|Hs^M*t*%0T>X3hz0>;D2B2a47B5nh$B&EI1AVEgl6$AS)@E8hG~wIcTPwJ zKL`RMfPJqOA2P*-wiJKw7n9LAhFF*gF7SIeGb_`$OU(0mOnH0rr3VrPGAn{|DF;iK zhX@WcLZ=a44k&Eu=#Hf~dJ*`L_6TkG7FM^j0li$_K0%nZ74X2^buI(R!{~O zgI^Y4XW28+=8)}R51r_Vnz%Cwm0-FOD}>dM4n+XX=Lj6xkt>o`rD4fRjL|lR_y7$gqt4_gg1PTSFp-BIyDtmwfb~4*k<4 z{r8(Dz%m;U69SMwqx659i3LAZXpul<<~Ty>=#JG1k7CJ=WSN~27-jl^mTmQcYdM1B zSyya!f^i9FD%gmAWjO9}kOJ0_`H+c#$!ET2nACxZ=;WQEXcJxYn~DPPI4Kh( z0v(B<_lIb?Ig%g{0wi@Bp5vL7lag3+I4o&)qj_?=FbQ?{3795SNdR|N5CTNGlls<^ zDrr+9di+G;si4tGBPmTDE?va;+DWLiJmkDWD?vQ** zd0sVoiWd2%4Tco!z&`+brV2Er?=TFMiJ%amnFhd$v1osRx}mnvnWFYV1^}9M;{?L^ zeIEim6Y4Az0%_-WX(~~ou=tw#Hl(cpLoPZ?Eh!8&=@$!wcQrJ$&mth%K}V|3&3k6xOdN;i=7 zlT7I$b(n~z$XbY2Ds7F`knovaJ4xr5f?0Tx69X;!{Z?oBU#=T=@X$_^V;1j!i18e@LUr3ZCEEdRUsJ zAxNGL0hj1GmwP3$v^S=ml304VmxFnjeTkn>d!KGPZE|xfXZjs$Hgt8m9lIh}rU;)5 zB@gpL00>%{LCCPtn438%LcF5@;>r|Y=vv!%qB|*?9X3G*OR0udIsV2ht?8P{IC11R zBvsK;bC9u7OSS9zpE-l7h}*P>8BaPob;+q|wQ6#e7feZ&a=hxYSpc&&8~=LO8Kt0W zUB}9;I2%XHx{p6wf>>)S(JHM&n}R5KttJt5$QP&E>X$dmkmePxt*0xr@-~fyD0O8J z=juO7$+&5nJ?{FhY2c?E>56AdltihM4s!t8%BK`+aw^J1b8>Jei>P@kBs*p|BwD?l z%CC+Asb(P&*1@I%K|U4PxQ#2d-r*cX0V&11e3EKSx4E)~!MUdNxiM?NPI|xzjKB+w zx>dTgJIkygIKS!CtVI*FaVfiyxPnLf5xu9af(fTJO2708Kq4Gye>uUzE4;*_!ok}= zUGp~k%b)t|u{txt@w=z+kOr)npd_%VX=|^4Ykizqu8Fj-jM}}HivI+H%Zm?`J6h0; ziV?9hwg`NClj(bSk|4Nk0ulSEvuo9xA-ub9^&Pu{KE*x)sd2&&m+68@m}its8tLAaT3tKv3}Mw0vy8<2toCJe=v; z!k1jhEzC7BEW-nVy!NSP%sUUiG7s(}sL|(U#|jM2r3cPW%A|elzz+<2J?pc8tVb7&!G=7mVw%Vp0sjF1^}*t*y{U`B_Bog? ztjYci(3c#%nQSXsd#1JGwT~NMU`sY=fRoYJ%4mz3t1JS%GXboprm`Hs0B1L~i5G{O zWO~R`sDUq2Y^j*qw^%%zTg)TK+{nrtzc|>jkG!?ALKFP!x`MoBX>3=#+_{+Lxpyqo zdQ8-#ivZ?KU{}hl?Ch-6OdehO&hQMTh-|X(A$4MGzmTkuZ>m6cipenS)e6DMPAyl+ zn@HfArmD(;<}d}+TfO(X%49fq65s$BozVaw9Fyf*69v8?^pd^2t8@(*1Ph|)I|XsA zzMd+R5Jb2U(QMiib@pk`u-t4@8dwXVr#Zd9?pAc_D*q1n(z4zhJ>yK;o`-CuJ=8*N zx=6jI=v+r+md=4pSMCguQf=G(#K9Cn0NI+D_Z-VGt7Y&Da|De*|$rh zDJ|V`ZQVJ!R5ZQa6~L!^SIj#S+O;-hxxiUmkpJEf;RK*D0B)_e=?#De7vD3O%n1#- z&n?Df%HL#M;xD}7kRkvCK#p|mH>Yjp;ha^dt>A0U4H2Hyl>J^jTj4`P+v^$6-4>U* z?b!Cry!d>&s!HO<{oln+=z?zCC?3`-PS&36E6v>;R<5nIA~r|tw>Mth(KzP1fHm^{ zRsw(wv0AyG8DaOP%OnIB7UT^^p59h5QBJPD*fQ#8G38WV zh(6Y~G7gzO#99;!XU^tQ*5Jv$z{IKvZa!uc&a<*@;wIQfvHNCuKF{yV+cF;FfZpH5 zjp)4o>*b!}!T#l)e7rm{P`m53jjUc+9{;Oo;N+HWy;=gb<8Uo~fCu}gx8eQM*H@Z! z9RZ`^VOs$uNg&;Z`V=13((hYfGf z=Bo|u67Jg4o{#Xg&awy9Q9ZIVk9$d{=KRX2HjEMw>oL>_fBM+j-TyNl~yR~9)0SZ%eGgs3#o4En;FH5q ztDm7*-_QL<#O>>lvhj+pGY&xJ#PhNu&+Jv!_kX|UDIe{Q3Bg!O?ffVn7%ua+XJ1sk z?~dK=ea`cRU$qz?^oc&u=8pNBPyh5RzV1gapBCTSn0Ugn?DXmD)~ZZ~6>WV~AAESd z2XN-^!!YD(;q_^u32AZg2yZy-os%$nhTaERX@6iL?hmCnSe6g$c0csS>khdF^2C$C zC2#V4Z~ZFs?SeljaQ?*HGmfHrm^z3cck(U=RvLAR|a*C>}v1=TRgg3?vUq6-yxixLvQ{+XVwMl`&ut2n-Kw zgQKEwx7$~H)|$UxvIhMhKRe)1{oxQyk_8q==US2+)7vIGYlfU2pA7DU#X)X~gSTK!2 zfeRT1oB*wiflB~RrSjBi)0cVaRNboe$|Fczwp@v%b&{k=l_gn*WQh-EOqujvnL=?B zC(cyk)UguPB^9SGLkKuBlEi>fCQL$f`C~`HPjb+{sHub~Q6WFUggwQ=XOA#q!Yoxf zbZCNt0Y*a%Sc3Fw8%kOw_9JBuZ#a%Yd$NjmZ>7s!vjE5Jd!TD##lqlZb4=|qc6UMwp#t#1c7}GY1amr5O;2bx^4Fs2oYLc zin%@)7qb!{O64S%H)sCbdGwSmXI8h~+C)z67WMA!#Dyb9E+x4TK;YuM3n&EGFF8Wu zJqZz90{r6R;9EveWvZ%yQ)Ytz1-!~DuEGL~0t5kZ01r0?QQ$!cGl)O|D2O=9!Aa0F z4-WG-OaMd>cMBjia6r53oC1jJOAOOeOYS)sSu^Ym1M`5;0SXzc!m=JE1I|atuA!y^ z%or#U$s|)Wj722FdveL}PIGO>DrbZZ%UAR{z_%`a<84GRy&TiOGK$PZMebVEZ%uhz z)I~a%;)HHGIsZpO?z%kntj9I&a#GVobXZ~Klrc&e0ntN`NOT1FmSRFa`kuN#1N^)> zkqfH4xC8+JJq?bK0&N85z$_xTLjncZTh+r4)0@x*_((v{Jt+`aAjAa{J)r~$AUG6Q zMukOm3ne>a6TCDv8^yIO_r$JQm;@DID=ZXqAieYu!;IS?9~u(GZpHQXm}QMUS1;6{ zjkd`Wp)9VnDcxDu*(>X%QntFt+;^bd{8i&$h4{7203bilqVz4P@59jEmmwhCeR|0`5}_ zTv!z`1^-bkRcNAx&O_H=l@`{5L!BT2*hPWmb?IJzt@`9clPt(HoA+vs*;dlwGbX|A zgiTssY`isB7ZiAa+;Jbu2Hd#qCK6;i*P0L@XRG9V@eE0E(HE5J5f2_@<2c)4ucf5 zpanzOkV%x`kIPEn*It5=%XMxg4l|+VXt_dLLQpj}X(0^3Q?k*h?>G`OL4bTHp#R|& zk$1sUCK7pOrf&_2iV#^M00m>4)Agd6#E*{+Y()c5fQn93D=`Xa8+x`S4K9Xpk&RsB zBeNk%i^A)d9@OY1OXEQT3KW+ijbFY>Nv2Vrw4?&jAOzAhIRy%CIvI6Xb13%5H=S^n zD7@uP*HyZL^75t<6Dqx2>9JJc5UDt9CN!0~JZD-1iAGH37HjB&0HKG5Q&0mkE0E0K zl@n`VErD3cI##om6|HAg>sr&wR=3s=0*K7wT-!$1xk}^!5J5~mS1>*dPt4A>|)Q|hF0fuGYYb$D7GPj)hw?C>dULqSlV>V&X08CyYx`7>6r zX8{!;;xVuY4Q4vCoA}i3x5^OJZf>BicI+J%_0`m#$u(DYrD}R>h|V;&bp-I;-g?7J z-}>GcuXEe09LG04eCm^*-3eJK3U^CDsuH1$IZr;_hCj$c^ud#*Y-BPkzz-&DO&K;$ zfJKwQkHS>6A{DVINXpvYsCKm`?uP<`V$feI7d6U>A$UwO)aA(3w*NWRac_CNF5`BL zxGfsaagU1&qvpc7tV+NThse|b88HLjeJXc(C)MxTcf26dY5gXRn2rlLoGw10ckh#xnS^6RxZ^ zBnw#yFZx?-tgvR2)!7(cc*f?yWM~=aw-JB3Z>LppZBh)d3z~QU1n3A~E&?|A(UwrK z1Y2x(Ygpd)xU|0os5*Y^ryzepMMM_%kB7RURYkXm*`-^uqnt!$c$LZDb!U3BoPaGW zlXzgZFPXPpW^Q{s%-t3@Ro6UcH7|fmd7Wop`|DpCV)mjR7XN5K{RyX5u7kXURwyDF z>R?4I2^>Ar1xmYUK3GId?BheHL6uDRAF|oO|b5CrdiqXkas-shh2G| z0zLL!ufyek;-w5{K5UIExvOmlPBM9J4P;tb+Cq13@c*8>v$V)lno=Kg)yEBPr{6x@ zPscqCuO6TSv^$P+yd%K{+iddVJb*}?y?*nDe14nS` z@CrbFufOUkt~g9QzR&)|5ATpqUe2HZY!1EF$?S@ux}Ip)?klX}O1fvTC4-xm$O|Mq4 zo)Uo1{A_PBOp}gp2Ia2{a4@TIke&9;=6J9N)eiasF8YSB81|{7T4d8SEDe)T$dZuZ z9F7K?g-8;zo=LbFp76Ztxbz7i9t%k5l zJO>x3@eLusyo>?6kcoJ@iqXC)i1KjFg6AFwAQ84I1Dui^b13Q9(JBp*5U)}zvy!Ut zF%I;R-S|-wEldeFk`*(`{y>49Hj(+-av|OFogj_+kl~?9ak3U5B@0XC(k2Z_WcZ+wxlui6M)D6y@@fTJHU9?6&q&rZDn=DpRvM zQWj&f{6bO{=s+=5iWk|hO9rgo7>@}z&s|%|W?G<>X zHk(ZjxynI}Dcb;K9RrcvvaHQ4AOfH=0AerAsxmh}v^POiL_gF#8Zj)#M+NrGJj+r! zX%WO&@>1-FEvu78;c_9Ts7P!yL(!-O#v z2mE48@zOF%C$rMfk|T}N2>*|;!z}SgXYmw>tS`f$^7sT^jEk@obeWWm8;R+MfJhu~ z6P(7XHpyxPa!6_7>;h*qDo2w-C)76sRZwwrPzyE9mM5-&!*_5JWVSSY5N=-T(?y%p z9}#Kkq!Ux8b0IS|Tb`*#b#ULfa}?RgTAE|M4la!iS|O5bJtkaRtz&^?>9B*8>J z$Am>ygfs8+KAUl)SQSPEas~-U@5=NQh)gmqXLOiwCto4HYV*1zlz4_`hd3~J=Cqu? z%B*~6trCS>t>`H?6aomKoZxUmM-*KRHC@-0LlIHC-VJc}Mu9FbBRxY#89+xrHDB3@ z8T7SA9TH$GBV3A5S^xFV-ge7C0n;B}5Lm%aSTB*1FcUG+k5+B<3LO(y_2_HR^8PT) z(ms~%U=lsalUS2;wv+?@Qg#|wRzTUXW5!fJ-e)=eu~}bX0Z8+PfG8ZRj5JxmYJ3)G zvlUw*V298Oc(TTKrho(8idR^`X`ePk)m3WQbwt}$QQyrh1u{R8Gg^X+7AdtG7;;Aa zRZ}^UQ*R|2&X#OBk!~CkQeO$>K4MpmB^4F+IAalFVYNVGbtEg+Vgm*xm5|la^KTB5 z3{{c~Q&w@|k7ZZZWgi!EAD6gzweHHHrOpN@kEz+Zsm;hSXiH!MuvM+JRZT^AzPN`` zd_|ndYWGO6b^p69Y6}r7W!Gxo6)(G{a<^6$|Fb|#WNr8LcM$+b{|k7FmpVfP-U9Go zW$hO1w!@tFJO?vxUzK;0^uU-@G5tj%o|Is41V@4=bwFrw$CrE?mm1C2aiy^hBbR;K z*L~mjaaUD8HSIeIi?9x`yL3ndg#T}dY8WAasYZ3d%8Q5gixoJX#wihZFZUi;jnBA3Yj^2pxWXLOgT=vkNBC3w zwT|uhLz34YJ#%x?reU41hCjA?r?-%kb8oS?hu2Rba1`4Fqyr#%10tD{C)tv>5hsVN zWuusrJK2*zS#ssqL|Ry=tg+;D;RGbKoSxw3HsEu4C4hsrPNDXI|JRmpnT+9!c4wE3 z*L9b-Qt74=cq;gg9u*+x7lXC*B`-J=Q8l8J^?$lc$*0;Ff%W6pLpWtZe9&e-@YZxmUV4 zX#e{emlya@*0`4wTA>B8>9{KaAb^-vRCmkL05w3$zi#n6H+P$)QoZd1U2S zna#TUJj9thnr)?-t)W>5!@6rC_10jGr0MnvaX6(VHl%l$V*R$9Eo%;=shNCmu!5S2 zg<6Swnz0*urvU{NTI!w+kGXikC6GFS`B|+XI)P6Bpt<#yty;9NT0s%fbrUg-7h1Iu zI<^0mh>PA;uDP!mZJ3$ub*-QIj@|mLYjmV(IDUFtn`aw94cTz<<*$4ArIXu&(s8Lo z51Oo@Cv)1dr<=MRdyu*z ztEaAkOBh71_I6kF{){me=XgBsw5o3Vt+xTU2Rs2<@(DBeuDg_EeM?Hhc?yqvkr@+w z3VS;rJ8Zbg>$B!z#T=+rQuYDp^~rt+KUE z+s0$O>Co18;V!|EkTAXU-~?b;0GzEkdbjiSqXQYZ&9mWRFr=4LzrnO?Svq_13a0;0 zO5L2w8R()7a3k2-dBek8%sF`_JE5q1QgPi7xoQgjYT;&m*2KGN$e|&f47$Zh5Wc;- zz4M&UKQN+uT!zWI!8=X4(n!6Id_#zNQ{j5RW4kh!{LyXLBK20Kk8u~qITyJ%4W64A zxU|oWS$Awt=15J=QO$1f?C^xb>xC0mjOeyMhEe8mPS?G+DYw9o)lx z=1ScZw!6(g9K^5r)Ndiy4_eNLrQP4l&S^cXdA-)}eU1Cv>3+T015?%$1yldAGk6sp z(HFhn4_=Tv_Dc;jR;>intFW9moyyDvV*|q5L0#N4UgQ5H&Ceap$9H+>WQ?lj)rsBY zwTHcO-S%`{h%&Y5P6|J|+330Ux*tL=T;%09kN8;#e= z*dX2K!`|i(I_A|@w|D+It2xp&*6s0HddJ$V7rd z_jT`8KiGrZCH-D-j-K$b{iGDIskq(Z6~FO^-|;D5iq%}Pss8YQI%U_rPTns;HGkGm z7xc+K^h@9DvtRAA-}Px9U8Z09|2qde1@`Tp-@U)j8g}kxf7$I%_tP)pnfv#+<1-np4yzGu zwmS}=%ir|7J&)h7(|$8xA79|0oC^Yh0Kg(+qvIoFq~s)JrDbHnm*%Edr>0q3M#aQo zX`CtQscI_g>g6kJtn4gpt)wDuuC6U_tm1&dK)^6@vGFl-vhx43aIiBlbhGq;Lxt!@ z26jd;bHJiNL4gnodAa#Hdb;{Ld%OEPe7yWTefrT85&L)p3cr7UxqtqaOd+TQ$rL9D zlL#^4=^>|xBMAMeNbzEwLK+2G?1=Fr$3PIsj09=YV@N~2Fs*F)D$9uyF=xuGN%N*n zi4Z5Eu@>P>tDvif3UyJyCDNoympa0AbXP8>N}F}7YV|6yY14*jRd6lawFAt)fdiKW zAUS^Z*0OEu_AOj^_wePjcSPJ0fCBdsgi`R}+__N(`)#-rrp`}9Gp5{_hw)>_jVDX4 zO!>0LR8luHJ)HQn=ZT}lEG}vq6#-P7SF>*I%k@gEvSjRCC`&D4f}np zoZjWruMfJ*C{b4Ml!&?Pn9Ya=d!9kZC zhaGy@VM9zXhmLb);WZqJ(HRt?b}L>-7>q?^a-Sz)w8vji;f;qLdpkl#UXS7Z2-HxC z2zcZp_R$zxXx5C>36ZDumzrisT6rZV!9itWf?aya6oX-6mD_F5JX6hA+Hg}0h8Y4k z+=p?>IVU{?xpNnnt450eo^$*848C6FVyjaF)IxrzE% zswqaU`W~T6%Bz!;5OkXb9R3mY)Vuw%nxU+*x;n6|wrUECt_jzMTba_Fxtp47w)rM- z;pnL>#u+0w#5~Sv8O}QcIk(-5)dr?*cGsFfQ+HmRlrGHZ0(maYGgGQ=xT(48uT)le z`tzqi*LTglT_{BEa02mO6aM1LQ- z7DT3c@R6fu?n~32YglpUpTjRT{ZsRcc`m)0F6M&}J`*9tMBxB$97qZZc@yn!r@#fe zZ6C1G#l>_7Ji$50NFvvR%f(kz+B-{z5SG_r+>y3vyS_@yO$35%$UE!QR!1d?n-e3udDH?aCq(teV>WF_l|L;^tLiBRMkgrqSc zU`^nPWINvQ1k%M-CdYSniJdw0^gySCY=UQ`mKwE%Dsjw`Z{-V#^zIlq62=ic!!#iZ zzoo)<#n2Lbd?qxb`Ak$a@@oHxuqT4E#z83h_kTb}W5xr9|YX9`6Im3{+YQfY9IoKpeHa zrHmd5qg3(oe)#&J3CR^gz#(&^hJ)imkhM{V{|x5}1M;4+n6^Qzn3zEH4_AV7^8$5KN&iJMHaDNPBBC3RMEiHZg4 zLwkAvct%#T59*pdtH{GFc`mF_4Txtw3)+B0@L7p{A8Ef4)qys&c+n**=*V)2%N$La zY=tdtV+&IJ>Q}DzO~oE1!P^1f^`w;mE?^Tpfo&!jBFcU6)tE~)_c?d5*$V1Vt2?H8 zg0gG(%m5>sa^8v!##rI)Y}>wwA`w|NiTj)#3ZCGU^)9qeVl`V+3mRhi0yCo+{qf7% z+TYm{S&nqP?+pKEctaNwxXIBhg>N0v#x4xE%JdPHaSat^2rHMT7bfF~p9@_b|5R!j z*6@apDvhlr^(#^4T>SF#kFxFbU${;>u1NkbU6brbCv)1Qja)#3v%KIQhPo3w<}#{Z9Mt*cxWZzN9b-*! zW;E|g&7st;1-#_v!&>aSbiOl)rsc|O(T!QWd)1!-jeP0{%$o#D| zwv9YwZ4>#nQ_}PwJ6%#2R6-6p>6VqTiRG2P6jaDfH4#Y+>*k6%*8G(Avw`CR>e9N_ zw$5&wL;U~dXd@APHejX(YV{@au@{Fk{Hq9eOVuT4CmOSj#c*;U{605JnF4?xv$S}fG>$C;=B-w` zJfF!)UPo@zUGCksd(7qv?w0R8H8-qbN6?*a;%0!|>DD>ozbWrr@_g0yPKea@ZJ~bq zduv-;voL|SSJ!tT%%+Rcz*o8&> zH~;?%Gw!~}_U})g^4{yAgY`W=YD&TKR^Wl=^x`4Q02}=hY&r2dFWbF$+Ia2%Ja4n6 zJTpRx-OOvZZcbOcXr+EEuGe^ip$B%x56aMh96OSMytr@@j;&|69gFzBeL4M@1&jm% z7NDaAAKu-a@F`#dS{m{3-fQ{H^i3Hdg-UTH54$18v{@eq?vsV}+?d&$h0TTJfnVkQ zTL5UDo5f0mO%mv-k}9=D48q@_;gX%iSpBgeB_M&aHJj~mo~ogYU3? z15%n2I^ggP*#kb|s7zp~F@ViEUkP-e7NXzcH5Q4D6$tVg+kKx28r}Gf9{^bvh{692 z`j`bKsUHl|V8#@iop2wk)l{)8oOit!t*DXJ6hQMF;1Vui-4Ws-GN2(Q;z!g+aGhJr z*o#V4peE$p7IxY8(OZjboa4lwtT?-c~>s`Bjk{BG>0pUFgN(udUxIsz{zm z702YobZ}q6Si=0wp2F>6bQMqtj$I=VK%}W0rZwRqMxik}VKNS)6sCj(Vj(nYK_yP3 zh$WUCdfb=s8b6UCD2ig~px^;iQLX`@C77RPQJt}*NL9V!K(L)(094z|O}_n{KKUX7 zrQK6$N6`77@w^@*#N06=qChHQZ4IPB7GkAwK}-mqBi@|QR39~RpqGW=|3UxWCPv=S zb)zwSqc@TwSE%4MoZ`FX)Y(|e>BS*R!lHGUNJ6DeicH--gcZg7oW&>-{lk#N@B6!aloKVPNWDH)WO6}ues$wIe`bhU8~&~+$kks z9%NA>q+k+eVWI~FG$oadPYwl5R6eHXW#3GuAMu!tPr=G`Wu-=XBvo+bM~>PGsuoUF zrAZ1;o}px4aGt>&+*`mU9g<@sOeN3N%3h9$)$N5|#%At$-dtK{I0XOWFe2k%F5pq7 zoNFazU?yjfP@q$i4`e>)#5h;>?c?HQs3W_qgE!M$XArY0r6 z*hCf-c^(^e;H6sP<_7-8p&^=O?&wwC4*?pXr8HxKBISS%r;!F|eice%IwVv+s1`Z+nuT5{muBaN zhNMK@Pl(a}T!pr4H4o`nA7{S+yo5-D=h zDia=Qppsl-Mnsie6Qc$z?1Y)4dS9eYDwbvqmWI+WEEw4EscNZ@SUz2Wl%%tP8QJxk z`_bXZylD`@Sq913{K*Mi0-$bQDmEG#{IpiB>guk#tD&~*yK)-^B5DN+E51VMB;`^G zGDwx`%Cd6gH&*ITonK_yq_@(PS)>f0oL^3z7BG(He3t)67oO&vyd*&g5xUY3k22nG zQroTK>VJx?aS|uI%4>NrrbBvDzNXKuxGa=n-v?$`z}76XYN_R2X=lO{bz#?r#^l}1 zh12BN(>ZLlR%GX}B&kYl*(`@W&=2X=4otQx7tY(B?i3NmtJb0{*KV!X5~jUo;ElHI zYP3(n;*`E#BCPZ)t#zrE+J+}8fCBU^SL6o+RAgnEpV4ASTzcr)=4UT)>ZcACPS(Yp zSmk(OjBB2!w8knQek|aftmKAlu9B?f^6J+zsA6K^7L+R4GHuvFl8e>btl6yT;;b}; zzyQ3^6|kHPjl>adL4KT!pEMM@e$02FX_zK#ndbj(o|0wn+-2f+9gWh-)b0&;U2V{P ztN;jO*UsypPVUzpW}zOWqxEXo{*32-?xyzc!2Ruaj&89EofDm|QRxh5AgKDTFZ+g% zHQ+7rgot${RPK)I+w88XUJ{u$t>Mbyov`Y822p&{rMk`(Kwj?iR;~ggu=FNyK_=>g zZZG!&ulEkziI#1^ByAFv?#-Gn`u5dZqZ~Id z5Ch4AE+`V0tqwBPm4Iso!wVW|ajV`|v22GAaLXAs#9O%SRt?hW&l(iGnXpP1q)Nl9f)e^Iek_^JFI0 zuElzB!qu7mLUKxC@}A7%3$7(tL2mMLtY6M#j)CkQckMLaswrPHl0M^NdQU66a+qF? zroL>ZFcHyaNFg8c2P^W*GV&)s@|n8gTe71(&!920jY=vrRPlxFda_u}Bj>?Q9tZRs z|EWMLVD%O>NoelRbn`f0F#CLSLl6J$?sCch1VD4roE1xHw&d+u&a+3S`ba7ZUG3&0-mg6TcwM$*R+rlG+Cpy zPA7CBjcre3$}P(B5-Tml{$|%)bQNYm0`S}g{l$oGHDK>q1;yxK3${Zk1TQZ&EfR-T zpD8Mx=Tnqb8!GkkR^?ID4=&*JK+iPgTC>S&_F7MhLLc#4laoUu1!8Ftp=q3~#nXZ+ zXhSlm^npPm16_>tvS6!pHunGKs%fKiPIO(u>3GI4)(tZ;kEm*92c}{U2Cdy?KFa}# za$1M>XO}f~U-x-%id&a)+5gAQzh|@Fek@)WT?(L~K4|ZM)G31V#`ujdctI}u$pKl7NBRNkxS2CeTt9S7$4xzGgPW&!E@#sO zB(+4xS%6dLlk#qpKRJ{e_WtRkiJp>jYdLyexqyq|l!tDjd%2?Dae_0NXX`4M@A?8% zdgfd@1xvWJRGkI1xiGxB^KI9W+cqB_J2^v(Adfn>o6|$XC=Ek7VfVRkn=sX`D}j{y zqY7y^`{cXLS+m+`{`Mv3E%p9}AwAl9>- zD~pd{=!PoX=&++l?~I>y)yF#zlN-wW%fh2MIQP!itzTbs;wPH-dCN@;){IN2fv^qNageRy&HSji#;!{6HlckAOji4peE+hEY1|6EWx;HP_Xrz?_U|LAA-)oZ;?H$LM-zT(rr z7$m>t13^I`Bo2*70YJHAHj$=A!&I13r%~y3N;nt--_j^N7MICqa~i!?x7q8Y^Jyg3 zt<`%TA9wcse84#?BD_7iL%Tu5B0(i1KS-xMDXdCM%1bCs%FWI!p-8C9N<7EAJpete z!b!tOD#Qc^S=xo#+gaOOT;5&ZUf^G0-Qi;5V`JpxU}oiJ1YKz9XzJ}~Gr zZ}0!`ZgKK);z-9w^mX=jKzI0f`FZ-qMkIGTRx8U@7lQ*|0=OYq(BMIYZr({PEd4UR9*6dldY1L+A zE3cuxlUlfV0h7?~o4a>ezO~u6W7&ms8d}t-lTIPR6%BvH3D4q{D55Y;c4^XOm30Ls zlUhlUCS#QQ_+?R-IhHxFsl~2V-8!t-*RfsCmR+0nZPu$HSb%_yEV{;Pg%2lQ{H_1G z;D7w`3;i!(-Mo0COF#Hq&hh0B%?4K7RwNR}-8(K`%rSXBLY389{?xuyX7!k3x*C~C z6e`C^Z12b2pZ|Z@0@MmX0{s)vzycFgZ5W^kP_8%#Bb1Q1j3_!!Il|JT$)7F+qVB`# zK%@>P3=7Lnv>6iMtw(9Rk}3oImCZ^zC|%kjsQ#FLSu&~ALmI`l|H zaLL?IWAaG_os@FQCZnX%zbmVplBMucobXF9C-knYzBX(QAP=Wgb4`Mfybe4Q{qhpH zm@wkb!WB!4aiQ=SxehIh*ds|LLW_Km2`b`>D=_kI)R43}B>{zi0S~OyN=*M#yVO!n zH}$krC_e=?Q&K}s;82RfRCU!=R|IsCD^$5+t~5nt^Hw%Ud=oo|P_(g581V#lSjBW2 zb|diseUwLLSDNX*6>%Jt&mjtnb=mOTI%FzSyVdgBQNt}2TyniNH%oMHVKh%x+jZAC zJGX71(0; zkSW`F7%7T@-a;0+w{uI5Qe2eBH92LLhbouUy*Q5aU7BlFajk?$Lt$r%xfr0nno6pwMKGy>W9ly1X*b<>vP6bSP4>Qx5zSvH?sfB7TM%<(^h$H zv|n!fZMO?VQsGo>w)^h7Z&p?xop0lP|mt-tiomY2WFLe0s_KmApcnm!+C&{2W0namzm;Yw4~}F_u!P zLYh>VvfJk^?oi>c9rxGeum66zB{eqs;QRMqMVoA*0X(b1y@0kn=_pSPjENka_<}Te zQBQ)^qafrKCN*Yh?l%rh+z>1#q`y=PbQM8g>C93>Vz|$2y5s+p{a8pp+s#jgD|?~t zTxh8hP49m@bS$+)xEEc(|jW;sd>%X!EryE3!x5o`OV)U(rbJ0f-L`QNlSz@uaUeeBsl%o%Vr&E z5Pw3XdE!H@ZUTZ*`&?!}36M#8I5V09C7CF#X-b10RH4;OC_)$Nl?_&MmF}ddgmP(x zBgm47=v2cAJo?chZf>3IbjU<2YEdIit%?-WT;obV?zoAKW*yIdY~_L1np!s zl-1{J776>2RaDp8Z-@uXp@C*O)t&RKF)stGxdopAZIr7CrbzHF)U=%$%p9@Qaz z_-9$ode(miw2a$$C_~%&R#CdOp>S2GSTW+wtmZXLSJjC|^EFk#+J%nmG@7yW8oaHp zbTf;Els&IH3Q~0Li@IC_S}kkVd4%z;M3pODeX9S>qJFkY`r~F&4}04DWXA|l9o#t! z=c)!0vXZF9+g2+BSt%grYK*{b6))z&*>s;p2l}{oy zEp4qU#mO?_XXY&28BVaK5xJ;$B&F_@?>rCw5u&k%YVu0PFc-?ZL0 zxuNAOe>>|!(EiuI*C4G`t7~2aH^He6VeMaCTevIQk*{M+aJ%5b9`&*}z66tudV?g% zywz94{veu@hnHs39~susa5633@yR%nTM-a;h=S zM93WYMj?pBTXJM^q-35o8MmMTCJFU>Q!M|R7`ZL(ZW?4eS*Y?%kPNmC_FPmdR1Zt>5_UVbEZe3BoOLp|!l_FTc*kiC*-ln#<^e zzJ_-Pqp+PVjiDP~YlsfsvA)79=$U5c(iWaXh6gx6Kc{-hs%CYQH4LPT>G3y)ruCsA z4PY;eIn2A(wK-Oo4Hu9Wm!l@O6k+-j=gjiWxWby~$Xo0?Gc3ujj&ikIo$6DMB9bLF z^Ob9D>;2w(xd{q)qs86p7B4hrlS~}6-Tep6(Q2d14w`77N4|H9snnNnHL9=u?^b8q z$#yl9mA?({bMrd13D32{1ALU`l?VUL`Od1EU!nkvN7XJrrnfltaPJ~eT;CJp_o)Hy zZ-4(=;NtU>N9Z-{EFZkw3?FyG&5iCy-yDX<;0jK}J8_XO^)OkGH>acXQI5Nob1xrf z(`6L&2pzlR0H?aj*=uiZUh6$KMYvQp)$oUh-Qi;gJK53ZxOM{_>h9#66uO`Qjo+om z%g&?Iv2NJ46KCHhw>HZ6+;1jl!Vq|m(O+cV^(@gm=jqmYAC=`gqOGsl*YEU3ye{5Hao@~OUZdCDom*MU^$TJQCq&^~A?I|5v|+QT)o zJ3wZ*9@v_{ck?THTb>vpK3S@d6GgAPnmJKLFG|o`Ny(^FGE) z!N(K7QBo)CxxjoAi1dpPxuv!k!~J6il`uUz8^gPb2Tg#Q(?P=Q zQ$iH{J{zhzI%Gi=%tQY?tPL1+DJ+CG8f>jC+!rge!Mf_1G12euqtgt{dxDbWBIc3M)!9u%BBMW5fsmoW}tC#O))+Qj9`; z+(%QKp)kZE%EQJw(#BVmzP#@{oK9RRPnt0< z5xXo10Z)%aPWkN2`K-^LxRc3L&%8QM^kfrj)X&Vq&h{j8c+-sF`x@e_)Jdw{7?@CO%Cl#u3%2w2{&VVP77s~38m2KfVS%d zs2H^*8l6!atWkeN#9EsI>S|BJIwg z0@D8lwNo#pLnO5)CUqsI`p2b0(Djhj8IvfRVVed^^3p~!_~FuRcfu)H^r~BqEU1>E(Gn? zJWbU)CDz!mDwrsiWv#GStteWp)iOXX5t~+PjaPXkRRQfpU@cbVgw%T#*KpBSfBjd0 z4OoFSp(GU+P0gxxE!69axqecMd3{)jMJxP@R)(!U6Af2X#aM7{P<*9F=@T;5+>8JH zY*u!416#ONY|S!9J)wwgS(laAiak7m70XrKSDWS6nkU0N+oS*MLzhDAaa3&o$*SgZZkeYGnlh16I5CP6(ZqTNETgW9LfR(W7r zw0&5lm076;){V^Bi|tsP-B}Z*+y4VMIi;jEw5XX2HlaPru%)-LH8HZiG6Jnp98Fur zwbHUJ+`DaCtgTv(t=WxzTNsL4uN$Nnzz-4t(gfX*zg5i@^wqcY}bHgUgA|>6=q-vmf#Pz2Fx_U>D}PirC%Gi zU)sfA9sWNL#-#o|H1Or!NC@DB0IBKK;Q~fs1zuw0ZQ&$#Vkmau7p}t?&P;E;;>g9* z3s%*9-QfxT+u%hw`vODpRaY8xGt@ldWHj0oW?#~MS(L?6CN|tCuH*kH)?ZnWVXvfN z>us6m#o<2=WI)d0JNDuiE;ON~V*Paw7fmj)|5YCe9pS)=9Uvy$_Wg-EUgb@;Wn7kGR5oPiibH$c zjbA8bQXb?R4pCq3Wif78ce!OZey33jV2;~NWwyRvsODvc+`7f)9WLft7$`G}y>G?f zKKAAcOs zXi#n)b8cyfj%t`rD4CXMehL#_ZRLz+Q78Q|7C<_%p2KbSWV%giPOj&wCh3#L9dJ%* zz+~x_hHHe*-m?B)v(Ds0T~j5tY4>#J^2mCE@Fz)5M4(JF@You=MJ#A}) zcIv4HrKe78slMzIwd}@TX5yWYc=cmwQEmLi^wrkm5 z>)L_r$d>EsE$7L`Z9mp*U%qMxVPU==Zf0(kcMNRN9&P`DK;zOb?Xy8^#D;Bvj#^#5 z*w$uj>|Sk1b8AqP?8@Hlgbwdwc5UpwX;7WlcgblN+3DyM2m=U#(?)GyNNwNll>GJX zY53^a*iO0CQ;^~Ad(7C}9&p{J?YVZ{%+Bs!o;c#340xt*w$E5F| zo@nj1X2YIn&ebvv$A-g>Y4Vz)b?f9*8CawD(YC%7H`ntT@9aTaU{H7PI(|;) z6!a)^@1I%prP%Zu=Wiz&>$9c`Zq{s}r1b9o24k-C7~gedp7EoWxtd;KX%&;nLs(Q7 ziX)#Z$VGB*h)p49bzCRwveaUsvUQ#mQr^bzOy_l7hw81!^IzX<;?{CGl7ID!H2LL3JZ2d2Ip=mR=65uIde6FfM?ZR~xAT|x_37T_ zfM(Z+9Fno5|?2dd+UTy@P-}o>BSqBX&B*<`~!-fzaN{mQR zAV7=$GGg4QvE#*xAU`rg=B(t%Xeh0z#HNL1wwF@Cr7MGG%uIAR)7)goQ{B6Fh3bv^ zcT%Lsqeh8MOd7H&$EQf6LY?ZA;Yg`inQG0-b?ZfwV2g&el=7O{YcRv9UCZ{{8=D1o z=A7HJZcle~3Rw^ukLs_gegStC{4^oqeTFCEON>}?W5A3ZOFlf9a%IbbGh^QD)fiFF zVm(DdN%ri@)2LmhPRpeY+_(Z@&#kG`u5EL8;n|Dab!_O%or4GT-HWqv;m8O7UamYi z^W@M$OMgB+6mjaI%bNdYiMD&&@8H7|FxLgP+}XF&`^0-3xO>gf;aj~v`EmXB_w(c5 z|2{td|MK}Kpmo^Mr(HCwC8%0B3#LY1dFIuI-g)X(_)}g37Fg7L8TLogRRIDhVuvA? z*c^!+8E|24E6!m6h#to1oQ4M0c;I#lil^g_S3C$8g>+RYq#3#KwiJg-71!5@Ode_C zlSW3V-<0+tMCFxDW(j34Dqd*7mtP(rrkE;r8NiELritQLG(LAkX*lL6XM-^L_@jgq z4yh*`a&-A5ah0_B$&yK3!X%6~BC4j0i*9*hb%hFwotZy@X@CM_V(Nebn;uXpqmn`@ zYN(_pdgG~a=1Bi1kF1Kr4xW3)`bLqHDVdm}jyh?jl=n1x9Isdg>#MNDf@&;Bl*Uo! z0AXgzsRI#23$2(K9y_eHrDlsNG^%zxCpSRSWK*89miq;*8xcCHL3s?7ZGQ{!b#1Qm z*1M*?>)vY=i!HJ!CbSbo`z28N=8J7_+P-@68Ib&^XvhL*8^~}l+Y^I=EXU(S1Juj>_jt(;eam3#h z9c0pUgOUHS#{`*O_0?4OU8UcA?=86A`6vMP;Vkm&V%UTa{%G20tGx=)ZLcbHk9Lw< zx65ex}IHg|8{B`SinTe*h+rZtKZ6>?)+9|uZ>{km!nqK(Z2&ftMJCR zf2)n5*KWGH@~iT_?fBR4U%#+h-~a3HSVI106uAN#Fgp#ST={q>9wE_2eL|YU9ESHf z_cd*P_k&FHRwlg;axjAfgpdCa_dgOcZhj~PVfa>Rv3Dhy%@hj_oy(e4&z!eI#kxWspKtqCEFA`7R;HU*N;atw^249P)5+1SuF zN-SRSKu52p9nX6nOyl1)_(3%iQGZdf9`;5!N5ttyib8zc0U0K~K9T%M9nXKakBgspyjM9&}`y(k~Xoa0jtczU~ zLm^qYj;~FHn8lRj^3JHuZWb|=GRoN{f5}90`fm;b#HJ%D$xdXR%8CzjSQfppN-qCG zbB5HcrY1#Zq^Z?1m&8g)$z}=Af9}$ok{o9pgSpU#b`qfnJI}WEUf} z(RzN-n&5oI3bUCfBvog@MoJ#hN+aM zM?cy*eio#r5G7ebgQ5p9Hnpi^jA&A&iqr%sl%ZF(qxGiBRbSz>oyp{B&~BnpLJD=7 z`E&zFnd7ccE>flMp`cUenup>owW@Gk>4~x`)0Z+-f;P?TO~LxrDjHUvrew?c?rG0x zrgeRQ#T81sN?BXhvaZC-tY$Y0fxA}Lvv^gLr~K+rbC!0Tg8l4Z2|H0JAXfjZ(#WV> z%=#L%KK2YTm8>E|J0KrI^r>{6D-Wrf+Q1sut;QXvOqILbnL<&o#bvE@D|*;c?hZVt ztc_!D%ZA)8L!iTjl0iX&%V9xvv*?W~H;K#M==y`Nz=UsnFF8Ba+IMG#ovt!r`wgNk zHmGLhE^hbf3f{nSA#DQUeU%Da^kUY1?Tzpxk;^8Rn)ZLB4W^3@%v*$k6FZ7-0~Xw*w0RNCPuqxtuo6}u@3Tv4{d8!e;C(l zbMrENjXq0XOxPMCkfgiq>{wz2(4~H*>9|erGZ%Kut|K#;vJKPcuDZ)=UN@U}ZQ>nS z+T8g>wsY3nH8jfgZZH41M^5R2lUof4H{nEkcTC$f8;0iU$Mbz1PAwj|D*hAhF1nm1$Dn8)$H z@uw|Yahvork9&h?_p7C}6yDtdf>dY^9@!kgbQ9Ms}ar<7LRd52~ zOKN4(U|A@@4<8=HElXnd4wG@z*|N_@w^) z#gwJUW(lBJ>Yd6dS1IK#p73e)uMg25`B5bdV!XF1~`4F z$AFwLf7RzY_2quB2Y>mue*))bXy+PfS9mLTf4CQdyQF@mHggV0ft}Y*(?NR>sCVdu zf=G3N`j>$O=R$Foa=@cRAh=_RM|D0pCCi3m6!(KQw}2IBc^YAT7bZsp2!lJwdl-0F zXSabkxH%tae+hPk9IZJ5_kPS|Zwm|xx(TPOb)hjQ3yy2oMh(tuNiW$UMaeW-;f z=yFO3Wd0V07FdXENOU!L8VFZ~b)$op=pKKzhXFNldB|CI$cLv0iIsRFvU6>|l!-4m zi?Il4o+xQ027b>(ilxYiPuPi|_K5u?f=V)pb7fS-l!k;@jB8jL7osWHwk%q=R?-MR z(5QyJ_<3z8c)+KK7;|{2cx;=1hs%d`FV>2B2#dNHj~zk)F4Zi~VvX7OP=m;cW;Tsc zID^>m7UZaOI1^&VxR1Hgg$YP=yNHnd=#OL(HnwPw*;p#u_>2b$kznZg8I4O?ivy+F^Cn{o;3MPGJbdvA5k|bG{ z=ZII=SSk6aI58O}NO_b%caxv!i`}7>b&)au)`@;NlndE@a9NWdQDsy~CT+QvOGtmM zQx$$s*6Vu1 zl7eZKtOZZPsCHM#n6&wpz`>XB_Lr;ajR#b0km3T034vv3ZNrI}#pyq(`6W`xjg9G= zR~d?krA3oTkY~7*ow<4@8C*k1mYo0jXo|^b$7!6x8J^-9p4g+9i>aGR>72+Uiqkm` z+L)KSNixjHij+8?>dBm@LXY&gZ{_))`39hhW}X5{pyQ;NJj0E(`Iz8Xc4GBY@2OVA zsC_*cZI~H^*?FJcSs2{u9KorCr&*x%2cq#6fuaeXCu)5`mYPW*j|{3U?irzL5T7r4 zmi1zSI)sw=*_;cCK!s_bKpLbjD5B>?q(&;F!Bm*~$(!s6km0yHGHQ6IH;7WAiatn! zISO1F+M!?BlK7FFhKZy{I;I9nriV$QXd0p>N=GajjZONGpqB(udPC*MqvAzSUU_B6 zCYT$VQo#A6Vj8GsDx~9ShJycksDqlQ64;i@`KF%8k)bz!qDPq%>VE2HrCge+U>Oph z*>h?7sb`9!h&rl)=&30hWT={|stT&1%BoyxkG5%@{Y*G$1vr!^Xw8>4{sm!Gn_ zKiHXouhM#m-B7It*GjnBrCutM4T-5m*lwM=svsM(t~#;`OQ!NA zvM39xD%-B<+OW*3r1SsEo+S2U)>VHH`>VD2h0%D8CF`<1d#)|$vqJl`glec}XtYS1 zv`)mRWEZhc8=Vl!UA$9&zKXHm3Q6J`GbzimE8De1+q2*~nqE7!?n<^_o2-Nht#lcq zX~j%0p>h2BuW}1%X=%20d$xEhrZL&FdE2+edbEHWxChHz&6=Lw$7}bx4p~HBbXb-g z3$8Eov0FR2lv}x$JGOj#w};bll6$$B>$z%buAeKOf~&N~x~O8(v{xyeHpC75WwUBK zw|&}_Jgd2Wd%Lz9q_MWQVVb+R`@5wZyuv%YF=Z+K_Owr$mFnYDR4co)TQvfUxm+u{ znmf9g`@0K;jl2Krb+~K2-uu02YPy6=yyQz}`qnIm8&7h&ISkUi+yJ++tF6u}KOc*| z_UpU&OQh6$ztfw)f?B=+9KZt1xzg&OrerQYl?057pI6(!d=LOnK(N2Ppc|-V>%S5F zy?I-?%UZ!2yulodv;iu>AY8uiWxP12yy+2qZb*;B7`F;Kw|DoM{5!!EJi{?;q~c4% zrdh-38p1r>!#upZl=GxeOReTG9tqqHuq&UC+lLU`zZ^%JUbnwcyuDED#5k1Z5sa|ri^znm z$yjW``1{8#xX7Oj#vPo>xTs;K+{&)}sxK0)mXN}aECrPu$1JSGZWF9No5&yh%ATyp z!92`{%*m%KqQIQY!&}V%TYJj<%+S0z$|l+|Ir{FllUE-*U|>+@toK%gI&8#2LCVE`%FMjZ0WHq{oX!M& z(8Qe3uU60u-Oz`pzHV&I3S6n#tg(mExlfG8-we>ly3Z#&!2%u99xc7B?9k6V(ErKK zCcVy5gjM8>n!8RP#9F5cq z&DL%hza)%Bt+S zV%*d^9oY)a*p$87Mo%q*APUk4i2q>Y~E4c6(6-t4{J?oE;J-QMkOs<-XSf~%Ii zD>h<7#b+wb%&pOZtIC6$-G@5ctJ>Gy9p1H#-PDcSXKmf!9pRPz-^^Xb1b*Rz?D`2+<#05AY700000 z00aO600#fSfN)4G8jr}Na>;BupU|juiW~^7SgLl*?Rvl9a8*ne8nOe!IdD*;kLa^J zO@C*ZR=IlL?)(3JSb7c$goX|gh>DAejRzAMksFeel$Mg0m6)5FotK=So}r_lq@ty$ zrx|NxcV(}zv9hzYwYImouyk{*W4TnWzrw?{c2~hvZdq%7$j5?dd~U4JzR=PNfCt&y z+uPTL+~MQohlUEJp6a9P?d?j)_!i^jyP8@DShsTg3OP)-*vgON| zGi%<=xijQ!00YLcN;c<8(lYzO33#)0Y0?Lq+}#`Vwd==v zmlR+gUyhu)^5@KxOLq>PI`a$Ir!UuTy}I}B$+2s=pkZ;M3(LiuXQa3r-@JK8a^FsW zzI+bjpWE-wzd!x#&DA#_1pw|B;DG`rsGx!VEr?))4l?-Qb_GTlAb}WO;Eg!kyl4NI zZKYu&7&nk4(&0&UvA39DCd%buI1`jufs8cHXk(2s!iWftH^$gwjE!{A!V5zZS)`Fj zCaENmOE&4`lT02dWt3DxS*4X&PPt?S-k1~;A78@9p_pXewilUWrdg(o7?sqLg#$_` z=Y$Z}X(yd`QrAlW;PvSzcj$cwUYrmLYG{DvDQ3=e*a2D2hL4=64PKPWcfkaNHmE^< zhju#WaTUUuC!V8v`sb&mrmE_o1j6@+MmU`bh$3y6L+OZ!bvWW&hwb>Ik3jw!fv_q1 zx?ZrzF0w(9T3*@gv(82f?X=QXOYODRRykw~5d>l;FSo)Mu6wYe7UsCOrtAN0S|7bh zDys0BD(|ZELgxy-re+vwzM-O8FLdf1n;3zcBI{nHlB#>Fe2Hdi;CLGRyQ;(4^(!i) z@xEK%ycB0lvBnfSj3=aOzWVK%lom$vizTb5vX10*1FWz#^XTX{E(@G&A{k)obGAVL zEOgLA|J=a@E}En?y1IJE^2%)9#x&I8ifh|$AcuVM#`t!f^{RVHEVg$1T0AzX0cN1{ zIXE9OW72BW709G?WAng{3Ye{_zh`f)_t&a=e5&ApXI%J&*;U<)xs0DXD~XCfp6iP- z#_T}ciPfw*vOi{n*xKe*z%0?E7hU@4rl%g|(d(Wp4PbT0{<>YQDIEW~?5*2&IN@H4 z-M6Rr?#}P1fRd*-@9EWfbKTg?*y7xn<1W1)&L1E6?|xTk{et>#530x$`<*Ao$j3`Q zc2sKvV!N}G+;;j-w=FP-c(X_1=J?xubIy+L?5pYk3>xVKNWcIxi*?ZB)zZ)ezm}=5 zc5R7Y`Wh%btb}AKh|7@pwuU{90d9O@>z(gdRWZJGkR<4M-naO&JQNY4eK3q+AArOu z5jNj>apYgRnQBNWHBM z7mT6v#vvjT+3t{otRWp2`Mo$2Do_G~uW!|X{iaOUfthRU?QHXX56w0a=yN*2BL zC~9WySQ zj>7DrIh$$BVwR_xxWwE%;fc+K%4~lbwP;G|Y%`-B-HUMbK5HkFG{jaEV7>85u2vS|#A+SRi9%dSe3SKu6` zOIJ$K`XsZW5{sioSv5~Fw#lAsdz~ReN>^&eH4SK8YhL$?R&J_P0)DkA1ppB@KoSQc zz-diH)%eN51og2ziz4SDOW7X1laORO#In8^)q!^Qv)dZdRNdOkctUiDV^rvA>$(+* zP?E2D{i}B{D%-Tml$e1_6x`Z3$hrDdwX*B2q&iyLu^tn)xZT}tHyT*t-ZVp4Jy9ne zx=>QGmSjR5mQf|EPsrhlevQ~HXvgc>^0r{KRbBs}JGlzXc5d~mCe^9}MP=N*mKAVk zMeg97t4t627MJyVpwt%$E)?3_`?KlV>Uyn)S-%)yR2Lh z6;JF%s5*7UDt^{^3C!3rd$qkTRv>%tj8%prC9F+mu%JO`oFGTJ$AB}{dpz7flG6FQ zlsoVq%E_ezkVFp+Ms%h(jYkG)06!cswATL|Enhckn9FqaGWEW!tPrEQW@=KADpy? zTHGjq^0PO}@|kWD&r;*3yya`Ig%=XLuEzH&HC_(J^eO}Y47e;m@PUEDveNa|Grbc| zZ1+Z_v>&%|G^qSqi))(4Dfi$VGl~E0TV1>B_O!9|dgMOAO2Do<4fn-m`#mX(I_uz+ zFqGv)(;%)qWHYCl!Jk$vF!wpwz)l&7?Wyy8tB^#Qiv&F@5OkxLj_5)MXgzP~IL9G; z!+Tdc8Tq`+NsD#43Z6TlppdXtM&?r*P7#6XyE zoxatpWQLl1(`flLRaUzpM!twgn$%%Uf-tlOIIpL#Yw7Fy10H}p^|zlb?t5SQp+)+_ z;ay-zF>kuXC$`wJ=ELF!I_u7+{kX|Wp#3pT`8=m$)RQcGtL;Jd6H$Behhu_69_dDP z6UR)g5^LEuO|j;8g3&(YCqe&^WoCu9a}Nkj&j&M%XMK(rS{{&n+DCLn7kRQYd90Ow zmxo$;rh+SoJI0lOqi1f%)hWqzceb~7_H`+#mog#OE01<;tEWV5L~;T+fEKr0{xy67=rM~irUwX^jMFpGb9LP zLBwZT+LjV7sBNMYjl@WRDaVKNH;q9?9?WQbifD*ESYTh*iHwML&)8p+sB)HgkkRsoT)j%3JLL%Xz7Iw$&i<2m=QT-8uowDbdV_rdVqM8 zpI3Lj!juX4a$07RBPoRsxIF8DYJrzPaJiC&w_%+bhIr|UHR%Bnc4dCqih&6&tmAvB zg>bV-SkpC>wa8@w7@4JMdefZz3Gk!YL6}On<4mV z4mzBGsi6N7T90$4k3@8fipiYA5p|b&NNcd3UKyfTCz`ZJgCi-GW72<|$!(kjEY@U{ z>B)-(S!Ea^h;0yqhOrIYgCU$Ziu36{vWbks*E+WIouPQ7wv<#el#a$ClQUV9W(bd0 zmVF+WmtTZlR=99L*?Fxsht9d2mr0^CTAe=qdGcR4v-#>D3M_%Cg-S`pJQDoc^DpNq`0|V4vAH>8D0}wrPC6n zBq*g3TAbS#e#luaEU~2>$|f6{p}-WKc6xF$$bn1S%7ozIv zhsPyrS_w{~I;vCie=lRLwq!r!%8&daH$3TdaDgb&9795})y!cPI)e#;9$(iLByKh?eGdRTg2| z(W`<+PK}v0mdT9+TR4q!YM95JGuMc4nOzI}tcB`v{duHxDWHdUu^$^}3@V&{nSK9M z+O4RQk4|T7SZl6mPcjLz zovMnN+Og=y0js#F30k|XyK5bA4|>sMg-M@S$xx+=h*I~m(zdsI7_$f{gPIhKmHT#t z6|0&km4#b}f;6qTw~HQ`9h-t5yQeXZ+XzNGmXPaQ`jVe*=d{@o2{QtirgD&*a2!%TB%m5wri`UMA^Q!2&Vt9Ij+IF zm7?U%>OE8MZsBIfx zx&m3Hq_9tPj@p#9%Q(UkYmR;gzvi&I?AW?P@|OY(oFy!red(7-99jTOp;27LU}R8- z#ZERXb}7r5aY(#i{2U|NR~+S@^~D`^vp3U9u^CFbK+G73Kro0aLO0REa5s0e6T^U| z0nf+24SacAvX&sVweowlz=yi9^RR17ELP;5iZY==gN9}+zqWCW#g3=I z-(s#cT(bKa!R@=Rc1yE#dmtE$%6~I%O&CLiGOR2~xPohEgh9%DU@8A+TE})=qJk=m zy~xMn*P|S2pF%o?JG_W%`ganf$fXO!YGXwpX3WrJ#8!;NXUMU8DW%)nm+;t^A7IT% z%(3t&&ewd-3XEYernz<4#ikljE&H#%tig9Y#!0!nGc~hy61shBb_$tFgnNj>8^U^a z%Hn%VAv(wL@{zA&TP3ssC`igR+Sb1;{rD7NY?>m9 zyg{wJjnq=Gx*ScXpR%ZMNw>}!x~Pzb6benIu_zx`?Ui`P%+^^##v{?7yUYX%O`55{ zOpTeSYu0mvGae0-7+6j8>j3^+GyxosbRC?rE1XsaoDdq*Ej_!IDo|MZRf=88>f9C% zyl3p^({td9|RrM;_cKSm1oywog%t-y`0cO7P>-hcRv}*{3zB5%LtwG zkcK*VKHNM(%+@86%ngaTy=~5cjk{-v(o8HKm)y-(7Q6r1EZlW1j|D#A_~@B69o@*5 z8#+y!hS}3%YQ}sDtj9|PrzzBNg1zTNy`R0}1C80_{YcvV-VY?tAPOjiupEYb+j*0a z=p?S5xkj7FGZ(F0>I=v0wb3z?$bs-gxowJSL$98LE0A1y7I@x-N-jrC`k!uu?J-*sq)p{;QNta$M9^QX$ zaB?o=lq)ELzTT#NH_ximSPh?=ODUe+K4oTnnN!F%Y<%_i*@GRCY@Avo`=#tz;34th|@1(Zhj292$k7b7}AYgWIk-4l%DB@ zl`w>tzZS>|%%^{ial1w`wz1Ba!HL(ZPTZD!xY}#UoQ%Lit4c*xAYADK??m;`$5WHCLMe?k+B9_&O zWJJWZ-KZ|KN)Q{(JOb*G@U{7gv9``@u^#5FPV4BF<*bh7LEqK)Uh6#{*Bt->+GyC- zy*RnC?7w~-ZxWO?``y{C^%GAMch27FsvZASpYdZ~_8Bj;A%C+Ue>YYQ@f%F-{4w(V zc0Fy^?RO@gy&dG0p3y%#uW)>^NQFDnx_nHI^ST|))r#8$u8LJW+^&A*+iPW&Z0|%{ z`47si5_*F68j`Hd(=kWPqWbMHX2PjV`jP$K3fLUS-t`m@mKy*1upj$%mFV~|FXwV; z9bejgn#+R1(L`7XtdHUGS#vW|*7=0r+@3qvk_#NN@jc>_P zioy+QfCCJVuHNOU-S4Y@wo8xBBGVOhR?l9Uui011l#fHzOE5>70xpCnOz9Nl{N#6*MI? zOkGbu>lW=cI$CmdcXc~Hc!6~{G>A1gJU%!yE{dCvkDD)~r$CT7$)G2+HOP}l)fGBRXIlP6JQe#N*gh1!jx01RIo;&%1TKVRI5^t5<$?x8@Lu9 ze!0cf!Gp45b(JyWU@e=6Ys;oVsm83Bw7z(c!F9WsapMl4NHV22nX=E36)|JJG?lZ+ z%$gSbCDVw0Yz1w!8EU{nGp&ip>oRA#zYDACEBmNjv6{VlG=5ufih(6d z#x|K>Vu}4CY?;IU8cZk&s>|j-yy~h8KVZ-~N5TjxoKT(IRLV|6=!y`?!-2lrurfo2 z_|U}EmV0fZ5KEp#ieHI(Z?Twrgbt`>Z0@R_>)dS<}C2*k!P>k-0p0p+y2ihes{wfCrW>bt8?# zmH0(1lA}$GNkOd1MvHz;`AXPmCDP@WVUC%Ll>w1b#v+*%QdgCm?X@L0>?BdLpAgg- zwq+w0cDy*5t~Oh5u_aR2F}-c4;ug(nGdwt%t|i@9&{f(12LuKNFMN?Xn^1ohgr=~- zsDazi!Pq`#;JY2IpkXi}W|%}IKE#1YN8VH8L8*QNwZq;ofA5FRi0@c zEw)^pZn;*^LFX)UEq#L(XV-iUwN=Qmka&z;gXZv^q90Nq<2ODI3Om2YnqyiysY+A4 zGUfX=R;T~We2P<&pkmK?uDkwZf(^7r)9VcEqf1`@&`#Swz6x6qP`RHexx-Fa}_pXbinPqZ{*TR4dHCdZaA<@8e5nSunxPDhH$3K1?KTcj4=pZ9E%`a{0FfA1_0A<8 z=}ixnjM%^{Ux__l^k;~IA|hPehK*eAD=!&4B42b_Ab{EHFwe-3gS6sBGDydFro@Xq z8JYsD(1{Vc7RaH3F8eU`u{BP((l7 zg_gZEjG)f9OWKvsh{RN1_@20+D{3!`$#hEw_Qs;Y`R!>+(uWukNz>b$v45je+3Wva z>(TZJEC~=!Y;T3Do%9LH4?3VM2yI&D z>dHwX#8n74bkjrUVM%h`a08AiEM@Bj4e&YkR_yiV+xX(Ada*^mdUEMYxjX+V816S# zMAZyc>3PO|XqB9*4eDQ+1I~(wMz*v~s)LnFCfiOJz)=;+R4FXF)Y8O`b0rDYhm*wKv9@izk-EBE{ugNAaRR<;qrSNRT~n? z*N{X(_M_C$Pp;#O zBjkY#m?Xw6n87ATsyvUaXK6n>?uCb{Q*%TVRuv)T!~G{Bi08=bx*1CpFZy*OS_`84 z?yiXjsF{wHo<}ELhgs2a=c{MebKg7PQa8oG`ea4#9ahzY^5{t4Thb&OXqc0=OTHF` z*(yhT;u+6!nyuM(6CS^Y54%|6I_#=D2)+~=NI?o#fc@)VU+fqQN(23la`CHX<>fcO z%SF{&9Hk~@jt{NSwCxnFM0ojS4BhbL6x~uT8k0YlZng#>dDhC3qKncDkSz3x06(Gi z0_#nv4jB3b>yE7Uz=w$%i7=!=y>btImVsXu1o#%E)zbe?9rBCI$jCc(1Mc|i@Zb)O z#_u#N=q1|f6|C?2vhN19Z~Js`2YE0B%17)PEZUT7*Aj>D4DSe$a4L9B=3)-!0`F#M z?(szLxpc0*zK;*A>u;K3c`VMLM(@SiDr`VWyX4Pzx{K*>ES0W8-gFEBp>Ez>iFz9F zO$LS=I*OQV>0gx0+U{lRGBBm4h)}lS$Oz+4gfE#$u=vWz`08Ne-p}%~!x6Ct74(o#+WfYxCIVfi2;|=vN4Hl%46{r#43h)tx%u%xLd#J&qym1ft0KQI59RK5q29fNZ zD2fV^iqz3D)(&*~;h*fJVv>&{7jdT~5zL;D+R6yrTv8G^F$X~r2LsX{!LkQa(MIa7 zDwzZfVlhbCE-l%z7Kw%Z*6$W2awSYK3W5L5OX!c1Z0luGC?rL){~Arj#0|W-0$mQ! z-Jpl_^lbpM#@-sxDK3gJw1)z7k`i$TzPb+gfReEW1>rhP8|vaH7lR$YN%`LKG>t_C z8L~9z&YINEEFp4(Fic!t!4w1XHgQu2AAlfd>)HykXJUuhl5jPLGat(g+N?5-gz<}f z&PuMJ>mC2>$KUtGfjO{o5r^7bIo9yb|NUayLG{1 zlYS#QqVLYc4{=b89&*C}w2)0Uudak3PYVin`YB=p)I<3ur`Bx@i}C0}Z#xr^>5Pji z^adqUay{NdJPi{S%yR)B(CXj?-^i{qzY!;$MX>_pNl&V=LW;_!luEIy{$}d!2;wPg z6ytc~%T9~(TCo*drbR;{-zNWHLH&_Tc~BH}Qxzonaw4G@_2u>KJwypfof4U?_hnW7elGl&mx1bBV@|iVo4G^s@`2vi=~Cw0LSd zDQ}C|UPKi@XeRekVVS!SjqZ3r(~`>j_xbZ9e9ej576Hoc6}*QZWnm(qJ;t#IvG0)l|)^ z15n_iI^ZY$@KqkxJsbbY_MoUIE7pocb6}`P5k*VH{4->AHK9TjHgz^~s7g#t@e{GH zD`6HlA2eFYG7F})UQq>Sm6Kalu4m859V!h^iDEODaGu(4P(gyrJXinxYXFRHY!;JU zdF%*?4k|Yz*J=$?X^c>%#^+AbMte21x}tZP&R`WSuo4zB9fM){gf8CZO5mnhvIGiv zQupM>5aFU@J(jcDkxqk=aI@5B<8ftO@kB?0YjjW)BR6JwaAr|vH7b`{nQB-GwB<+<8)-BlHFEQDwx8XWNkd$;Ylhq4r0^j@F!_~_hIzUP~oKAxQ0_p5^L+vYH^n$ z1CuYQ6NgyS5T5^0B}svJ)evo831N^&mL${aa)EiBtUV2K>wZw+py*XyIByx{-)^-9 zU=4z3H{6gfY~!JOtF~9|%rfV#5zuSuV@JwwnA z?Q2SjuS#Kb)+VK7xn$W4rTCJF8!<6(`LLkU0H`x~>uFz&d7M|jFlhsI;X~^Pu zS;cx;mihm+a>eCy75Ui&ctlUvhUEB<1T9$n2&hnp+o*||lP-~!Rsjswph`i8Ue~h( zm6*Nw5pQ%k_cdDR0Fi4K=qBybO874w6W{Q;fz=iq7}gu;D;$UQNYkh62%3g#H5oqe zhI6=-bCrS_F*OtUvlwyX2GoTsdi;|3u9^gnvDpHbEBwMZo%O4V52BW__*(*JDvc*w z>(qXpl7Ue=kg?TAo*8AIAf;0`rk&&fU{tuiz)GJ_AqK(@#5j(r#c*2Jjvb*3O=%sP zat1%QkN2)MYI^p)Mg{7OK|+~PZk3^V*hPbps&T4kxAZfJRgE#nkUa+u>GHo=BB-Npj*eDlto4v0C~6L; zakz@G^EWG$8H8$KAro7lE!Z#H7;K+rBf|KNt~jVGXA4C;fmK_MKV+{q+jYm8Qk43r z$41ir)BdFRG|!6d`c+=JqdQWFb%(e0SC8&eDCTOfGzCFxfCIbDoVhGEnzGla3>aGhET)+8dvr;< zTrIn^RTqR(+L&2df`(b7tMh<_QEI^Wk4?li|C^Uzl!{Lmar+fDavGv-)~N9Sr|bVA zsTY%}#Ugf~I=#*Nyl;0uzj-pO+Wv^gwmsL613YV+uD0{pY$JfIms*bBWEP@}O^#JO|l`Cw%4kVgNN}v4(7J` z0NPYbMhU_trkhWs&W|9RFHRF<+k}1Rj(s@7SA1cP5un^prV+`k4_l1qifa~Ax$RlG zP5nIY8P&_iKZ4Rexo3$FeWY5rP!w+Km>kM!$vz{KPoP``2PSPj$5IA2(;ff%ehC@F zN19lV_5}$~(kn>JF~|%QNCWKrm_e4k)0}5`I(KhZ@~I#ls``USqel_fKy~Jk)L1t9vfbFZ@xZ9foc$ zwuyO;Df{!jGwM!#)v2dzU-B9Wi+kE8FN$H!jXZrOFiC0n<8SzSX{pzJfym}%ZzDy# z(K!j*yL=bW53XQ3~_@w^h`ToP=)o$1`%eyO<6` zf4xnU(IAc8GF-pFnydQ^of}+W-B8kNe5^4Zhm>0m%SL;^Zmeb<4WifkW*7)47a>J* z;b=U_OA_h%K%>v7R7q`Nv)U~e+o>GFhVgh{Fq_F@^?1#8zvKU8^ni|*!}EB3-hQtC zrG+W*xfwl3ASxy;stq{8k~S8_ z*22=t;>LQivQ2QV`c7m_9GtGHs`m1>3Ogu!#y$TAw;?xUTPIiF33i{2O+$N!6Tam> z_O1yk=tkZhILs;plN*?8Q%io+KO++p69X?Ez-saO(IZxk5W_(Y9lCL7Z6COA_aI_? zcyJp>iFnW;+(r$UJ~^ulzWJuIV!mR-0A`3eQkeoxF*n=<^GxAQoEbWFQL;qCkVCqT zDk_T3RuQ5@lrn0|37@?Xsdg^5 zFi4j&jWpL{Gj$hvf?-7|5aDi1>zoQeLG0MEX7EDJ8~JeKvEttT0&|L8vS#)g+dIh( zoaN4>`Bau`a#|Qab^s0T7ojy>d~*&&)^popJ6Mt~j!9~ppD}Q<35lvB`roE{EDdD- zHSy#Dn{n$p-g}zuS<%$p@HKOs;P0t2PIC4 zNY)5clO^@kfQeMa0TWd*ag|md2<6085Pq;9SQqkVo-4Is*WqsL@#Udkx`|TQcFPbb z*(B@vky>iQ0257eGoFD?Vy0X`kBD&u7}+f;E=B*Mi}7qDOhB8J!Wv{yor>0)jlA9Ht4^QjE$jbGdE`;_Tav^g>h5-f&J+?%KE3w~q?E4e z0Z~Q)=}u$|imZWv6eI}Mf}{GEV5uLdT5}exdMV|rs)6_`um%^KD=X0U8gz)$QX3u5 z<+=vqv5DcbvmCah_$;r^M*Z_fCB>Q9W&;osrPLx;>uT8q>)h8i=7d)pp%$x#SI!b6 z6tTM0ohBYiXY->iL2jm{O+m1}bf@7AZ|SAH3y*g=ty_!KajQ_$3*K%ghKo#_k#@^y zeg(Dp(&rX>VX3AhTV3)Yoid0Er%AL<1cOb)Eb|jK+l;f;wvL@E(!KvKVj`(q?Q6rq ztHvl8$A5!k`d_?zx6q z5@1>tpNY?42T$y?arh>;FLTmWL|@Sf*c#-7wy_DRMI&llgOm8w zP(phd?N`maoFLv4#EQY@d8Tj#@O%iZyAY2+GW^|495#U)g+XjNEDiU}rp4*(4ue(F z;d46THWJA#HSjP>jco2?9$W={Dvy=-4(L)?W0`|!rvw+i&?_B?VFS15 z%2&dYk&fZy_1ZN3#yNTR05ZH#ZTmotnU4;vP37?ZxGh zgXB&G4e2ePP4181iky>ZVmJ;K1fC4p+%s?@O4Jo|ln{N&k5JeMQJK;Msl?DK5$VSs zf)kgF1gR|ZCQ_4*bXH^>B1Nb-rjz3O8 zD?~kex<`NRldb*4SU}B(xM;AZPTkt;LnzskgvLjM+jyuZf65fu5%y%QGi3`=1?^L8+Fk`3FMBT8jY8mLpZTNTYxwSn~BcadV&ZGUBp zy!^%#T9*IIY>?kHk&nay1+@s^nBDFv^P_T3D^g-d8u{$Fd z$E4OSLL!`Gj@bm)cZ$Gzr5$Bx8>m#FNMIFtM2C z-?ch*>1cDDlWlV>Z4<-5*mhUj_g`XD36cIi8}DombiBs-&*}{8ZG+1*cE?mp_O=+A z_EKR+->;E%{Bw2o&2NQZdS_e?lq2cbN`V_xyD2@lwYz6zaXS3r=p8$&FWzcMAh;6H zslfjdtegbL_Q$baurVLW{k!~zkH1D=Jg*5Xs}vzv@lgk`N@fDywmI+2r*QVOp&cMv zqz>HxJi0Pnl5IEfyfaw`PS}Ya?ttc+N!SK3Rm0^~Jh0oA?oM&l=lw75SzW00t_{A? z>h;1GUBe0=WIR9Jk;8p@BV{X?3GNNm6kiimyCQu*!}Lq`x6d7k6FFs?)h$|}waEGG9qQFf#PLf*fZe5e+oo*>tFfiYA5nA~9->b0Ah4oE;sxwHKr9Bp>s*-!bO8T_SV(`!S`tj; zhpiwmmRV)_-wjS4<}Ka;5|5?zT#k;NXETqLvs=IpSJgM8dB62)vlB3YIk z;yK>s0i!zZ0vJ~2<)F@!8AaonV>7L!cHHDhMwtIWqDAbjnn1#**euiw zK4*0foQ86)r(#jO`9l%H5;$8SU_W+~O^4N+$929NDu zj3A+M09l?gUWw``j_RN*$xA9yXL*%fT;*nD@{k3gm^^kM9D&4aMvOg9nxf@n+U+Bk z;+W}qSfeH>Bo*qRwk4I!kvJt^4r1STv?xTDqN!z2+O1kK$XuN|i$$0}w1F-3{LTj{2YqY{C7U}0|mZATy>ZUUcsZrut#?aYmYQ_=j z96?#rsT!(vf+~il>y8T2ScK@DHXNT+9fdO8bVlcgwrFT58CVw1SjH&8DxCD4CBd}o zKqBmqfvXsF##t2S-axF2E~$H%U?d%BsZIkty=n~(LNZv~|1p6!2Gaxh>XpjZl?p{F zgz0`=E3*bsuSIIh>S=-%tIMv$wBl^e>TH`n0yYK^5@D;h%7Si8(rzgm<{WCDO2?nx zYmfGsSte}MzE*{ztGg~-4Q*=%q8(MHn4(VO+x=)F9^_Z@Tj;1@WBTK35iGk}V%5Uu zgrd!*5~7c~>TE;~=nxrx%xWrXEP0XRTY%rk8W#V&fP!Mk4$5lWl@i5Et)0y}uChYz z<4UfvI_u6_?&U%&BuJ=A@(52Vf|r&{ElX%8ta&1?)~B~ znQE?UwdA7;?JyZ#(XwNG5^!fK!=D}l{Q4RL4%N>+t?{mvPJ)zfYNl{<*l+4nwuvI| z-f1%YE-E_X+L-OxlAFT#t?mjR#zHLwDQy22Ih!>7ZIB}Gc)Z8<>MefQA$2GzD4xX1 z3GDxll7XDAFc|^LbY$$%f)Ew(xl zq_<|3TA*2G3^D3r>#y0+5=$dA#q53(Q`Q3Lfog;$Txi$Qad|!LA%(8sD9?(vYWEsm zKD}ynHRK_ytBy_*4WebxMXaM8?^#-}OUNpc78EF+q6hmh80qErx?+{#0+Pg zg~^IBI%PD{-YY#Z&H68#?q+QX)PEJC7j;+vt^QA{rLs8g)}mr(>ay6PgL?ixs$7tiA9K+UC`kl*$5A#AfMvvLUww3jws^Hi~# zu0RSb^g@%VLo;-lLi8-JDG6z@EhCR7|6fZSpBtBPgBEiz8MHO|qWaEmUa+hK&nX_R z@O0A0qN!Uaj*Us3XsGAa&=E`s1utvNrQo;Q&nEPNz6cnMHvw z2~mSlAQq?iv!?{?9mh0dmZeq`YB;y9T3%>aXSIB)Ae%NcL$APDt93-Lbz8IbTc7nq zXKuA_F)*W4E_-n=L!*ANF&4>5NMpKO=Dwh+RPsw2J<5a0~Zv`z~=CcX5BPcXjqxZt6M`iTt~EAQ}=XR_p@?l z0_V_P;%$VmDdTm7}Keq<-1}Pxmc7trRJvcA-@dAg>g-I7+2X>SD z_5-W#Nsp3i3JrS;Iw0{Gmgh0R3i6+e!6SD00Z|yINv%IYOk1*`Pt$sU=QmLob#t^f zap!FY)uvDEwnj@7g?G3A1WY(6n=+jv2>NpPhIjk7bK+VNX{~$pBOUDS&MUaL^dp%y zS~qlByL-H+^}A2{jjJi9%QjsjiL$PukKvZ+P@PSkuvg%VH^8DaI^$P}Ti<9fKe{p3r}&|`kG z*5q77x|!B{(t|$Jqp1X-L3V5R8yi^6)%hMStr9nG_GtYn9z4SD=3sL@Ea~YM$&YS* zJ+W4NAj_4i}+Uj%m4h-e{vZ8(SLsaUw4{b01ym>gGbSL z6b6(5L(}<$HV4kB)G$3}C0Z`U=ml@ZW3o~4oK({3=d>Dy$J z-=!s1+vtwacr4?pcPyKih*0yo0P_c^L7qQ+)n*+;m{4J~g$qJqzeWXfm(Gqe zaiy7_BNt9IIKQ9aEgf}KHa*$DDo_CUD0l8MgMHHtd>8oe-^KgnL7rUsa^`y7Fo)h; zN_6Vgh1|Rev*vc~-MMccX%lFAL`SrC#(8ZG+cJ3PopWkbBeG@40Y?|w;QQi8`5fTq9|`;!hysEC zq9g;j3m@X_LW2xbNI}7#0y8L4z!ai9jra9=1$yD5o%TpOTU`50N8z!6^O)D`+CQE~-Skk)jk;pVM z*ip!2nA|`|2M7w(KunFWZJrC{iIO+^7&MODa#cO|OF0}}v!gO$g{nETV0F{XJnS_q zIehPp(@uZsq_f`(l-tuMc0YP=KKbHPIE=cQ(NUT9Hp0)*at`HhVvRFCgyXdTA_cj> z0YTOl5zH%1JIuwT!XU`e3a^R4g&1xw8#t3FbowAf`sy6GN16=wTa!I6@hH zjrHfEQABpNXNgs9pB{O{y4oF6lUCMdms%LUg~Q$_0&l++Xt#NCWpyvNZ+<)FQO!jJ zO;;@&aLm3;jU#XMV$7@KgWaH@-#J^F2`0o9_paX*lESE9gX1-`ay_)o7E}Csl@;TP z)TF3Q%SAV9Kg&5HUD3f+XMJ_o5pEsfqagp-Zw3vBvt_|w#@Vf$OT|pWL3l^E=~r3Y zg=oq;-`SF`x0wdlw)a-W!Czz6#@1uUIr?I*Nn18+tk1de`K_%DJ9GH|x;>kNw6U!+ z-Qsbke}4OYtGoGE^XC=SYo=HSD=5Ub-`NaJ39`hy3Ly!{0Sr979|fr~d&O5+&dhM~8;?J{?)Umo@7L8EEUGfP92VxBmORyCn{Ib&NA zhbPEe>`Fryh?4^)=`Y9i&@2|b;OOd=uTAbmja&fAQ#i*INMUgQQ93eZD#cVnQ--on zG33<7z+}TPdFcV+VaZ5(xT!eE4|oLhpD%sGGex#Y7o;HOt%R6NKQd2%%s`nHv{x>j zWn(GCdRE!4$u-c3?`q#5lKE^?Eoi~U6rTi?8mNT1_~9{K!i4Aj;IP0%xM(v_3=sf1 z^qId150HsG9usZ$Oo7g)gZ(VgltK+p8) zA&)ji8nXCfTT`)9dfqLrKw|`8@M%}%i9!R0i{Q#+!nl_Nc03&PBp1+gLF+81atT8x z5*n+A6me3Mmz`HfNXW`Zy^N$E+vrAP*}@sBvLdpnr61s=Qj#DRfuMQ9H+140lNBnX;u@XJ^x^=B?6{ItzkW)mkbuLJyZ$5vL z)1?v9kOCwxB=3tIh8h+-7wBvgl(bmO7MgURODvXUgW1$2RU61*bSEXP$&0m1qKX=^ zg&VNx2B5&xp9XcPLEULim)g{uHbDb4ooZHNxWW@IK!`~^YY`uy#4Sa!fYvD5j!tZ~ zJLI8j7vZwkHYA#O>m_%2OUJ#o7kWO0wvglhODbO2cz(gMr&U+O%^*!}j8h%oI;Ic- zDwmr?*41i|k|ZbtZTFqt-J?*6)8_FGfX$VmtG%;bm@}uhywoe^S|K-)`sx^p$i6c| z_snMqIu4p|DN{rRywxKtvQZd?je=RXS&bWj02A)Cs6{?cxl9EVZU4iJG0TCi}c>hpoI_oirrUK;N_A#dhHFDJB#&jn> z7f6OGV^qgURi=rf+*l2kq8m20f7rpRXJZ>pyIfYWx?7rR1=_!Dm1f*J4X*<~FZGni zG&#G0&H$g@-RZ3WJuh6LlQ%q`QK$UWs%|-~Ck)~aKbS$YoSnx1q+gOyK)`_uOVm9pZ;mQL zW~A8dGMj$%VHdl2JKj4ul51tyoF;1E=HAT6{dFHf5#1&0DsI*&e%#>L{BA$LcmL)g zzvmm@D7BB{au06O5qxnx7W^ZE?Wpt@*dx?uXQTzS51I4GW~zUw=N zZu^nca4u!Qt*oIk>WU4qTQ{N`|eEyJ6nO1iLyGExV+ohB z6Bi0Nq9bcVax6Oexd`|F)3<+H$0%CDG6Se|oE1AHqBvWoyYnXiL@(SK!0+I_#G?WQ zx<83yvCzVbZ3HEQ)R+lOFagsDj$56V>%dc_$V^1VQB1*%48c$g!H)dE(nCFw14)bw z$y6M*m5YD~)1F7*rNObqp3_BxJTY8DG2D_VVtlsVG8axzq8zg(GV`ZqoIb2OE+}h< z>5?+9Gq-EeMz7+=g~34FD8sLE5`RcLTu{Tc%SyMoy@Y!^aq~(%WUG3sN5Rv(S*bIz zR5P_4Bt`-}k-)D$(++{5N%u%ZJkUkLiLBRgMEPp1BuBCv^hA7 z3k;q4A~tyGH(LNa0lL8dd6(+B4heFgg&P=4sTYFGxkI!|bmGPH>>fUJ$aKuU2HZf9 zYs~soOvv0c#;i!ll+4Jy$dN3?$~-xhGlBZ+xX*+!_bkn-XpmL1onFi%C}@==!^wG! zM}{y*wev#1xI!vSk=vA-q~uL#@lB^J3Ne_WQRCk1Q#)*b$Ik zGtEGVkP00}qMI|j@e*lS2%)^X+APXOJ(i%tgfQYw!eddfsVL#B)ZS2wo;=3n+{zlO z(@>?;DizgAq|;G#lg>NUP2xwuDnP@-)6BXo^Q56Q42~*5$QARC_N>u|jL%&}i_r7Z zF2%57HP&NAR%7K-3S&hwbyoi5#56TQH6=Y00Du>=i#SctnB)j02$CfrA_GMfKMlUr zx*vA`04g(bI>oykG_<1HWJ1K?B4mk1TM1EjxT^!OgAx6OaQsagEQ2+m$}nIj1+~sz zMbZw^C;)lShDF#^N~!T^9Fw>&1gs@Yl2U(it;jMl2*Ena`y5kcRrxcG1_ZsA`%+?M zIoM;lRZG^FjlGzay;nolEp^uWq*l^XH2`o^kI+_>cv$pc1D|Ni%}l@&svYZi6qmwD zLFF$ZWH04JRD}T1De4u+h?u3b4J~v>rL3|+GEsCYn}0Qeec2bl3{13D+qFg8vsKl% z71=12Tew}-x4nXX+_Uk-JjRo(BejTRd<(KITkqmkhn)xvgchXrIFudMly%(3h1|#g zZPt}b%wdJuVg=Ls)Wn<}PyzS~kNMe?Xi0^Y$t$RXXIO#@%qR!#xb;-roqRf{O~P|| zP4k1-7)9Lc8;w_K#z*ztTfv1i;MXe>T(MhGS)7eVVa`w$p;et*=PlLejaylbUg~|= zi||prjguS2ER03fj^ZVbEy!ARQp5d;%^I8yWY$%a+{ks`_RZW>yII6^UznX-5-ds1 z9W`rh)1V05$KlIvJzdj1U1_^sQEB?EQ^1YYG{RT4^at<{tN_0ll) z;mLj9$o*mYrQ9E$-y^2n#>HR#CAm%WT(u#;w15%~fU#YY6g=pLSf#Aa!B$vO6wm@+ z*v+kUCF6H&z9T!hu3d^@0upBtqTWphY1FN)>$cpiu1rPVg)uukwx|R4WAk;Lj;a*cLFAQ?fSwQs0#4nc?Oy~Y%>X8qB4S`@+rM4x)OY>E zW2{=E;#~{g+Gn=LZ19n9fnr>QpYJ0BEkj=TaN892=5Ge)a298@Ej;}Hs@S#VTO(L3 z6LR2mo(+fTSVkyZKaM*!P+UxO*-?gHeb#4Fo?jwPOjRa13Zr6IZk)sgDLBoJB*@~_ z%~M&K&DZW}Q^Xc6LW}EfIRmq7L2NIlf@)+bijsfP#jxhO_1@Q-%&M z+rc5^>78lmwdu_)ecCxS$ z%cbgkw(4aSuLUI8bOl^H#Jbv}y98xA;W? z4lsS40L^ny+ht9sz2WC{!zheqh(<21t>8E|hkaVEX?A4 z>iV_quJ&in&EHXT0kSse6+q}nd4M1_3E%tH5<0T93dlS?i#O4>7V7ZSH5OGet@n*x7YsT`UYhC#&34{X@*noo3PmC z9c{-0*`wB|;{6XteOh-h>H)WkOrBFn^RF`DIx^~4!2p!IjwFbaFHAT$4eL{!lq@-p+4Fittsdm zk!9XcDXhXIhvp(ImUn>Dl{TD}mXA!;LYVH`&Tg!m#%wI-@-Bzl&wj~DBH;kv@h@)Y zch;PEZma#6?f;YIJFBOU;@&@U516Cymc8l=f8Qbga3%I?%iZuh@8{blQ)*?_5?ASz zFd6$;bOHg-0cdFFHtdM*)2xv0=GJl4q~wgAT^3sp@9y-hvsaPcspu=|YXV9k$(qju zPbnwPf!!8$vvM(5a7$j^)XCWV&~&=S!RDpuUw?2vDP%?2?*R-Y=sidy1y|T6Hu3Eb zIMxMoUD{rLzcgU*_i}^UKGvD#_HMVZsh;YZ74)tD2I4`zb3l^2G>m#Kt;i z^I9lxb(DBYB`Kp>M}o7}@?XD~(`Ie-lV~s3>{PYvlXomIKj$$gNH6B-Puc=^T@}~% zbdHW8j4PoDZ`p81cVq>3nYCY1K4QoXbPs<$Y7OpCYXHxsYx#5KM(19uh*(>+i519q zzf9d-KjTQa@kn>*{`v#uTf&2H%g@c53uUJ4VS9B@R1aN4-TmDl8Emi3fX7ZKmw8u z@|lDVqESJ?gw1NVjcj*o`f|NwtXQl?le=fIJ9~}8-{!Er-N@(dZ*~LqhaxygSZH{N zNZ3FiL?x855-BN3aa06_nM9E?xxyL$nG(vR5HMxxh>B^s%DP(mTAERTWD48bdf=d- ztB||<8v!iLJIt#aOni__WYn1XB;=bct-M@aygL!Gt<5x%?cEga-3uI^f$X5IK8|eg z08ep|u~A>IF&V1A&o7xG5sw1EfdUKm)8{Far-htSTp$=QB1DN152U*I45LO&zp5Sa z2=b%IkR(BtJb5yslq6GVv3%(=W)vdVP@1$^&gOv!{Ce_paEn={mZpTnI~orOtc|IH z8bt~`a>abZ8W)B!W zffD)sn`lw037r62lkOh>g5Zby}dG#u^sxr~*%`<+#*z!yG;?>))zy3mWej3=Xq+cCpp{4f~ zdiOiam12u3 zx(MToW1LvPS!v{zB94IV$fF>FsRAAa{yfx6D;jMQStr9)K#G7Ks)XTY0tRq^EQ`I8 zQfxJyCL1>7siT@DO&LJ{nme#;*7T;EISJKWE^;QOp-4`!W+|l4 z+=QQNQC1~^prQ2F>LpDL6xEcKrRvNr12~!>L?VDdLb1g{IAMiZ@p|ETU=cR~DwjC| zfE;pNi*2^rZi|)x0DKFsx87Pyq8We0b)z4A~eiIq+wt(=I(SQDv=?92+EqZXhXk4ovu3DT58us1iRpL01<1fvB*+&r_>#yiZH0I*%M5v zW|7<0iRkvhV&Ha_yJC!n%ZLYw;V#a&jV10;W8`PWK{;J~tE&ef4FHgtEJR#Tq>*G( zu?cbsRryW7(?Y4UC+_iDo&Gj}RM%u59te6ny&svy0=?X1!*vD~UMMHIkB zM36MydaFwRCOuR4su_%+?tf$A$$qZmpYk~DDN@lDv#Pd=@ByniY-<~_KyV;s<&AZb z@|A|DfWJR|V}4I?9Jfw5hs&W5h3FCy<6gM95ZxgUR;Z!mZdkb-rqEn^5EmvqBS5QV zM<%igKogV#ntzRHG@-jt5wL^~%G9k@R3x7jTSGeqer!_```w&Gg|Xt1r*~HoV;GT_ zmE)0vdHeF80t^$K=uxjA7f4z5RK_yy%@J)#SzQ-9@{0@+rc5AzLL*@%o02L0*>8040#E2J`HtvRB2`Gs+am)c#jd=Hff8gRK#oO%3KmO3 zHG`J_{6sgQt8K`Q@cT%SKxD#ewQwzWsf)FW88{b>C~-MLoQ&!K!)S)f3VVPY4zHO* zzU_rC;DUzd0O+4olS5FB?z8c!MCZWjH6(d2ZPv~_H>XHz*LLnw7J4yvIVRwyp}Z2D#N^7(}rxtrscf< zz|C(xp$QHOfe1n{H%?x}fP+HF6r7O6romL7t{Yp<1^^yglq4wX)8(t!xxS7_@}B|4 z=O0nBCW{etjC%Ub2&HiwG-e~A%#%m+PC8MFGK8Ybp$;AUiamyOloQ`$YAd;O87uB$ zqg?6*OpwN;R#D-RPh+HU-8mM-Qt}B2NY4&VA;xbzn-G>XID z0=!JY2{?slMXQdwki*IO7r@)>ApndZ0tQ?#ma%+qvofNe=P1OS0q|U4m$**<$cFQ; zh$I@CQrtu#A}mY~vn&%YYYu--W5>5KhsUI5n|4r07ijF*Hk9{KHl9>SifS??F3P;w ze&UW)*1&BeM!g3#lMtrQxpi~aOX)ULXV7!aMY>dUH+fh{#WQHD5<*YR zNy6_e_g2(F;;zhG=L50Qw&@kfu-5A~508YZ-+huQb$Un(EwMBR5Nir2++R#%`qDZ) zA>Sg$LIyWjngUoC7P5(I0AFrhn*%@y)(dM0K~S%m2r)uPd`^ob5vEnUBEm{hRrHWq zCo>h|NP)qc*W7r`mu+!<+VVh<)AyPpWa)>l=H;5TH`gf`^w4pZLD!=*u7 zqlV!XOr2`~(gAbqI@cnG}F99XiV0qO{#dPynV2d1U!&aD$?+M7* z*jS}mC#)AaHD=51BEtw+fCz$TP#_b}Hqcg{ckd)_f>_Uv-|mD!_9$gJ!$Ry-V)Tm_=>?j2!A;WI(Peq~OTcv5&+zjwCb1+u__`;X0?i|vO^YUTq zc@$`9UEca4h~f*1^#wEj5k_kzRP!}6b)ymTlp>d~EG(1 zpNE4+;6|wjHer)WxI|M92vP%aXzaCNu$M*gm0yNcUqOWb9$BSMAkljPhH@!4eU+1G z!Dl%wSA}ChU~cey(1&wZ2y;A_2MhoK+P8gD6+lHN1+TDl<0XUaVScLDAP(YQ2+#{w zVuH5AM30v~5yN%(*N10f4M4YH7smhr)mZ_kc9}&X8+T-Cm)Js}fp? z=Wx+5g(nAm>9|(MXD!ReeAmQj6LyYl5r)!qb81EZhFxfVXJ|odsEu<-Kd=yfr?(N0 zbsr(nUhQ>ueyAmq#}V~se}MFm@mGk>b^s!vhHD4{izr55^nV>WD$y`dZO0^q#vBf@ z51#jVcb9G;aYxsalG>vI5A{8;xF;Be6zt|SmxUSI7iqBYZ;VKV@VRT z7C#Cla;&3B^O%M4xM^wOj&=Et)Te5B=>~oyeD!E^(dP$tsgLQx7ulDQuJ?~rVB-V}_V@z2G()$zB}Uhy>*upIKR)HEout zAfT8e%9xC;MGzrjj4ZR8l7J2t_yC~uWfFyeOZAmB;U&f~NQeg{^+rztBb8$&nl?3h zNZE(6xOgY1od$4~#mG^~2%cKWj6Bn55d zjZ<_4mU)>8xsXI@olnGqbT)VpNp*zSSTA;U6XT(%2~{M}4e}>#odSv1(3)h#4aFG` z;pwC124!dSl61F;E3iB)8E;=#i8y)x6ySKGtk`);Y8u!vNrb4G!q{)Ik%G~c1n?)K z`G-?QF_bWNl+e)=RtX(hX*&8aFEF?>XCq$g2ZWEsp53F7|41PV)rM2qnXLdVvnQA4 zXrS--pDG8S#3wn3N~i=nE`wSl2s)UVqp0clgg_hGuk6;8(T^}Hqo?ACkLa4Yk)zt+cN^toO%@7`(jL2MZm#$k zC8=%cpsc#UM^N%*6#yAVS`7A-Y@Kqdu1bP7*@B7zgKB0^Ub>yl8I={8NZq-BYkIEB zsFMG1Un@9MaY`lRMPCO}rxcX`sdvRh3FVFD*M#$x3&&G5@&PdVSyt2sbMaVw#doNa zY9j+mm-Gm!HpfE?T77uI0Q`5SS2Z)l6*W^QumU%)GB#7ADoY<~svZG|bfat`i=q(2 zkh@Vk&K6ZLN-O|qT{XI_% zr%?g>Af;2flh$yp+exlq%C+<-X@jV>WG0l($*0k2XExZKXS$4PiyRa%0B#zNtLKA? z#g^(pXxT=yPgN376 zUWBcXr>$4|Puv=kp1HHH>!sKUbrfl~%3!X+`=eXAu80A*?>e`13uu)m9`z~#Xo;G> zBzyAdLF9N}^*~hM)uj|hmpJ680k(XSGoXjsRt74$(KJI4X1QpQ0E;Q1-q$qLMIYh! zv9~vep^Jz{sJR?=ET*fz@`tLJv9kQ8V)a>?*=mtCh5&iHUS%gIbMlck`b9b$Z8^)U zOp0EznO-ZI6tJ@Yu4bC1q&URPK_vOn69FQyQhU8j`*pviC#bu%nVEmo2)0<(;6E5(u3yiqE z*SHcq!Hsi|)R$2SpGn2N&2qQc(Gs;k%$twTuOdb%?V ztInl{w7Y$_8^mDZh`AceVZ@_mNddhRELmcS??!iq5hU8yyxTFw!f|)b`fP+zwXpba zI@WlO_spx@PqN#pS~X_6Brqzd7A*9a!{Iu%{TjzjLsen8{v@3l0}_lJI%K^n$L`jhUm2>`j$BhuIATn zW!uf_HF20tnMrv2@VN82x(T)VZsY$%9LooE&tK-O0xiouO&ZrHom!ytiX~ zs`RA)(TO<8s>#v69IUm7zouuuO^Mb6%{lt}YViD2 z-8auiWNg&<&~#H-f$ACve6V1U*Vp25#^+X!Td)NkxylDu411Skwa~7djj$(hO0 zciJ%4QXv{4updd98`A3>#XJ@9A;W}ZMkpl(il@}soJEK?#>_c+9q?I$sN9-u*)IYA zgJemj?NKsZT~SO7)&p}eVflju>uCaTTNURrdCcJJg;#OSArhzHf|k#K3bIG=-hO@M zTSzYSy+VM^$kL~UR+t6--R0~xOYb=*l{7%)XUYRbmL`314&FB8>pQ6tXBU2TUMJx^ zS(>UkqRC*=nhT@I&6*%N4W<2>ouW3@Xst9|(q1@uz z0nWM9(>lZClg+b!%0xg9aXDAZ>gz2X!_4K|RTrH=y)sNJD@#1if5d|z(`S@n<-uo| zlmh@+ZtCC-Ya!r>2#&v92z2iP!@V73?>@;~QFyvGH>%ZQMF`?rsN{1Pq zQBCgTf>QF%SdL{Grv{*x@r|&~o`KBUy!$)Jc`E{Ob1HLs&-mL6b@I7SV&&zo^_}Lp zhkHWnZqPhr!Fzd_587Vxo`{wFjF@!wn*4_X*hSYM=%}fX*;>7uiSXRanV>0eIN!~a zj>B;6S+b`vx?M5KTd(r7%lF|JB?FtcY>DcMn@T0XGXHdGH1nGML$MzJ+(nLcna|lG zE$n1#`9(h5N;6lS|D~-1cxfx1T@Am-t`F_nW}%AREs@A%zjCb2p4c91s@Xe8mFQ2B z-Uduv)<-}CL}o(A^p&KLUq?ge1yCmi^i`f+Vm7L0-K#*g5!x6+I5?{^-f zpd9cW-R%nB)ZB{U>PzMwp0Pk&S$G0pJX<6=*3%|F5I``QC=3sY5t&3D3{B^=Nl8tq z)#)md)kcoW?pI=+db@_f`D{Kb*lmY9TY1mn^tTyrh-@}(X5Fr?X`t!v0MN})anTQs ziI6a3V`#7`@X{^+1UshC|o#uCOFw~ui35P1gr@}X?cK14O(9=t@}(L`M9801Itr{M6cb%~RRjk_lai$j-AwTi zqhy7tSYMrLDzYqZ0=GGDaePbAWhmGxP#+mR#k5y6AQD|9jN|G8A=r@Xg2RrWix4t{ z3d~Ht9>ol>GR;MELTA4`aoS`lgCfkjB>s%DjB9L^s)QQ`O_41Dj>2AkFVFyGbe+*a z-a0ioxIi6sLnBj9-!Vp@P2mHeh;)sV>(CbeqaFDiZVV)_x(Xsg5DhBJsq{Vkq&81e zc|Ii%YDarc9qiY-AFt{eiz%| z;lA7HfMwM6R$D*vnAe|l%te;~PR04u0YJZ30(_sT+``C>Mk%|W>cUIX>`ST?Y_$Dk z4^y2%B^7`HB*u4mK{f-XN+DnC&3+R9$Ddd6$3hz++S^K!t6f0{LVMF2ivWb2ZUyf! znCJ%Z6f-QgMT}YsFyRNICn9ZCZaLqgk+|5P4Z+Z8b>}kz-MFBmTNv&p8i-dFP`7{u zG#~*n%3=EY)vVXuCU$$vj=)wDht6m$S%gcF#~voQ-g$9&f@_Szo?yH&8BB~4>d4MqA9FpuOzmF(@d(r`m%)+OO@d1ZyJ@cWu-oieiJxV$l)MEE;C5{DE?^XtY z+Mo2ds!CqcQVh|g06Doec5L!H1~gTDyhA{CNX>x_d|(G*A_C_)h<0p97 zgnwh)6;cWc6Z1m<7xA)%ehXT(WT-itT<>KEP}mX&2^S$$QwiW;$O43D z1r2OqA>s7S0^(w?XL3SG%Yvns_M(!4tuYT3u#%Q~Co*M$QG_A;Qt@U}8x1upjraTv zEe{~i=N+aScLd>eXgNLWB?^*ubkG5~r$|9A1AKlQWdX;AO|-NHXe|nXMn>vE7EXsm zlLJybl~%u9X=fn(+uuwrxj#{X(mOfD=_Wgg4^SqMfMOG>sStP{su_a-6@e854e3hz zA+3USdura^C9Vq1FmL?v-4lDNWROC{FWXwI*V!&}}z8FMi*!ZkCG z%HtVv$FMJQOb@$z;w^dk#W)s?Zk>q?BNGa&JUZ$deC*>N#TK6cc(gc3Q09d&GSU$I z7PzfMKmrhnT19cBBy?3;lk7H8v0)NzMy(ojZJNKYb+Woz8=x?DHdOf()w;$o=0(yP zjo(%emdzE(((G1CsJ75HyA9e&Cnqvq8Ae?4g@y$2o0zg9Q*s1QQF*74xyLf%TX0aK z4tK;Z+{UgJ2q4BtDv(5_rRNw(0H_}fyF%}`&T)RT1t74tS>yV(+C`<{MA+ODy+|6__aDYTv@jF~rE11PCC23(_&^N>iHB>DbP6r$%t<9z46@Rk#8d-f-;-XWidj zvqIz?XZuuyz))TrpV!&r^{Qd2j>=%+IKk-_(O27&u~oFQbP@h=Jj99UpulY=9)nal zt(Tjl8V{3TZ!xyiXsU`4m8&$Av1KH!*5Mok92yB1W7gT|8g~n5HA-96!wSGPt_MNA zeM>fmOxaVs(1ud4X)OhE-T>aQyjXiWNbY0J zYc^ej567+_y{pL{Amw;#MBY@Evb;r*L_udcI`1>UNI(%kn!AHlT%)eT$NX?JW0+M8 z4`9{*BR+5hQoQ1C{*Imvsd2>ZIk8ieYM(Ek?a!F}Ro`?W3>;r^(b)t7tRe7G z#aA6=Z3mVY-TGy(fcC9)B!$2YfE;We^&>Amdj(F3j z(mDrlN+5~-wSkk(u)1u(*Z}IDDr2&cZTOn)u>uC#%me`eL4KUanU*c>LN1c7kKa6m zR|;y}6a%(M1}kQUH%xC-1fbmp=`3*0BB%`Vd`lw|iGn(i0yksUAj$Q{Z1pbXs1VNK zc95pRBfGeR2PtWkKFQ5u4+R)t`VM745=p(3#!2F)+_Vftk}w($ZP2R91q7hd;DG@D z5=R9v;0rOJTb>{&l!*XD$@Ma&3#TtbTmT@Br~xbhi7*ZXJS_t9r2xh!XWT)#Oo^k+ zjRKA?{?5iXgyH%sAiMx6J_L-eJ_A$oFUW+gt;8;>)}-$83GLjDp7QDc{!XESLKERB zB<9Y=&Thxh>+bwQ@0=!~CQ1WcN}U9+5X?qBz^s5C%*psAKqODzl;`qNB+x#j*7D*b zd~j+`PxMAF;jC)}=|fYj3kcVP^lUDsG70u7E&;MCCZ=y9E>QO}z{sM|8xV>NKZ6RZ z&;UlttfbHove0Ho;P_NP9_3LCeQ>y#L&7l4(r6A12Mz#Gu=f1HsLW;|=8O*i3v3;X zh#`i`f4VRJbmkq*#|zkk*Ct>efUqARqow>12lFsKE(Z~{P~aXB0H)|n^sfK1=#Ih! zj55)xDr-xc?cbh+Ga{gH3h-j2Elr+;6eZ9UW3bzX5%5~;F*Z;$Qc5ypD-8_Kf3D_g z0H6eQ=mH{$q!K{;M&<%Zp}9IPw2}@FYEWvT5g9{82NTXLsg5BpB@NFKx}M7#uMy5@ z?;MqaAw1-6*1!q5QslHO^B7?+Gs5v!YEk%csEp$H4igNKPadL=8EMfY;3fF&p9#sZP2`(;wd<0W!cT#b%xVfCcV;OkXB& z6fF_QTrw0{VmE;aD0qi(dQ-9LX0A3S2 zpHd=C&`0Fz)U>eo z88=fhlQ04DV<35ffdFIy9AY8Yk~0B-03H$|0nv3jXFb`{Pu{a@;PVIhv}&v-2g_1E2S}X2 z^Djlf%>;|b@UjA)FgEw?tQ28EC1}bbPx3ZmK+e!qCiE~Vv_eVX3~5nRA3{Kbs!Zz> zAPp%UZgf+Us)I&UXVCJf@{z5kg|qII6YsD?)I9M~yKW^dQO0h5JPXfd}D!yZZ(JM z9BB?1#1g4eX(HS(7y^rD^{Rhb^B~YPH!!jxebw873O{P*BZ<`?S+68b(y_b^F;ub> z2M}AA^lFi_O3kViL`zJruoU6wBc{|Rqcl!U3`i4eUvI2}1jDYx^g7q|RLxXq)3nLv zwHjb#-W-6+-s^1Vw51+1P&F+-j}em=R$%qCBh9k(c#u!0PE_+DsTNeLyaUbzQ~(-^ zbD&UdZAqX)1o+q|WJOj5HPyB}RSZ{2WeZO%kv1X!JX9DAXlA<$RB$GjqJc0A0L=oc z069R$za6U1cY~E@aA9r@&PEGPZncnU*Yycwt}!T##ONrwuniQARBNwxV|&LOBw@9n zBSEiq+oHEKwl<7F0`Klb9N3O?J0T5RhD%KhECMA7GD^uJlU;{OMxBhBibU0tf<}bE zQsXV(1Wn}pwkwm@BLVeK+w))%c3=evry7=l^Yn3da6@&f9@K_%tt|HHGO7%ef)Mms zS&q>rm2@xl4CaFXMBr0XS0XA@XS>f*1oC%Pa}j5^8fRDy|Fc+%^)>s)X$vUMqZ#<>K_Vr zax>T(62J%(h)UMWQJvFsyEiOC_hU=AZ-+Q&4HF($>48u-A_!N3YZrk__8hYYhV?@^ zL_luU*h~dNcsI%yir0r**+rAMe2_KSYRzoHcf+f0YXhib*@)tGFblt%PHWDFr^k4m$ zRT+1IMR4^RH-Z}uAaII-6gh8=u+8f7p^OY){}m0MM8lFV8Iw0cJ|?u2S@=~z88JiI zY2TT<+@Ydx#zX02Am%ciA#yPx4OHtB|LT`)yg&hHD^m{9bxBfKX*g(|!$)azj8f8k zo%A@FIcv9;V$5~WzKDiKs7aL-r+w*c*KQW@7?14a6X_sK)7X`R_M7qd-L8R(vcLs^ zYk-k!pwqc&k=HXn)PL?&P?-{*5!V(Mm@F4JHUDRxA-5h5lAi(8pDXrB@{)7Kk%Uqt zRjDwc&ubvG<+nO{l^7x-Dm0=)I^s$bJ~A4k5!h)tTCq8550S$iPc=KKF+PNtY~%qY zE8tF*7KL5fAktS#p11+o2*o0Gl70FT|E0&oy!2eP*tPvadWV`o%a@q7*At(av|lS? z2Fok7`IWhOiQD*HQ_v}_x}47$3Od>;1%lQ_!URwStl{~r?YXV9JCMt=fddGx|8p)$ zzyjv_%?bclcJBod)T`L!pb?sMJ7ORhy0BM?0T!aL!_cr1dw3b!qZP66IBE^4kY;Pv zACZHV3D2<;TyB4ewglo3jS+ZTdTb)8|9}*V6Hu{?()SP)^YWU;9)qW^xwSuBceu!l z8|RK?Pm^Fyly!NJtlU#Wlmd+RCf(!#af$l%i zU((g7$&|VWBGvO3D=Y$X#B*c;7|SI79b_Heha8BD_8Aejf z#JzH&UT&ejtIOTosoq`DT@N3;YMgyVd>j%AV)EA*wdP&y9V_5x1Bg0Y?KwQ{NyK=} zc*N}vZ6|@?5hKN!v!!Zj9m(+TiRs}7AM&BnqvCZLfB+;M5=E29#1#sOmd?6d*#mcXH*^-*LZIw>8C*y zs{)e4>Jhnq{yxA1^aePnpd!L5SEv*+tU{$;!-WwgLbSIbg9C>PQ5{Ig@gvBP0_)8y zxZz00fGYjnW0|m`sF*ROWDqw&CcRS+4`fi0K_Zow3cL}OzztM@qyfM&#S*|M)S}%e zP=#96Dpsr_V&$XtB;u8(&|HkG5Kk;xY{t&2ZIg))+_-Y*DsyAk?OQg7_ImAWsFI_= zAS5-b^vsNyniUu41S|6Kslt*$^j%_$86gM|eyRA?i-ZepUEb6sy;cB$$*~TDyQNOF zTG-ac|9m7G7Mjg4#!%-9@frZTZUV7Ph}YNMPKX9r{>xXkxdZ6=G%Cn&GNnkzA+03W z4taafi>h4R!*0HOdB)2xRuGENrTggcfs!1ko?uOJ_fvtC(lJ8^?NzayP#0u{RXs{A zl~x(SNp&EE4-&N%EJyU#S40ew@x?f7Y4{;AW!T}zX{D8T;#?@ADA$UZw6PbAO~}Xu zA~fa#m};^J;~^S}RS-&jL3ZaISkmNpOJyNkc8drIET9DkIaCh5~3kqSL@ z|7f8N_w@ zD2At_iGoLw0bN#GNoCPAcAelg0p1DeMjxwCE&-pa107k96`)=O}yTW%u2;g+{tD#EdQ!)OjVNSF(BzP>QLg z77NAQe)d&2F`(W0_vi*bHOiky2fQjlf=UI8g{wasEh|+QE}}H9=q7@SyC4eu|Lf3I zvq~(ncB!Z}*IS>4^_E?;flixAMwz4q=H^}L>J8l+=jDQAU|_EJ)A7J19k z^b=#QDQs$b520@ve_Q=-83~`cAwNg36@X>w5L2{m3lc!U2qQSr7(D<<2ZrdS4;t#s zm3j*<%=WErXs06IeKX1*n?kqB*O_-|%e=od6Q`u$V-u*N%+7jB8QV;egsCc6wN$Z6 zFCnZ~C8_rHw=V7Gu8f1zt65jWy*T;9hP}1e>8tPch|D_6SB#55GVNe`gSI)`Yit|y z>voH4_#^tVg>wP!R^4mLF#PbY!T}I*WANJ}b^(F|mahv>kqW@ZATY^o|6vIPNZ?r< z_o@VRu7pnDnM5KcADX$WVj01h?jCii+99uh92?LC#Dlz^ee7i2azF>J!?Gc|2Nm2p zB7umfo{1bF0y+ECfE4nes{}16&}$yjum~ZlTu%{|sgYL<0j#NsuYzVI6WG$|HL$JE zE(DO#`>;17uK+L>BoKik5)idIARq&wn#b{MNT~nSQ3wLG1!3AYivk*gg5a=H)F5at z2X;Y{kZ|Och86(KapN{yAWP)VLOCUAiC|jfh6_sQxh0V7bl9mFpNQznPvnk=J=EC4 za@nyOK1@!sOU^An<~k?Fu!!>!$~}-6s=Ilrc&_86DwOw3Q@jc(|3WKc7N@7pY)ZgD z-u$8$0S8VIbPqNfEQ<-wsLpk!1xx1>8?ol)HTT6a5lmZz1S42KMII>_?YhPq`p7>) zMud=sJY*Ik;m>>?u7EZoWjo!uqy{p@l8?aTzJMjd2QloEGz*av8mBPorE&yH$Q46W zh!Ennu%$c0jt*5vJ6qZkn&hD&PD@dN7^2W*KCP*JA`rw+$+CCJ)QB=^LbD~>l$cP2 z8G#a_#VW3fRTtFaRZf{jadH(`ZpvuD#6dN7lC`XY>rpn+7zd}N%bwwM+7lvi(6vSI zj-ioPAATexK)M2@-YSJb>u9F|CbE73>?$J47l1;7fRYVc|BYF>pv9_U0hIgzX-LPg zQ3>gzq)$jr=r9$_qC%uiMa2{@ZF)PJ=9C^ed}=FmYCD)v!Ko)ZCT=x05Si4+c;eWW z@enl%k_{E9GnGO$RW&qDa`R{hGC)_O8%A=16>`1`mN_G<-F4!%lyFFATg@iNx6&#Y z*2)nH7JvY1sN+lIM8Fydu->}{bU#vHt6M>)Un>L_fL{5HkqS#K$~cr3Wh910tL4}M zLUy}YIAupGo152+GP7J@EQr|q9O6&S z!x(%N*Qm%`aZ+VAF}>#4RK-OqxTdPyU~N{bMxKy#|L-ZzuX+!19U5A{94uw+`eA8q z4ewi@VBT@PjZRE>;~`w&B=xG78YYl$72t|1g7(+Hix|M>-pEf#Fe1y9iEvt)6iLQT za6{BsY%DOU<@?K}p#Y2|IVsE|h1z2`H&5P3kb?BJpDORSt<=nbH$)TE!`*>1!Jm zTh`uKw~m?sqkQMmQm5o}dmXNjcrvk9(;7%X#E5ZSkz56pAkrfkD*$|5!USM-o5q%v zbmiMcCDUkET4IoN=r-NuLxp#1(y< z*2Rn;J^lnO5@5@oviKCOZg&q=YGRIin$_kwNi9W z2EaR@*v{Q%S2?#zaBAJ?O+h+#t2=Fp|1y2MM7QU7lO*-Rf}I)Sb`ja8`@W`uu?cK` z`yv^!&%=oTi)6n(+R=W!GH)R%>>zvfvCj=WLca(A*qo!s{OAM_0v7p33*NavXe{!* zX>kU8uOXf!ijTgDNcb!u9nqwz7URX1rUg@K4M=`O5kY~30j^xjc-nFC-$21!=PgkM z(A=n{#Ifa^uu%=*^_=S++pBby#0?!I5CWb-1Mc-6zciWBxnR}c7u0E6W%!fZ7+rkb z6AzXfj9l63$enzZUt`h2c{xH8+}jdlA2(c|IH6k>fWp}IMK{bF^vxX>s-OB9#*|%^ zk=aB7;2ZwspZ@LNZUK+~h2iH6|JUJ>!U-`RSUe!*P2lZB0q9*|6nU5_3{%Qc9s)vM zP%PX9atq46P)?i*>K&TuJsj)3-hp_N(Qs7>DGepM6)nKg`(;72?OyzGVn*>ED9Ru> zj8b^5+tYc4+YQ*%QGyPlhF7VL4?4@!^b`T4gb=D%=bS(Z=;A86&C<+I5;h^Son29w z9akAb6p&pLCdmV|9}zkOnI%9pMoBcz*@CUp{56kFFxwLdn=bC+H-e)$_FB{6;Srn^ zD-9y1yXt|6sN?0a~=6C`Odhh2r>hVi%5+n%&@lc}bOA1M_uT**wb} zr9(p|BrcXvIF2I`%FhryRQ6%tB50pQrdwr{ofABxwY;PjJ{l!l7eDpH9|~fo7p03g%H%lCWK)(OVp3mDwx&+zq(v%Y(*R{q3Z)SA zWnPlmQBE1cJ;*jP|DS21=4qlP;3Sp*T}WGIWjT6f6eS=$_EO9QpgQ(g;l-Hdt=Py+ zXA;fjrP(9IaOdP9mqA)3UzR6sYGFhMW~^wYZ>Uo8#b&k%ntR%W?QJAeNG5(xW(-yX zHl|=^+F)N4(gF1(fL`KLIwgXRKrh-$Tp?lj(E@G4=b8;&e|?+bG>LgS2^UIPHcCi- zAcXxT=r{%^8M3Ez>R45prRdcIS21Th#vx;nWmtygs|6SB9G8ljoTzE%cIsntLPS1h zr-wnf|L#s8<*O zm~91xEL5Hr256k8Zn~d?d7*RtrkyPTi4Lc5f}9BT@BV}plB*1S) z97lesem<*Y%3y^(RGjJvW>^r@Osik0WDVZwQ!eU}_@^>P=xf5Kl3i7Fy_Z5Nsuhk) zt}f`5>0LqagJz0=w+0@3c4|7VXsATjkGd$pa^;K;EWwJ3TQ8!=S0EV`O27#b zCA+q3Kk+5jUeY6F5{O9ZDJ>i~uI$l5WxiU7rzVv}KJ2X4BSGPbiqclB!7ZwGTFE7> zcCHXYoNA{;?0pbgtWXuM4lbGP>c)2L^juF;h^%4$XAe5A+7PZ@{?_^lTgp1C?s=lY zHS5e`gVm%dn@$4p0qp~&EAX`>Be+1O+Nqr;t!gZxL;~s(auvA_>JuQ_PeR3)F{FB# z|5spa?UC?qVIUCs$SbAxMcImI7vV}iD2+0~C*LNg#9o|=R&T)u&yC*HAF@tcMxMp7 zZHxlXJWy;XVC==7Z!~IblQQX&LdZxyO|bUeMHcAk{$OUH?_TmlO%SdSGOmX8Z)L`8 zw4Nye$_uudpfEx>ZK=a@aj4Bd10$_zE!*t0NtT zDs4a>==#wa zhfXpQu%E%WfF+kL>iQ%PAHe|L4xuJ-+ZM4w5Ank?;HWCZI9QsZP$1o^{|6M?Dqb$L z`L-TJLh(ou@^sZhLS81dcH8pFvqg(B7@stfLNnyNB>wboCf;Y}+GkSmtll`*2xl`6 z3mXRSvDdxTACvR^BIZU9V^NTFUt&t#)sZ~UT_k&fI`w8zxavLA-4pCH4qIyS?J)C3 zN1--`9lPxhBlJOka&#_it%r}(8@(+(CDUh#6*K$d>FXB$m0n9JSLV*kC0{and zv(QLM>n}!>G-2Z}uq`G^iy3!Gvv$>S=o;Vj5rHFtv<+LNHs|yP7oTJVX9DzW5%MHx zlk?b}Y%m%K(?+#is}LirrVDRj*uhuaVc{+I@A+v6p&_!PMvgRM|MoVAfC>3>l29_c znSvdM09zmTLx=TPqxEw47)`XYLPH*@#>~bFhxbK+Q`#~!7j8xmH7`qK7KiL0T&8HA zjpO>pN%I{UuQ#r~^jMTJd{eP|kA^-UwoK2o-^DRx%N4bdL1o*qRc|v-dx>^y=wXR7 ze4-s`GeK4vnNjcM$t1Pj5Ty%zN%y!khGRHdY?dk{8pJICKg1+&-zl?1mnKgf#uc}5 zPqz{?RjI;ibF=L!&v+@*cyRGm4YRjgAQ|l`Y=U~!^ZePRJTziNIejMk*_+Dw>qnr00VUQ>oNHO zq^MFbfi>ZG2+*AuujHDdH+!phmJhp^tL(5p8i*WwU(Q#Y6}woxG2rkM{akH-m+pU? zYn@u~n;Wg`_N;jxxE%-bA?*1n;Wn-^%`+1Ec^Z1NpP!A4BdE`+8t6E@=@7y`L(EWIzY1FUwsV=TMtv)02q*B>kT9j!Rv zyMRZUbXj^-`uo3gy4tTj+qXSIEBtb-HK{ww(V#kZSG>hj{FW-YtaAfZB$|5VdLpO_ zy6}2E@x>F!ys$4jg*QG!ZvED?%4|cvd^!H1-<}!EyynNZn8o|Eqae;(i_ZJ55NOtt z0jNIlZk>O~!-cszu^%IXl3}8bMZ(|8Sq1`!Y*-9jQC#8#?6=f7jQ1Y$xx% zi~YSum-h}WC$s248z;ddJOws!iyyqFEBxFabi?oM!%MVxPsNWL0N$7Xs_(rc@_qF{ zK;E-}<=RMJPkADo{4%HfNoRgT5|fB^*#Bv{bkL4*kp!kQ3IfxiL#BHoKw(SSva7d2WmK+D#wivcbYI8l;?Nt7p3 zmM94!)<;ybWO{^2Gv+M{7br{^u~MbalRZxyB1)8vh-kc^jmt3Z*;A<0q#B_*gwItb zRkd#A+SSOCt<|E9jS9^yR4oFqs?kTP&f7e3?;zvjhg4C#i0CC-5K8G^WeJEiTLw7N zmwIhOcwF50=FG>DYvn2&SgEVQnKfr3o8?$Dq@Bx3AuE=MI@IAHq-MRY^#~V=*CdOO z*xkg7vh`);sFC7EiNSvl_gx(K|8e0ElOre2ytn}e;}kqJzTA5A>l$6fi=cov@bBTl zU$jYUKmhdjey#_~vV_Z=I%UqsS(Cm^`tdI?VBf!eg1m|nL=YENI>#`kDg$gZ(x&nX zE3CYda6$?xG_45HGCb=ntlY8fjX3NGQ7#hY*y9YgW=lj6L<(VXkQeuoVwXukVdO>} zKLKViPW+tu&<#%rzTp%)bq9ti&d`BJGizL3kOjM-%y`2}wHK_J)etn1%UsK3|oJ9peI1-yjV zC7Bj z&^Z)_PK7D#|JdkG1}T(H>UKMGnd@5gEBwifh9$9??R4g!9|rMu+@M63c;`E@2v3Q> zQ_HRp7y=R<4{FRy!T^raE&;vl5P^8o^}4{LE3{9HLSmZu%t*ePEM{~VNT0}Z!8SN9 z;zqD=Tl}PCKQD4`jE#^X|N1vW9qLVxeoW1;3^}zBq%D9Ra)CGErJJ6O$}5$lnH|vB`j$-4npSAmb%m( zH+c6mSCPs=OC;uloQOaviU@hhj2aWJNVOONYkD?>SsR?7#|2Rdn{s<(DgL0OSyU@z zY*bV^|KHdwwTZDdc7z+a@JP3J;xT{!qvt%;(akjs(0glu4FMAPjVG?5kzGO!(vql^ zeH^6-#Mq)fv@(tFu7rRRn)ZnG20 z5+jf-gU>5|lP2d|qXx*Ss4n(}eV<_G+uSMDJYKJlyJV|9>$weIxdTlVaDgEJ>MMW- zl%RWkjU>BLNro`YlLVPup9Xu6%VExhqx@)NNBYs4^wfB z)EdZF(p0h$TWCdAS>INpYzddSLwu`St;?FYl7c(!*sDLE$XD=&x1c}eN6$aZbM?b!=nzi(LMuaK9eFFQRU=Qbv3>h%&Wl4`cdZ zoURPQ6SgU6_13_3&o-;&lq*BWQ~~oiMB52>Z>m73TAiL+j^W zjv$Yx&CpM?EX-;E&3VNGZfuWgR6BgwHrc)HZsYk*IdrYa*ORhw%~v+#>POT#-Z6_G zk-Z;p1`0eLE0H^KBS0rv$>KF+T;sujDi@CcSLSuEt;{D}^E9*>kkbJ#GnecC_ktDS{#3Zg>lM z(7n~Lp%eXKXSkKzJ1UPX+R+DRb?b_EXXL)P-a_cXRR~p0?2(qbjSTcu_Vw^(v zxT(E?Yh(M{mF4zGhfZ|MHpRg2hJX;ueQv0)8{|_Dxu9DuZ;WF&!}XRzr}JU(ec!?B zw6e>;qj|#>5Bv)S$Aqc1HY8}lWEH0fwMkF0K7_vl)fFF1v9d04s~Y*^Ri|0YMLqSb)kG?si51q~yQ~I5o zegX(QJ@gNrJ-cmhVGCP5#zQ_Z)8qYjdS@Z4wLavEP#t!$yH=LL|H_LW)cy#zZ$d$N zrx6)R!I-+w{e`Va=k6DvfW7lw)4OnW#(%W7l+<48AtyZKYe@X;x?lc=2SuspUyhgW z_~hy6-4G1oe~lU;GlMquD?p!1Gtw(M1$;B3OQ|&@7?v6n1DwG1qdE($6zan!;M=;f zu!|Gxm0!6MKt?%ox@CLevAp zFJwKY_!y;AGtf&zEv&#GyfDD)!51`^S-P4J)SeJ~G-JRw|EhT%Hi*L(l!ND+xIn(GN7bLG#;|*V4fqTtqnBJ@-%vN@PPzY(q>;LKlz&C+tLCv$OqkLd%oCoYOgq zQA1Qr#R^lipesEFT)@&p6q_+U0uwh?T*C>=M5-&lK#aq`V7}K8mgvchWx|yZ(TA(? zgTzQifpa?^36Wk547xLg8PtawWTl;W!&7TSM;r!4(8fso8mybdOVq{O(?w5AM|A`* zP)x_k2}SxVMQ2M70>njoyhRq^!ZqVU{GvtH<3a-C#|_{@F1)s%Q?oKmGlH}LePl?5 zOg|vH$B4YgJZqV7yhuH22Q4a!7Tg5c!b9XSF=l+G{}dFEW(>T*i#|xh$Vo^NxH}T_ zu|~7;h=+?fNbCYY=*BCl3>zW8j`K;DWFKh*!rLoH_H(HrBtoTJ$8tDBb+i+ho4mwQ zERi@fiqxr!#K)LANNSr%sJkVc@k)riO0qmh&9X>ej7B^H#x1Iv7wHyA`z<}hqUK`* z0(icTtOFk89{ZR{xFZs3c?^o7VR|zkVpEWNA|qY`lQeK+)*CwQ6K$LR4mKc1W`!T zNdNqUOpBkR;?1QJ#$owQE$YJ%l}_2J&RuAf@`+HultJaJv3yE^Nb67+tb}9uj6aBm z>9kTJRZ}+2J2-mGKQl);HPQT`t`v>A|CM5jbaaIBbi67o#Q}6r*fh$l%uz!H(nH13 z!Rv;*sUPYbG~3LSlB81LYK8!{1d&`my1Y$m!pqzQMB##kmpGrCNTa!QgK#n~HUU!& zJxomqt_I}=F};KjO-oySR9}qIFqzCd?NwiWygeC*Jq0#h{k8M71<$L;L|s-zjZZ{< zR%ngZHRRC;8`6$jn;eNZYV$pA#n!bk(%US?N_DXkxr8Ql7Zg0w!Jz``qtuUdR7g0P zERD%irO;KlS67&WoYbDhB*qYJQ-EDlT-8q+;Rf%dQ!wGxgDt%4`qjT_M~B6~K5e-L zpwTj0tN{ouX5Cnhl}OT*){a#_|7#6O;Nv)SN;|{=5WJuRZ*dw-@H%xhNdoayxM5H) zI-TXz(t6zo%E&{0-Ly2VRe`-#q7~YAT7Za6S`|%)*wfQty|rQ$uLqS!&^*hKy;_e2 zS*_jLkOjVy&6dB*I+LwGOhJG`I8dTcrrqR`xyS@8s;gN_v6zJ!#RSJBh07`pyg!W( zGJ6w+Q-!f(6TDjhpWVp}oK^V=*r8oq#U0q9?Te*VTFPu%TpCsKD+MhpP_+zLsD*iD zg~b@3aMD?LU>jeMGVa?N|3MIjSk0r%(A~#1TI$`;>TOi)<-p)mkjTBvU?tC}1kcMr z6Y`x7^F4qE?Wh1q%3V#^IBhASEM3zTUBZ*2Ze1H~y+US8l^TL?_gycTwC7I$NW85_iN3I#-P-VoJ{x@}#@ zz250vVHMuqg6-aVqTHwr&xw^>{0oDSFd2MbVNTg)YU&3|O zwu3{&P23h{VIrPm|LLk@JkDMh)?U1U;hy>4r)@$QCIF1}V<0}{)O}w|xz;R3Vb_gD zpu{>OK4Nc0Vvb9IK71Kt5Lb6KxJZV8CB|Kq?L*(=%5B{RV&IFsEXH&?&fskfY7CUD z$v`LRzGgYlVdzCwdt}D#Wi6&-J1*M2>@DeXyi1zlUWGtEc7Z}RWM?KQI85STX30DT zW=8J4NoHdo3FS=TU2$Yf+@;%9^NW&YI~=*&XMAE;uEs)Sg=;}y4W8!0)dULOrp>5l zYo2Cd*5hj4V}Iu3e}*Tn`z3;w0&`I2Kn}L+GmaC=u^Amde})FS?M@< zU~ryWsCr~B?zm1S-mO|VH<)Sa)K@o_zG%$pi-zjCZfbtMYq|z#y%y@de&`Jj*3)9S zQ5@v@8i1*GWM8&qX+~_CK5WNk>_~)CnSJZ84h2Tw8WN~v5tdSth8@M;I2YSEN)4Y? zx?nNZzO|k{#LmP2JYm|@j4ys{*_LYBu50GvYZoqRfu5f(LFk317{jJ&o^I^gChndt zZsbnx|G7EBIb0LUHn;|DhHO4%=M!tsZrtbt66pkl;(civ1Yt>D?7(PM68_MjRc`dQ z>%PYA^=@zW-rj+R0vKfkzm9L?tl_0)=!%$a{)X(<{p|h@aQ}W(xOE2+8NW_&o4Uo; zl>(5Xn41QkB>OK=I)gaEvzPZAHidw{0A+RWcZYK)53a&jftFs_HI^1rCG3xwr|J zgwRgO%rJ$ye97hYUwP|r-)rON7Ut*;W)7b2I12I=cW*R@@ibp^1ki0ZKP}5EXe}{3 z|NXuQ9slt=k0Ty8g&KxWUivV8bj1ZSKjMrm8@PL^AhqP#10>H*-Nh^E(DbRczvg4^`_l2rRQfjQ(s)9+jVX~cwTV45oyN&goGnWr_hg>OU0aU|F