[Feature] Add center sampling for indoor datasets (#23)

* try centerness topk configs

* add perspective_ and imvoxelnet_sunrgbd configs with centerness top18

* fix typo in perspective config

* try topk=27 for sunrgbd and scannet

* try more regress ranges for sunrgbd

* try less regress ranges for scannet

* rename indoor configs to _top27

* add resnet 18 and 34 experiments

* ready to merge center sampling to master

* fix typo in readme

* fix topk parameter to 28

README.md

@@ -4,6 +4,7 @@
 # ImVoxelNet: Image to Voxels Projection for Monocular and Multi-View General-Purpose 3D Object Detection
 
 **News**:
+ * :fire: August, 2021. We adapt center sampling for indoor detection. New configs are in [configs/imvoxelnet/*_top27.py](https://github.com/saic-vul/imvoxelnet/tree/master/configs/imvoxelnet). For example, this improves `ScanNet` `mAP` by more than 5%. Models and preprint will be updated soon. 
  * :fire: July, 2021. We update `ScanNet` image preprocessing both [here](https://github.com/saic-vul/imvoxelnet/pull/21) and in [mmdetection3d](https://github.com/open-mmlab/mmdetection3d/pull/696).
  * :fire: June, 2021. `ImVoxelNet` for `KITTI` is now [supported](https://github.com/open-mmlab/mmdetection3d/tree/master/configs/imvoxelnet) in [mmdetection3d](https://github.com/open-mmlab/mmdetection3d).
 

configs/imvoxelnet/imvoxelnet_perspective_sunrgbd_top27.py

@@ -0,0 +1,128 @@
+model = dict(
+    type='ImVoxelNet',
+    pretrained='torchvision://resnet50',
+    backbone=dict(
+        type='ResNet',
+        depth=50,
+        num_stages=4,
+        out_indices=(0, 1, 2, 3),
+        frozen_stages=1,
+        norm_cfg=dict(type='BN', requires_grad=False),
+        norm_eval=True,
+        style='pytorch'),
+    neck=dict(
+        type='FPN',
+        in_channels=[256, 512, 1024, 2048],
+        out_channels=64,
+        num_outs=4),
+    neck_3d=dict(
+        type='ImVoxelNeck',
+        channels=[64, 128, 256, 512],
+        out_channels=64,
+        down_layers=[1, 2, 3, 4],
+        up_layers=[3, 2, 1],
+        conditional=False),
+    bbox_head=dict(
+        type='SunRgbdImVoxelHead',
+        n_classes=30,
+        n_channels=64,
+        n_convs=0,
+        n_reg_outs=7,
+        centerness_topk=28),
+    n_voxels=(80, 80, 32),
+    voxel_size=(.08, .08, .08))
+train_cfg = dict()
+test_cfg = dict(
+    nms_pre=1000,
+    nms_thr=.15,
+    use_rotate_nms=True,
+    score_thr=.0)
+img_norm_cfg = dict(mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True)
+
+dataset_type = 'SunRgbdPerspectiveMultiViewDataset'
+data_root = 'data/sunrgbd/'
+class_names = ('recycle_bin', 'cpu', 'paper', 'toilet', 'stool', 'whiteboard', 'coffee_table', 'picture',
+               'keyboard', 'dresser', 'painting', 'bookshelf', 'night_stand', 'endtable', 'drawer', 'sink',
+               'monitor', 'computer', 'cabinet', 'shelf', 'lamp', 'garbage_bin', 'box', 'bed', 'sofa',
+               'sofa_chair', 'pillow', 'desk', 'table', 'chair')
+
+train_pipeline = [
+    dict(type='LoadAnnotations3D'),
+    dict(
+        type='MultiViewPipeline',
+        n_images=1,
+        transforms=[
+            dict(type='LoadImageFromFile'),
+            dict(type='RandomFlip', flip_ratio=0.5),
+            dict(type='Resize', img_scale=[(512, 384), (768, 576)], multiscale_mode='range', keep_ratio=True),
+            dict(type='Normalize', **img_norm_cfg),
+            dict(type='Pad', size_divisor=32)]),
+    dict(type='SunRgbdRandomFlip'),
+    dict(type='DefaultFormatBundle3D', class_names=class_names),
+    dict(type='Collect3D', keys=['img', 'gt_bboxes_3d', 'gt_labels_3d'])]
+test_pipeline = [
+    dict(
+        type='MultiViewPipeline',
+        n_images=1,
+        transforms=[
+            dict(type='LoadImageFromFile'),
+            dict(type='Resize', img_scale=(640, 480), keep_ratio=True),
+            dict(type='Normalize', **img_norm_cfg),
+            dict(type='Pad', size_divisor=32)]),
+    dict(type='DefaultFormatBundle3D', class_names=class_names, with_label=False),
+    dict(type='Collect3D', keys=['img'])]
+data = dict(
+    samples_per_gpu=4,
+    workers_per_gpu=4,
+    train=dict(
+        type='RepeatDataset',
+        times=2,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_root,
+            ann_file=data_root + 'sunrgbd_perspective_infos_train.pkl',
+            pipeline=train_pipeline,
+            classes=class_names,
+            filter_empty_gt=True,
+            box_type_3d='Depth')),
+    val=dict(
+        type=dataset_type,
+        data_root=data_root,
+        ann_file=data_root + 'sunrgbd_perspective_infos_val.pkl',
+        pipeline=test_pipeline,
+        classes=class_names,
+        test_mode=True,
+        box_type_3d='Depth'),
+    test=dict(
+        type=dataset_type,
+        data_root=data_root,
+        ann_file=data_root + 'sunrgbd_perspective_infos_val.pkl',
+        pipeline=test_pipeline,
+        classes=class_names,
+        test_mode=True,
+        box_type_3d='Depth'))
+
+optimizer = dict(
+    type='AdamW',
+    lr=0.0001,
+    weight_decay=0.0001,
+    paramwise_cfg=dict(
+        custom_keys={'backbone': dict(lr_mult=0.1, decay_mult=1.0)}))
+optimizer_config = dict(grad_clip=dict(max_norm=35., norm_type=2))
+lr_config = dict(policy='step', step=[8, 11])
+total_epochs = 12
+
+checkpoint_config = dict(interval=1, max_keep_ckpts=1)
+log_config = dict(
+    interval=50,
+    hooks=[
+        dict(type='TextLoggerHook'),
+        dict(type='TensorboardLoggerHook')
+    ])
+evaluation = dict(interval=1)
+dist_params = dict(backend='nccl')
+find_unused_parameters = True  # todo: fix number of FPN outputs
+log_level = 'INFO'
+load_from = None
+resume_from = None
+workflow = [('train', 1)]

configs/imvoxelnet/imvoxelnet_scannet_top27.py

@@ -0,0 +1,132 @@
+model = dict(
+    type='ImVoxelNet',
+    pretrained='torchvision://resnet50',
+    backbone=dict(
+        type='ResNet',
+        depth=50,
+        num_stages=4,
+        out_indices=(0, 1, 2, 3),
+        frozen_stages=1,
+        norm_cfg=dict(type='BN', requires_grad=False),
+        norm_eval=True,
+        style='pytorch'),
+    neck=dict(
+        type='FPN',
+        in_channels=[256, 512, 1024, 2048],
+        out_channels=64,
+        num_outs=4),
+    neck_3d=dict(
+        type='ImVoxelNeck',
+        channels=[64, 128, 256, 512],
+        out_channels=64,
+        down_layers=[1, 2, 3, 4],
+        up_layers=[3, 2, 1],
+        conditional=False),
+    bbox_head=dict(
+        type='ScanNetImVoxelHead',
+        loss_bbox=dict(type='AxisAlignedIoULoss', loss_weight=1.0),
+        n_classes=18,
+        n_channels=64,
+        n_convs=0,
+        n_reg_outs=6,
+        centerness_topk=28),
+    voxel_size=(.08, .08, .08),
+    n_voxels=(80, 80, 32))
+train_cfg = dict()
+test_cfg = dict(
+    nms_pre=1000,
+    iou_thr=.15,
+    score_thr=.0)
+img_norm_cfg = dict(mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True)
+
+dataset_type = 'ScanNetMultiViewDataset'
+data_root = 'data/scannet/'
+class_names = ('cabinet', 'bed', 'chair', 'sofa', 'table', 'door', 'window',
+               'bookshelf', 'picture', 'counter', 'desk', 'curtain',
+               'refrigerator', 'showercurtrain', 'toilet', 'sink', 'bathtub',
+               'garbagebin')
+
+train_pipeline = [
+    dict(type='LoadAnnotations3D'),
+    dict(
+        type='MultiViewPipeline',
+        n_images=20,
+        transforms=[
+            dict(type='LoadImageFromFile'),
+            dict(type='Resize', img_scale=(640, 480), keep_ratio=True),
+            dict(type='Normalize', **img_norm_cfg),
+            dict(type='Pad', size=(480, 640))
+        ]),
+    dict(type='RandomShiftOrigin', std=(.7, .7, .0)),
+    dict(type='DefaultFormatBundle3D', class_names=class_names),
+    dict(type='Collect3D', keys=['img', 'gt_bboxes_3d', 'gt_labels_3d'])
+]
+test_pipeline = [
+    dict(
+        type='MultiViewPipeline',
+        n_images=50,
+        transforms=[
+            dict(type='LoadImageFromFile'),
+            dict(type='Resize', img_scale=(640, 480), keep_ratio=True),
+            dict(type='Normalize', **img_norm_cfg),
+            dict(type='Pad', size=(480, 640))
+        ]),
+    dict(type='DefaultFormatBundle3D', class_names=class_names, with_label=False),
+    dict(type='Collect3D', keys=['img'])
+]
+data = dict(
+    samples_per_gpu=1,
+    workers_per_gpu=1,
+    train=dict(
+        type='RepeatDataset',
+        times=3,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_root,
+            ann_file=data_root + 'scannet_infos_train.pkl',
+            pipeline=train_pipeline,
+            classes=class_names,
+            filter_empty_gt=True,
+            box_type_3d='Depth')),
+    val=dict(
+        type=dataset_type,
+        data_root=data_root,
+        ann_file=data_root + 'scannet_infos_val.pkl',
+        pipeline=test_pipeline,
+        classes=class_names,
+        test_mode=True,
+        box_type_3d='Depth'),
+    test=dict(
+        type=dataset_type,
+        data_root=data_root,
+        ann_file=data_root + 'scannet_infos_val.pkl',
+        pipeline=test_pipeline,
+        classes=class_names,
+        test_mode=True,
+        box_type_3d='Depth')
+)
+
+optimizer = dict(
+    type='AdamW',
+    lr=0.0001,
+    weight_decay=0.0001,
+    paramwise_cfg=dict(
+        custom_keys={'backbone': dict(lr_mult=0.1, decay_mult=1.0)}))
+optimizer_config = dict(grad_clip=dict(max_norm=35., norm_type=2))
+lr_config = dict(policy='step', step=[8, 11])
+total_epochs = 12
+
+checkpoint_config = dict(interval=1, max_keep_ckpts=1)
+log_config = dict(
+    interval=50,
+    hooks=[
+        dict(type='TextLoggerHook'),
+        dict(type='TensorboardLoggerHook')
+    ])
+evaluation = dict(interval=1)
+dist_params = dict(backend='nccl')
+find_unused_parameters = True  # todo: fix number of FPN outputs
+log_level = 'INFO'
+load_from = None
+resume_from = None
+workflow = [('train', 1)]

configs/imvoxelnet/imvoxelnet_sunrgbd_top27.py

@@ -0,0 +1,126 @@
+model = dict(
+    type='ImVoxelNet',
+    pretrained='torchvision://resnet50',
+    backbone=dict(
+        type='ResNet',
+        depth=50,
+        num_stages=4,
+        out_indices=(0, 1, 2, 3),
+        frozen_stages=1,
+        norm_cfg=dict(type='BN', requires_grad=False),
+        norm_eval=True,
+        style='pytorch'),
+    neck=dict(
+        type='FPN',
+        in_channels=[256, 512, 1024, 2048],
+        out_channels=64,
+        num_outs=4),
+    neck_3d=dict(
+        type='ImVoxelNeck',
+        channels=[64, 128, 256, 512],
+        out_channels=64,
+        down_layers=[1, 2, 3, 4],
+        up_layers=[3, 2, 1],
+        conditional=False),
+    bbox_head=dict(
+        type='SunRgbdImVoxelHead',
+        n_classes=10,
+        n_channels=64,
+        n_convs=0,
+        n_reg_outs=7,
+        centerness_topk=28),
+    n_voxels=(80, 80, 32),
+    voxel_size=(.08, .08, .08))
+train_cfg = dict()
+test_cfg = dict(
+    nms_pre=1000,
+    nms_thr=.15,
+    use_rotate_nms=True,
+    score_thr=.0)
+img_norm_cfg = dict(mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True)
+
+dataset_type = 'SunRgbdMultiViewDataset'
+data_root = 'data/sunrgbd/'
+class_names = ('bed', 'table', 'sofa', 'chair', 'toilet', 'desk', 'dresser',
+               'night_stand', 'bookshelf', 'bathtub')
+
+train_pipeline = [
+    dict(type='LoadAnnotations3D'),
+    dict(
+        type='MultiViewPipeline',
+        n_images=1,
+        transforms=[
+            dict(type='LoadImageFromFile'),
+            dict(type='RandomFlip', flip_ratio=0.5),
+            dict(type='Resize', img_scale=[(512, 384), (768, 576)], multiscale_mode='range', keep_ratio=True),
+            dict(type='Normalize', **img_norm_cfg),
+            dict(type='Pad', size_divisor=32)]),
+    dict(type='SunRgbdRandomFlip'),
+    dict(type='DefaultFormatBundle3D', class_names=class_names),
+    dict(type='Collect3D', keys=['img', 'gt_bboxes_3d', 'gt_labels_3d'])]
+test_pipeline = [
+    dict(
+        type='MultiViewPipeline',
+        n_images=1,
+        transforms=[
+            dict(type='LoadImageFromFile'),
+            dict(type='Resize', img_scale=(640, 480), keep_ratio=True),
+            dict(type='Normalize', **img_norm_cfg),
+            dict(type='Pad', size_divisor=32)]),
+    dict(type='DefaultFormatBundle3D', class_names=class_names, with_label=False),
+    dict(type='Collect3D', keys=['img'])]
+data = dict(
+    samples_per_gpu=4,
+    workers_per_gpu=4,
+    train=dict(
+        type='RepeatDataset',
+        times=2,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_root,
+            ann_file=data_root + 'sunrgbd_imvoxelnet_infos_train.pkl',
+            pipeline=train_pipeline,
+            classes=class_names,
+            filter_empty_gt=True,
+            box_type_3d='Depth')),
+    val=dict(
+        type=dataset_type,
+        data_root=data_root,
+        ann_file=data_root + 'sunrgbd_imvoxelnet_infos_val.pkl',
+        pipeline=test_pipeline,
+        classes=class_names,
+        test_mode=True,
+        box_type_3d='Depth'),
+    test=dict(
+        type=dataset_type,
+        data_root=data_root,
+        ann_file=data_root + 'sunrgbd_imvoxelnet_infos_val.pkl',
+        pipeline=test_pipeline,
+        classes=class_names,
+        test_mode=True,
+        box_type_3d='Depth'))
+
+optimizer = dict(
+    type='AdamW',
+    lr=0.0001,
+    weight_decay=0.0001,
+    paramwise_cfg=dict(
+        custom_keys={'backbone': dict(lr_mult=0.1, decay_mult=1.0)}))
+optimizer_config = dict(grad_clip=dict(max_norm=35., norm_type=2))
+lr_config = dict(policy='step', step=[8, 11])
+total_epochs = 12
+
+checkpoint_config = dict(interval=1, max_keep_ckpts=1)
+log_config = dict(
+    interval=50,
+    hooks=[
+        dict(type='TextLoggerHook'),
+        dict(type='TensorboardLoggerHook')
+    ])
+evaluation = dict(interval=1)
+dist_params = dict(backend='nccl')
+find_unused_parameters = True  # todo: fix number of FPN outputs
+log_level = 'INFO'
+load_from = None
+resume_from = None
+workflow = [('train', 1)]