# Copyright (c) OpenMMLab. All rights reserved.
# Adapt from
# https://github.com/open-mmlab/mmpose/blob/master/mmpose/datasets/datasets/hand/hand_coco_wholebody_dataset.py
import logging
import os.path as osp
import numpy as np
from easycv.datasets.registry import DATASOURCES
from ..top_down import PoseTopDownSource
COCO_WHOLEBODY_HAND_DATASET_INFO = dict(
dataset_name='coco_wholebody_hand',
paper_info=dict(
author='Jin, Sheng and Xu, Lumin and Xu, Jin and '
'Wang, Can and Liu, Wentao and '
'Qian, Chen and Ouyang, Wanli and Luo, Ping',
title='Whole-Body Human Pose Estimation in the Wild',
container='Proceedings of the European '
'Conference on Computer Vision (ECCV)',
year='2020',
homepage='https://github.com/jin-s13/COCO-WholeBody/',
),
keypoint_info={
0:
dict(name='wrist', id=0, color=[255, 255, 255], type='', swap=''),
1:
dict(name='thumb1', id=1, color=[255, 128, 0], type='', swap=''),
2:
dict(name='thumb2', id=2, color=[255, 128, 0], type='', swap=''),
3:
dict(name='thumb3', id=3, color=[255, 128, 0], type='', swap=''),
4:
dict(name='thumb4', id=4, color=[255, 128, 0], type='', swap=''),
5:
dict(
name='forefinger1', id=5, color=[255, 153, 255], type='', swap=''),
6:
dict(
name='forefinger2', id=6, color=[255, 153, 255], type='', swap=''),
7:
dict(
name='forefinger3', id=7, color=[255, 153, 255], type='', swap=''),
8:
dict(
name='forefinger4', id=8, color=[255, 153, 255], type='', swap=''),
9:
dict(
name='middle_finger1',
id=9,
color=[102, 178, 255],
type='',
swap=''),
10:
dict(
name='middle_finger2',
id=10,
color=[102, 178, 255],
type='',
swap=''),
11:
dict(
name='middle_finger3',
id=11,
color=[102, 178, 255],
type='',
swap=''),
12:
dict(
name='middle_finger4',
id=12,
color=[102, 178, 255],
type='',
swap=''),
13:
dict(
name='ring_finger1', id=13, color=[255, 51, 51], type='', swap=''),
14:
dict(
name='ring_finger2', id=14, color=[255, 51, 51], type='', swap=''),
15:
dict(
name='ring_finger3', id=15, color=[255, 51, 51], type='', swap=''),
16:
dict(
name='ring_finger4', id=16, color=[255, 51, 51], type='', swap=''),
17:
dict(name='pinky_finger1', id=17, color=[0, 255, 0], type='', swap=''),
18:
dict(name='pinky_finger2', id=18, color=[0, 255, 0], type='', swap=''),
19:
dict(name='pinky_finger3', id=19, color=[0, 255, 0], type='', swap=''),
20:
dict(name='pinky_finger4', id=20, color=[0, 255, 0], type='', swap='')
},
skeleton_info={
0:
dict(link=('wrist', 'thumb1'), id=0, color=[255, 128, 0]),
1:
dict(link=('thumb1', 'thumb2'), id=1, color=[255, 128, 0]),
2:
dict(link=('thumb2', 'thumb3'), id=2, color=[255, 128, 0]),
3:
dict(link=('thumb3', 'thumb4'), id=3, color=[255, 128, 0]),
4:
dict(link=('wrist', 'forefinger1'), id=4, color=[255, 153, 255]),
5:
dict(link=('forefinger1', 'forefinger2'), id=5, color=[255, 153, 255]),
6:
dict(link=('forefinger2', 'forefinger3'), id=6, color=[255, 153, 255]),
7:
dict(link=('forefinger3', 'forefinger4'), id=7, color=[255, 153, 255]),
8:
dict(link=('wrist', 'middle_finger1'), id=8, color=[102, 178, 255]),
9:
dict(
link=('middle_finger1', 'middle_finger2'),
id=9,
color=[102, 178, 255]),
10:
dict(
link=('middle_finger2', 'middle_finger3'),
id=10,
color=[102, 178, 255]),
11:
dict(
link=('middle_finger3', 'middle_finger4'),
id=11,
color=[102, 178, 255]),
12:
dict(link=('wrist', 'ring_finger1'), id=12, color=[255, 51, 51]),
13:
dict(
link=('ring_finger1', 'ring_finger2'), id=13, color=[255, 51, 51]),
14:
dict(
link=('ring_finger2', 'ring_finger3'), id=14, color=[255, 51, 51]),
15:
dict(
link=('ring_finger3', 'ring_finger4'), id=15, color=[255, 51, 51]),
16:
dict(link=('wrist', 'pinky_finger1'), id=16, color=[0, 255, 0]),
17:
dict(
link=('pinky_finger1', 'pinky_finger2'), id=17, color=[0, 255, 0]),
18:
dict(
link=('pinky_finger2', 'pinky_finger3'), id=18, color=[0, 255, 0]),
19:
dict(
link=('pinky_finger3', 'pinky_finger4'), id=19, color=[0, 255, 0])
},
joint_weights=[1.] * 21,
sigmas=[
0.029, 0.022, 0.035, 0.037, 0.047, 0.026, 0.025, 0.024, 0.035, 0.018,
0.024, 0.022, 0.026, 0.017, 0.021, 0.021, 0.032, 0.02, 0.019, 0.022,
0.031
])
[docs]@DATASOURCES.register_module()
class HandCocoPoseTopDownSource(PoseTopDownSource):
"""Coco Whole-Body-Hand Source for top-down hand pose estimation.
"Whole-Body Human Pose Estimation in the Wild", ECCV'2020.
More details can be found in the `paper
<https://arxiv.org/abs/2007.11858>`__ .
The dataset loads raw features and apply specified transforms
to return a dict containing the image tensors and other information.
COCO-WholeBody Hand keypoint indexes::
0: 'wrist',
1: 'thumb1',
2: 'thumb2',
3: 'thumb3',
4: 'thumb4',
5: 'forefinger1',
6: 'forefinger2',
7: 'forefinger3',
8: 'forefinger4',
9: 'middle_finger1',
10: 'middle_finger2',
11: 'middle_finger3',
12: 'middle_finger4',
13: 'ring_finger1',
14: 'ring_finger2',
15: 'ring_finger3',
16: 'ring_finger4',
17: 'pinky_finger1',
18: 'pinky_finger2',
19: 'pinky_finger3',
20: 'pinky_finger4'
Args:
ann_file (str): Path to the annotation file.
img_prefix (str): Path to a directory where images are held.
Default: None.
data_cfg (dict): config
dataset_info (DatasetInfo): A class containing all dataset info.
test_mode (bool): Store True when building test or
validation dataset. Default: False.
"""
[docs] def __init__(self,
ann_file,
img_prefix,
data_cfg,
dataset_info=None,
test_mode=False):
if dataset_info is None:
logging.info(
'dataset_info is missing, use default coco wholebody hand dataset info'
)
dataset_info = COCO_WHOLEBODY_HAND_DATASET_INFO
super().__init__(
ann_file,
img_prefix,
data_cfg,
dataset_info=dataset_info,
test_mode=test_mode)
self.ann_info['use_different_joint_weights'] = False
self.db = self._get_db()
print(f'=> num_images: {self.num_images}')
print(f'=> load {len(self.db)} samples')
def _get_db(self):
"""Load dataset."""
gt_db = []
bbox_id = 0
num_joints = self.ann_info['num_joints']
for img_id in self.img_ids:
ann_ids = self.coco.getAnnIds(imgIds=img_id, iscrowd=False)
objs = self.coco.loadAnns(ann_ids)
for obj in objs:
for type in ['left', 'right']:
if obj[f'{type}hand_valid'] and max(
obj[f'{type}hand_kpts']) > 0:
joints_3d = np.zeros((num_joints, 3), dtype=np.float32)
joints_3d_visible = np.zeros((num_joints, 3),
dtype=np.float32)
keypoints = np.array(obj[f'{type}hand_kpts']).reshape(
-1, 3)
joints_3d[:, :2] = keypoints[:, :2]
joints_3d_visible[:, :2] = np.minimum(
1, keypoints[:, 2:3])
image_file = osp.join(self.img_prefix,
self.id2name[img_id])
center, scale = self._xywh2cs(
*obj[f'{type}hand_box'][:4])
gt_db.append({
'image_file': image_file,
'image_id': img_id,
'rotation': 0,
'center': center,
'scale': scale,
'joints_3d': joints_3d,
'joints_3d_visible': joints_3d_visible,
'dataset': self.dataset_name,
'bbox': obj[f'{type}hand_box'],
'bbox_score': 1,
'bbox_id': bbox_id
})
bbox_id = bbox_id + 1
gt_db = sorted(gt_db, key=lambda x: x['bbox_id'])
return gt_db