# Copyright (c) OpenMMLab. All rights reserved. import os.path as osp import tempfile import mmcv import numpy as np from mmdet.datasets.coco_panoptic import INSTANCE_OFFSET, CocoPanopticDataset try: from panopticapi.utils import id2rgb except ImportError: id2rgb = None def _create_panoptic_style_json(json_name): image1 = { 'id': 0, 'width': 640, 'height': 640, 'file_name': 'fake_name1.jpg', } image2 = { 'id': 1, 'width': 640, 'height': 800, 'file_name': 'fake_name2.jpg', } images = [image1, image2] annotations = [ { 'segments_info': [{ 'id': 1, 'category_id': 0, 'area': 400, 'bbox': [50, 60, 20, 20], 'iscrowd': 0 }, { 'id': 2, 'category_id': 1, 'area': 900, 'bbox': [100, 120, 30, 30], 'iscrowd': 0 }, { 'id': 3, 'category_id': 2, 'iscrowd': 0, 'bbox': [1, 189, 612, 285], 'area': 70036 }], 'file_name': 'fake_name1.jpg', 'image_id': 0 }, { 'segments_info': [ { # Different to instance style json, there # are duplicate ids in panoptic style json 'id': 1, 'category_id': 0, 'area': 400, 'bbox': [50, 60, 20, 20], 'iscrowd': 0 }, { 'id': 4, 'category_id': 1, 'area': 900, 'bbox': [100, 120, 30, 30], 'iscrowd': 1 }, { 'id': 5, 'category_id': 2, 'iscrowd': 0, 'bbox': [100, 200, 200, 300], 'area': 66666 }, { 'id': 6, 'category_id': 0, 'iscrowd': 0, 'bbox': [1, 189, -10, 285], 'area': 70036 } ], 'file_name': 'fake_name2.jpg', 'image_id': 1 } ] categories = [{ 'id': 0, 'name': 'car', 'supercategory': 'car', 'isthing': 1 }, { 'id': 1, 'name': 'person', 'supercategory': 'person', 'isthing': 1 }, { 'id': 2, 'name': 'wall', 'supercategory': 'wall', 'isthing': 0 }] fake_json = { 'images': images, 'annotations': annotations, 'categories': categories } mmcv.dump(fake_json, json_name) return fake_json def test_load_panoptic_style_json(): tmp_dir = tempfile.TemporaryDirectory() fake_json_file = osp.join(tmp_dir.name, 'fake_data.json') fake_json = _create_panoptic_style_json(fake_json_file) dataset = CocoPanopticDataset( ann_file=fake_json_file, classes=[cat['name'] for cat in fake_json['categories']], pipeline=[]) ann = dataset.get_ann_info(0) # two legal instances assert ann['bboxes'].shape[0] == ann['labels'].shape[0] == 2 # three masks for both foreground and background assert len(ann['masks']) == 3 ann = dataset.get_ann_info(1) # one legal instance, one illegal instance, # one crowd instance and one background mask assert ann['bboxes'].shape[0] == ann['labels'].shape[0] == 1 assert ann['bboxes_ignore'].shape[0] == 1 assert len(ann['masks']) == 3 def _create_panoptic_gt_annotations(ann_file): categories = [{ 'id': 0, 'name': 'person', 'supercategory': 'person', 'isthing': 1 }, { 'id': 1, 'name': 'dog', 'supercategory': 'dog', 'isthing': 1 }, { 'id': 2, 'name': 'wall', 'supercategory': 'wall', 'isthing': 0 }] images = [{ 'id': 0, 'width': 80, 'height': 60, 'file_name': 'fake_name1.jpg', }] annotations = [{ 'segments_info': [{ 'id': 1, 'category_id': 0, 'area': 400, 'bbox': [10, 10, 10, 40], 'iscrowd': 0 }, { 'id': 2, 'category_id': 0, 'area': 400, 'bbox': [30, 10, 10, 40], 'iscrowd': 0 }, { 'id': 3, 'category_id': 1, 'iscrowd': 0, 'bbox': [50, 10, 10, 5], 'area': 50 }, { 'id': 4, 'category_id': 2, 'iscrowd': 0, 'bbox': [0, 0, 80, 60], 'area': 3950 }], 'file_name': 'fake_name1.png', 'image_id': 0 }] gt_json = { 'images': images, 'annotations': annotations, 'categories': categories } # 4 is the id of the background class annotation. gt = np.zeros((60, 80), dtype=np.int64) + 4 gt_bboxes = np.array([[10, 10, 10, 40], [30, 10, 10, 40], [50, 10, 10, 5]], dtype=np.int64) for i in range(3): x, y, w, h = gt_bboxes[i] gt[y:y + h, x:x + w] = i + 1 # id starts from 1 gt = id2rgb(gt).astype(np.uint8) img_path = osp.join(osp.dirname(ann_file), 'fake_name1.png') mmcv.imwrite(gt[:, :, ::-1], img_path) mmcv.dump(gt_json, ann_file) return gt_json def test_panoptic_evaluation(): if id2rgb is None: return # TP for background class, IoU=3576/4324=0.827 # 2 the category id of the background class pred = np.zeros((60, 80), dtype=np.int64) + 2 pred_bboxes = np.array( [ [11, 11, 10, 40], # TP IoU=351/449=0.78 [38, 10, 10, 40], # FP [51, 10, 10, 5] ], # TP IoU=45/55=0.818 dtype=np.int64) pred_labels = np.array([0, 0, 1], dtype=np.int64) for i in range(3): x, y, w, h = pred_bboxes[i] pred[y:y + h, x:x + w] = (i + 1) * INSTANCE_OFFSET + pred_labels[i] tmp_dir = tempfile.TemporaryDirectory() ann_file = osp.join(tmp_dir.name, 'panoptic.json') gt_json = _create_panoptic_gt_annotations(ann_file) results = [{'pan_results': pred}] dataset = CocoPanopticDataset( ann_file=ann_file, seg_prefix=tmp_dir.name, classes=[cat['name'] for cat in gt_json['categories']], pipeline=[]) # For 'person', sq = 0.78 / 1, rq = 1 / 2( 1 tp + 0.5 * (1 fn + 1 fp)) # For 'dog', sq = 0.818, rq = 1 / 1 # For 'wall', sq = 0.827, rq = 1 / 1 # Here is the results for all classes: # +--------+--------+--------+---------+------------+ # | | PQ | SQ | RQ | categories | # +--------+--------+--------+---------+------------+ # | All | 67.869 | 80.898 | 83.333 | 3 | # | Things | 60.453 | 79.996 | 75.000 | 2 | # | Stuff | 82.701 | 82.701 | 100.000 | 1 | # +--------+--------+--------+---------+------------+ parsed_results = dataset.evaluate(results) assert np.isclose(parsed_results['PQ'], 67.869) assert np.isclose(parsed_results['SQ'], 80.898) assert np.isclose(parsed_results['RQ'], 83.333) assert np.isclose(parsed_results['PQ_th'], 60.453) assert np.isclose(parsed_results['SQ_th'], 79.996) assert np.isclose(parsed_results['RQ_th'], 75.000) assert np.isclose(parsed_results['PQ_st'], 82.701) assert np.isclose(parsed_results['SQ_st'], 82.701) assert np.isclose(parsed_results['RQ_st'], 100.000) # test jsonfile_prefix outfile_prefix = osp.join(tmp_dir.name, 'results') parsed_results = dataset.evaluate(results, jsonfile_prefix=outfile_prefix) assert np.isclose(parsed_results['PQ'], 67.869) assert np.isclose(parsed_results['SQ'], 80.898) assert np.isclose(parsed_results['RQ'], 83.333) assert np.isclose(parsed_results['PQ_th'], 60.453) assert np.isclose(parsed_results['SQ_th'], 79.996) assert np.isclose(parsed_results['RQ_th'], 75.000) assert np.isclose(parsed_results['PQ_st'], 82.701) assert np.isclose(parsed_results['SQ_st'], 82.701) assert np.isclose(parsed_results['RQ_st'], 100.000) # test classwise parsed_results = dataset.evaluate(results, classwise=True) assert np.isclose(parsed_results['PQ'], 67.869) assert np.isclose(parsed_results['SQ'], 80.898) assert np.isclose(parsed_results['RQ'], 83.333) assert np.isclose(parsed_results['PQ_th'], 60.453) assert np.isclose(parsed_results['SQ_th'], 79.996) assert np.isclose(parsed_results['RQ_th'], 75.000) assert np.isclose(parsed_results['PQ_st'], 82.701) assert np.isclose(parsed_results['SQ_st'], 82.701) assert np.isclose(parsed_results['RQ_st'], 100.000)