# Copyright (c) OpenMMLab. All rights reserved. import copy import csv import json import os.path as osp import warnings from collections import OrderedDict, defaultdict import mmcv import numpy as np import torch.distributed as dist from mmcv.runner import get_dist_info from mmcv.utils import print_log from mmdet.core import eval_map from .builder import DATASETS from .custom import CustomDataset @DATASETS.register_module() class OpenImagesDataset(CustomDataset): """Open Images dataset for detection. Args: ann_file (str): Annotation file path. label_file (str): File path of the label description file that maps the classes names in MID format to their short descriptions. image_level_ann_file (str): Image level annotation, which is used in evaluation. get_supercategory (bool): Whether to get parent class of the current class. Default: True. hierarchy_file (str): The file path of the class hierarchy. Default: None. get_metas (bool): Whether to get image metas in testing or validation time. This should be `True` during evaluation. Default: True. The OpenImages annotations do not have image metas (width and height of the image), which will be used during evaluation. We provide two ways to get image metas in `OpenImagesDataset`: - 1. `load from file`: Load image metas from pkl file, which is suggested to use. We provided a script to get image metas: `tools/misc/get_image_metas.py`, which need to run this script before training/testing. Please refer to `config/openimages/README.md` for more details. - 2. `load from pipeline`, which will get image metas during test time. However, this may reduce the inference speed, especially when using distribution. load_from_file (bool): Whether to get image metas from pkl file. meta_file (str): File path to get image metas. filter_labels (bool): Whether filter unannotated classes. Default: True. load_image_level_labels (bool): Whether load and consider image level labels during evaluation. Default: True. file_client_args (dict): Arguments to instantiate a FileClient. See :class:`mmcv.fileio.FileClient` for details. Defaults to ``dict(backend='disk')``. """ def __init__(self, ann_file, label_file='', image_level_ann_file='', get_supercategory=True, hierarchy_file=None, get_metas=True, load_from_file=True, meta_file='', filter_labels=True, load_image_level_labels=True, file_client_args=dict(backend='disk'), **kwargs): # may get error if use other file_client self.file_client_args = file_client_args self.cat2label = defaultdict(str) self.index_dict = {} # Although it will init file_client in `CustomDataset`, # it needs to be init here. file_client = mmcv.FileClient(**file_client_args) # need get `index_dict` before load annotations assert label_file.endswith('csv') if hasattr(file_client, 'get_local_path'): with file_client.get_local_path(label_file) as local_path: class_names = self.get_classes_from_csv(local_path) else: class_names = self.get_classes_from_csv(label_file) super(OpenImagesDataset, self).__init__( ann_file=ann_file, file_client_args=file_client_args, **kwargs) self.CLASSES = class_names self.image_level_ann_file = image_level_ann_file self.load_image_level_labels = load_image_level_labels if get_supercategory is True: assert hierarchy_file is not None if self.__class__.__name__ == 'OpenImagesDataset': assert hierarchy_file.endswith('json') elif self.__class__.__name__ == 'OpenImagesChallengeDataset': assert hierarchy_file.endswith('np') else: raise NotImplementedError if hasattr(self.file_client, 'get_local_path'): with self.file_client.get_local_path( hierarchy_file) as local_path: self.class_label_tree = self.get_relation_matrix( local_path) else: self.class_label_tree = self.get_relation_matrix( hierarchy_file) self.get_supercategory = get_supercategory self.get_metas = get_metas self.load_from_file = load_from_file self.meta_file = meta_file if self.data_root is not None: if not osp.isabs(self.meta_file): self.meta_file = osp.join(self.data_root, self.meta_file) self.filter_labels = filter_labels self.rank, self.world_size = get_dist_info() self.temp_img_metas = [] self.test_img_metas = [] self.test_img_shapes = [] self.load_from_pipeline = False if load_from_file else True def get_classes_from_csv(self, label_file): """Get classes name from file. Args: label_file (str): File path of the label description file that maps the classes names in MID format to their short descriptions. Returns: list[str]: Class name of OpenImages. """ index_list = [] classes_names = [] with open(label_file, 'r') as f: reader = csv.reader(f) for line in reader: self.cat2label[line[0]] = line[1] classes_names.append(line[1]) index_list.append(line[0]) self.index_dict = {index: i for i, index in enumerate(index_list)} return classes_names def load_annotations(self, ann_file): """Load annotation from annotation file. Special described `self.data_infos` (defaultdict[list[dict]]) in this function: Annotations where item of the defaultdict indicates an image, each of which has (n) dicts. Keys of dicts are: - `bbox` (list): coordinates of the box, in normalized image coordinates, of shape 4. - `label` (int): the label id. - `is_group_of` (bool): Indicates that the box spans a group of objects (e.g., a bed of flowers or a crowd of people). - `is_occluded` (bool): Indicates that the object is occluded by another object in the image. - `is_truncated` (bool): Indicates that the object extends beyond the boundary of the image. - `is_depiction` (bool): Indicates that the object is a depiction. - `is_inside` (bool): Indicates a picture taken from the inside of the object. Args: ann_file (str): CSV style annotation file path. Returns: list[dict]: Data infos where each item of the list indicates an image. Keys of annotations are: - `img_id` (str): Image name. - `filename` (str): Image name with suffix. """ self.ann_infos = defaultdict(list) data_infos = [] cp_filename = None with open(ann_file, 'r') as f: reader = csv.reader(f) for i, line in enumerate(reader): if i == 0: continue img_id = line[0] filename = f'{img_id}.jpg' label_id = line[2] assert label_id in self.index_dict label = int(self.index_dict[label_id]) bbox = [ float(line[4]), # xmin float(line[6]), # ymin float(line[5]), # xmax float(line[7]) # ymax ] is_occluded = True if int(line[8]) == 1 else False is_truncated = True if int(line[9]) == 1 else False is_group_of = True if int(line[10]) == 1 else False is_depiction = True if int(line[11]) == 1 else False is_inside = True if int(line[12]) == 1 else False self.ann_infos[img_id].append( dict( bbox=bbox, label=label, is_occluded=is_occluded, is_truncated=is_truncated, is_group_of=is_group_of, is_depiction=is_depiction, is_inside=is_inside)) if filename != cp_filename: data_infos.append(dict(img_id=img_id, filename=filename)) cp_filename = filename return data_infos def get_ann_info(self, idx): """Get OpenImages annotation by index. Args: idx (int): Index of data. Returns: dict: Annotation info of specified index. """ img_id = self.data_infos[idx]['img_id'] bboxes = [] labels = [] bboxes_ignore = [] labels_ignore = [] is_occludeds = [] is_truncateds = [] is_group_ofs = [] is_depictions = [] is_insides = [] for obj in self.ann_infos[img_id]: label = int(obj['label']) bbox = [ float(obj['bbox'][0]), float(obj['bbox'][1]), float(obj['bbox'][2]), float(obj['bbox'][3]) ] bboxes.append(bbox) labels.append(label) # Other parameters is_occludeds.append(obj['is_occluded']) is_truncateds.append(obj['is_truncated']) is_group_ofs.append(obj['is_group_of']) is_depictions.append(obj['is_depiction']) is_insides.append(obj['is_inside']) if not bboxes: bboxes = np.zeros((0, 4)) labels = np.zeros((0, )) else: bboxes = np.array(bboxes) labels = np.array(labels) if not bboxes_ignore: bboxes_ignore = np.zeros((0, 4)) labels_ignore = np.zeros((0, )) else: bboxes_ignore = np.array(bboxes_ignore) labels_ignore = np.array(labels_ignore) assert len(is_group_ofs) == len(labels) == len(bboxes) gt_is_group_ofs = np.array(is_group_ofs, dtype=bool) # These parameters is not used yet. is_occludeds = np.array(is_occludeds, dtype=bool) is_truncateds = np.array(is_truncateds, dtype=bool) is_depictions = np.array(is_depictions, dtype=bool) is_insides = np.array(is_insides, dtype=bool) ann = dict( bboxes=bboxes.astype(np.float32), labels=labels.astype(np.int64), bboxes_ignore=bboxes_ignore.astype(np.float32), labels_ignore=labels_ignore.astype(np.int64), gt_is_group_ofs=gt_is_group_ofs, is_occludeds=is_occludeds, is_truncateds=is_truncateds, is_depictions=is_depictions, is_insides=is_insides) return ann def get_meta_from_file(self, meta_file=''): """Get image metas from pkl file.""" metas = mmcv.load( meta_file, file_format='pkl', file_client_args=self.file_client_args) assert len(metas) == len(self) for i in range(len(metas)): file_name = osp.split(metas[i]['filename'])[-1] img_info = self.data_infos[i].get('img_info', None) if img_info is not None: assert file_name == osp.split(img_info['filename'])[-1] else: assert file_name == self.data_infos[i]['filename'] hw = metas[i]['ori_shape'][:2] self.test_img_shapes.append(hw) def get_meta_from_pipeline(self, results): """Get image metas from pipeline.""" self.temp_img_metas.extend(results['img_metas']) if dist.is_available() and self.world_size > 1: from mmdet.apis.test import collect_results_cpu self.test_img_metas = collect_results_cpu(self.temp_img_metas, len(self)) else: self.test_img_metas = self.temp_img_metas def get_img_shape(self, metas): """Set images original shape into data_infos.""" assert len(metas) == len(self) for i in range(len(metas)): file_name = osp.split(metas[i].data['ori_filename'])[-1] img_info = self.data_infos[i].get('img_info', None) if img_info is not None: assert file_name == osp.split(img_info['filename'])[-1] else: assert file_name == self.data_infos[i]['filename'] hw = metas[i].data['ori_shape'][:2] self.test_img_shapes.append(hw) def prepare_test_img(self, idx): """Get testing data after pipeline.""" img_info = self.data_infos[idx] results = dict(img_info=img_info) if self.proposals is not None: results['proposals'] = self.proposals[idx] self.pre_pipeline(results) results = self.pipeline(results) if self.get_metas and self.load_from_pipeline: self.get_meta_from_pipeline(results) return results def _filter_imgs(self, min_size=32): """Filter images too small.""" if self.filter_empty_gt: warnings.warn('OpenImageDatasets does not support ' 'filtering empty gt images.') valid_inds = [i for i in range(len(self))] return valid_inds def _set_group_flag(self): """Set flag according to image aspect ratio.""" self.flag = np.zeros(len(self), dtype=np.uint8) # TODO: set flag without width and height def get_relation_matrix(self, hierarchy_file): """Get hierarchy for classes. Args: hierarchy_file (sty): File path to the hierarchy for classes. Returns: ndarray: The matrix of the corresponding relationship between the parent class and the child class, of shape (class_num, class_num). """ if self.data_root is not None: if not osp.isabs(hierarchy_file): hierarchy_file = osp.join(self.data_root, hierarchy_file) with open(hierarchy_file, 'r') as f: hierarchy = json.load(f) class_num = len(self.CLASSES) class_label_tree = np.eye(class_num, class_num) class_label_tree = self._convert_hierarchy_tree( hierarchy, class_label_tree) return class_label_tree def _convert_hierarchy_tree(self, hierarchy_map, class_label_tree, parents=[], get_all_parents=True): """Get matrix of the corresponding relationship between the parent class and the child class. Args: hierarchy_map (dict): Including label name and corresponding subcategory. Keys of dicts are: - `LabeName` (str): Name of the label. - `Subcategory` (dict | list): Corresponding subcategory(ies). class_label_tree (ndarray): The matrix of the corresponding relationship between the parent class and the child class, of shape (class_num, class_num). parents (list): Corresponding parent class. get_all_parents (bool): Whether get all parent names. Default: True Returns: ndarray: The matrix of the corresponding relationship between the parent class and the child class, of shape (class_num, class_num). """ if 'Subcategory' in hierarchy_map: for node in hierarchy_map['Subcategory']: if 'LabelName' in node: children_name = node['LabelName'] children_index = self.index_dict[children_name] children = [children_index] else: continue if len(parents) > 0: for parent_index in parents: if get_all_parents: children.append(parent_index) class_label_tree[children_index, parent_index] = 1 class_label_tree = self._convert_hierarchy_tree( node, class_label_tree, parents=children) return class_label_tree def add_supercategory_ann(self, annotations): """Add parent classes of the corresponding class of the ground truth bboxes.""" for i, ann in enumerate(annotations): assert len(ann['labels']) == len(ann['bboxes']) == \ len(ann['gt_is_group_ofs']) gt_bboxes = [] gt_is_group_ofs = [] gt_labels = [] for j in range(len(ann['labels'])): label = ann['labels'][j] bbox = ann['bboxes'][j] is_group = ann['gt_is_group_ofs'][j] label = np.where(self.class_label_tree[label])[0] if len(label) > 1: for k in range(len(label)): gt_bboxes.append(bbox) gt_is_group_ofs.append(is_group) gt_labels.append(label[k]) else: gt_bboxes.append(bbox) gt_is_group_ofs.append(is_group) gt_labels.append(label[0]) annotations[i] = dict( bboxes=np.array(gt_bboxes).astype(np.float32), labels=np.array(gt_labels).astype(np.int64), bboxes_ignore=ann['bboxes_ignore'], gt_is_group_ofs=np.array(gt_is_group_ofs).astype(bool)) return annotations def process_results(self, det_results, annotations, image_level_annotations): """Process results of the corresponding class of the detection bboxes. Note: It will choose to do the following two processing according to the parameters: 1. Whether to add parent classes of the corresponding class of the detection bboxes. 2. Whether to ignore the classes that unannotated on that image. """ if image_level_annotations is not None: assert len(annotations) == \ len(image_level_annotations) == \ len(det_results) else: assert len(annotations) == len(det_results) for i in range(len(det_results)): results = copy.deepcopy(det_results[i]) valid_classes = np.where( np.array([[bbox.shape[0]] for bbox in det_results[i]]) != 0)[0] if image_level_annotations is not None: labels = annotations[i]['labels'] image_level_labels = \ image_level_annotations[i]['image_level_labels'] allowed_labeles = np.unique( np.append(labels, image_level_labels)) else: allowed_labeles = np.unique(annotations[i]['labels']) for valid_class in valid_classes: det_cls = np.where(self.class_label_tree[valid_class])[0] for index in det_cls: if index in allowed_labeles and \ index != valid_class and \ self.get_supercategory: det_results[i][index] = \ np.concatenate((det_results[i][index], results[valid_class])) elif index not in allowed_labeles and self.filter_labels: # Remove useless parts det_results[i][index] = np.empty( (0, 5)).astype(np.float32) return det_results def load_image_label_from_csv(self, image_level_ann_file): """Load image level annotations from csv style ann_file. Args: image_level_ann_file (str): CSV style image level annotation file path. Returns: defaultdict[list[dict]]: Annotations where item of the defaultdict indicates an image, each of which has (n) dicts. Keys of dicts are: - `image_level_label` (int): Label id. - `confidence` (float): Labels that are human-verified to be present in an image have confidence = 1 (positive labels). Labels that are human-verified to be absent from an image have confidence = 0 (negative labels). Machine-generated labels have fractional confidences, generally >= 0.5. The higher the confidence, the smaller the chance for the label to be a false positive. """ item_lists = defaultdict(list) with open(image_level_ann_file, 'r') as f: reader = csv.reader(f) for i, line in enumerate(reader): if i == 0: continue img_id = line[0] item_lists[img_id].append( dict( image_level_label=int(self.index_dict[line[2]]), confidence=float(line[3]))) return item_lists def get_image_level_ann(self, image_level_ann_file): """Get OpenImages annotation by index. Args: image_level_ann_file (str): CSV style image level annotation file path. Returns: dict: Annotation info of specified index. """ if hasattr(self.file_client, 'get_local_path'): with self.file_client.get_local_path(image_level_ann_file) \ as local_path: item_lists = self.load_image_label_from_csv(local_path) else: item_lists = self.load_image_label_from_csv(image_level_ann_file) image_level_annotations = [] for i in range(len(self)): img_info = self.data_infos[i].get('img_info', None) if img_info is not None: # for Open Images Challenges img_id = osp.split(img_info['filename'])[-1][:-4] else: # for Open Images v6 img_id = self.data_infos[i]['img_id'] item_list = item_lists.get(img_id, None) if item_list is not None: image_level_labels = [] confidences = [] for obj in item_list: image_level_label = int(obj['image_level_label']) confidence = float(obj['confidence']) image_level_labels.append(image_level_label) confidences.append(confidence) if not image_level_labels: image_level_labels = np.zeros((0, )) confidences = np.zeros((0, )) else: image_level_labels = np.array(image_level_labels) confidences = np.array(confidences) else: image_level_labels = np.zeros((0, )) confidences = np.zeros((0, )) ann = dict( image_level_labels=image_level_labels.astype(np.int64), confidences=confidences.astype(np.float32)) image_level_annotations.append(ann) return image_level_annotations def denormalize_gt_bboxes(self, annotations): """Convert ground truth bboxes from relative position to absolute position. Only used in evaluating time. """ assert len(self.test_img_shapes) == len(annotations) for i in range(len(annotations)): h, w = self.test_img_shapes[i] annotations[i]['bboxes'][:, 0::2] *= w annotations[i]['bboxes'][:, 1::2] *= h return annotations def get_cat_ids(self, idx): """Get category ids by index. Args: idx (int): Index of data. Returns: list[int]: All categories in the image of specified index. """ return self.get_ann_info(idx)['labels'].astype(np.int).tolist() def evaluate(self, results, metric='mAP', logger=None, iou_thr=0.5, ioa_thr=0.5, scale_ranges=None, denorm_gt_bbox=True, use_group_of=True): """Evaluate in OpenImages. Args: results (list[list | tuple]): Testing results of the dataset. metric (str | list[str]): Metrics to be evaluated. Option is 'mAP'. Default: 'mAP'. logger (logging.Logger | str, optional): Logger used for printing related information during evaluation. Default: None. iou_thr (float | list[float]): IoU threshold. Default: 0.5. ioa_thr (float | list[float]): IoA threshold. Default: 0.5. scale_ranges (list[tuple], optional): Scale ranges for evaluating mAP. If not specified, all bounding boxes would be included in evaluation. Default: None denorm_gt_bbox (bool): Whether to denorm ground truth bboxes from relative position to absolute position. Default: True use_group_of (bool): Whether consider group of groud truth bboxes during evaluating. Default: True. Returns: dict[str, float]: AP metrics. """ if not isinstance(metric, str): assert len(metric) == 1 metric = metric[0] allowed_metrics = ['mAP'] if metric not in allowed_metrics: raise KeyError(f'metric {metric} is not supported') annotations = [self.get_ann_info(i) for i in range(len(self))] if self.load_image_level_labels: image_level_annotations = \ self.get_image_level_ann(self.image_level_ann_file) else: image_level_annotations = None # load metas from file if self.get_metas and self.load_from_file: assert self.meta_file.endswith( 'pkl'), 'File name must be pkl suffix' self.get_meta_from_file(self.meta_file) # load metas from pipeline else: self.get_img_shape(self.test_img_metas) if len(self.test_img_shapes) > len(self): self.test_img_shapes = self.test_img_shapes[:len(self)] if denorm_gt_bbox: annotations = self.denormalize_gt_bboxes(annotations) # Reset test_image_metas, temp_image_metas and test_img_shapes # to avoid potential error self.temp_img_metas = [] self.test_img_shapes = [] self.test_img_metas = [] if self.get_supercategory: annotations = self.add_supercategory_ann(annotations) results = self.process_results(results, annotations, image_level_annotations) if use_group_of: assert ioa_thr is not None, \ 'ioa_thr must have value when using group_of in evaluation.' eval_results = OrderedDict() iou_thrs = [iou_thr] if isinstance(iou_thr, float) else iou_thr ioa_thrs = [ioa_thr] if isinstance(ioa_thr, float) or ioa_thr is None \ else ioa_thr # get dataset type if len(self.CLASSES) == 500: ds_name = 'oid_challenge' elif len(self.CLASSES) == 601: ds_name = 'oid_v6' else: ds_name = self.CLASSES warnings.warn('Cannot infer dataset type from the length of the ' 'classes. Set `oid_v6` as dataset type.') if metric == 'mAP': assert isinstance(iou_thrs, list) and isinstance(ioa_thrs, list) assert len(ioa_thrs) == len(iou_thrs) mean_aps = [] for iou_thr, ioa_thr in zip(iou_thrs, ioa_thrs): print_log(f'\n{"-" * 15}iou_thr, ioa_thr: {iou_thr}, {ioa_thr}' f'{"-" * 15}') mean_ap, _ = eval_map( results, annotations, scale_ranges=scale_ranges, iou_thr=iou_thr, ioa_thr=ioa_thr, dataset=ds_name, logger=logger, use_group_of=use_group_of) mean_aps.append(mean_ap) eval_results[f'AP{int(iou_thr * 100):02d}'] = round(mean_ap, 3) eval_results['mAP'] = sum(mean_aps) / len(mean_aps) return eval_results @DATASETS.register_module() class OpenImagesChallengeDataset(OpenImagesDataset): """Open Images Challenge dataset for detection.""" def __init__(self, ann_file, **kwargs): assert ann_file.endswith('txt') super(OpenImagesChallengeDataset, self).__init__( ann_file=ann_file, **kwargs) def get_classes_from_csv(self, label_file): """Get classes name from file. Args: label_file (str): File path of the label description file that maps the classes names in MID format to their short descriptions. Returns: list: Class name of OpenImages. """ label_list = [] id_list = [] with open(label_file, 'r') as f: reader = csv.reader(f) for line in reader: label_name = line[0] label_id = int(line[2]) label_list.append(line[1]) id_list.append(label_id) self.index_dict[label_name] = label_id - 1 indexes = np.argsort(id_list) classes_names = [] for index in indexes: classes_names.append(label_list[index]) return classes_names def load_annotations(self, ann_file): """Load annotation from annotation file.""" with open(ann_file) as f: lines = f.readlines() i = 0 ann_infos = [] while i < len(lines): bboxes = [] labels = [] is_group_ofs = [] filename = lines[i].rstrip() i += 2 img_gt_size = int(lines[i]) i += 1 for j in range(img_gt_size): sp = lines[i + j].split() bboxes.append( [float(sp[1]), float(sp[2]), float(sp[3]), float(sp[4])]) labels.append(int(sp[0]) - 1) # labels begin from 1 is_group_ofs.append(True if int(sp[5]) == 1 else False) i += img_gt_size gt_bboxes = np.array(bboxes, dtype=np.float32) gt_labels = np.array(labels, dtype=np.int64) gt_bboxes_ignore = np.zeros((0, 4), dtype=np.float32) gt_is_group_ofs = np.array(is_group_ofs, dtype=bool) img_info = dict(filename=filename) ann_info = dict( bboxes=gt_bboxes, labels=gt_labels, bboxes_ignore=gt_bboxes_ignore, gt_is_group_ofs=gt_is_group_ofs) ann_infos.append(dict(img_info=img_info, ann_info=ann_info)) return ann_infos def prepare_train_img(self, idx): """Get training data and annotations after pipeline.""" ann_info = self.data_infos[idx] results = dict( img_info=ann_info['img_info'], ann_info=ann_info['ann_info'], ) if self.proposals is not None: results['proposals'] = self.proposals[idx] self.pre_pipeline(results) return self.pipeline(results) def prepare_test_img(self, idx): """Get testing data after pipeline.""" ann_info = self.data_infos[idx] results = dict(img_info=ann_info['img_info']) if self.proposals is not None: results['proposals'] = self.proposals[idx] self.pre_pipeline(results) results = self.pipeline(results) if self.get_metas and self.load_from_pipeline: self.get_meta_from_pipeline(results) return results def get_relation_matrix(self, hierarchy_file): """Get hierarchy for classes. Args: hierarchy_file (str): File path to the hierarchy for classes. Returns: ndarray: The matrix of the corresponding relationship between the parent class and the child class, of shape (class_num, class_num). """ class_label_tree = np.load(hierarchy_file, allow_pickle=True) return class_label_tree[1:, 1:] def get_ann_info(self, idx): """Get OpenImages annotation by index. Args: idx (int): Index of data. Returns: dict: Annotation info of specified index. """ # avoid some potential error data_infos = copy.deepcopy(self.data_infos[idx]['ann_info']) return data_infos def load_image_label_from_csv(self, image_level_ann_file): """Load image level annotations from csv style ann_file. Args: image_level_ann_file (str): CSV style image level annotation file path. Returns: defaultdict[list[dict]]: Annotations where item of the defaultdict indicates an image, each of which has (n) dicts. Keys of dicts are: - `image_level_label` (int): of shape 1. - `confidence` (float): of shape 1. """ item_lists = defaultdict(list) with open(image_level_ann_file, 'r') as f: reader = csv.reader(f) i = -1 for line in reader: i += 1 if i == 0: continue else: img_id = line[0] label_id = line[1] assert label_id in self.index_dict image_level_label = int(self.index_dict[label_id]) confidence = float(line[2]) item_lists[img_id].append( dict( image_level_label=image_level_label, confidence=confidence)) return item_lists