""" Evaluate Hook Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) Please cite our work if the code is helpful to you. """ import numpy as np import torch import torch.distributed as dist import pointops from uuid import uuid4 import pointcept.utils.comm as comm from pointcept.utils.misc import intersection_and_union_gpu from .default import HookBase from .builder import HOOKS @HOOKS.register_module() class ClsEvaluator(HookBase): def after_epoch(self): if self.trainer.cfg.evaluate: self.eval() def eval(self): self.trainer.logger.info(">>>>>>>>>>>>>>>> Start Evaluation >>>>>>>>>>>>>>>>") self.trainer.model.eval() for i, input_dict in enumerate(self.trainer.val_loader): for key in input_dict.keys(): if isinstance(input_dict[key], torch.Tensor): input_dict[key] = input_dict[key].cuda(non_blocking=True) with torch.no_grad(): output_dict = self.trainer.model(input_dict) output = output_dict["cls_logits"] loss = output_dict["loss"] pred = output.max(1)[1] label = input_dict["category"] intersection, union, target = intersection_and_union_gpu( pred, label, self.trainer.cfg.data.num_classes, self.trainer.cfg.data.ignore_index, ) if comm.get_world_size() > 1: dist.all_reduce(intersection), dist.all_reduce(union), dist.all_reduce( target ) intersection, union, target = ( intersection.cpu().numpy(), union.cpu().numpy(), target.cpu().numpy(), ) # Here there is no need to sync since sync happened in dist.all_reduce self.trainer.storage.put_scalar("val_intersection", intersection) self.trainer.storage.put_scalar("val_union", union) self.trainer.storage.put_scalar("val_target", target) self.trainer.storage.put_scalar("val_loss", loss.item()) self.trainer.logger.info( "Test: [{iter}/{max_iter}] " "Loss {loss:.4f} ".format( iter=i + 1, max_iter=len(self.trainer.val_loader), loss=loss.item() ) ) loss_avg = self.trainer.storage.history("val_loss").avg intersection = self.trainer.storage.history("val_intersection").total union = self.trainer.storage.history("val_union").total target = self.trainer.storage.history("val_target").total iou_class = intersection / (union + 1e-10) acc_class = intersection / (target + 1e-10) m_iou = np.mean(iou_class) m_acc = np.mean(acc_class) all_acc = sum(intersection) / (sum(target) + 1e-10) self.trainer.logger.info( "Val result: mIoU/mAcc/allAcc {:.4f}/{:.4f}/{:.4f}.".format( m_iou, m_acc, all_acc ) ) for i in range(self.trainer.cfg.data.num_classes): self.trainer.logger.info( "Class_{idx}-{name} Result: iou/accuracy {iou:.4f}/{accuracy:.4f}".format( idx=i, name=self.trainer.cfg.data.names[i], iou=iou_class[i], accuracy=acc_class[i], ) ) current_epoch = self.trainer.epoch + 1 if self.trainer.writer is not None: self.trainer.writer.add_scalar("val/loss", loss_avg, current_epoch) self.trainer.writer.add_scalar("val/mIoU", m_iou, current_epoch) self.trainer.writer.add_scalar("val/mAcc", m_acc, current_epoch) self.trainer.writer.add_scalar("val/allAcc", all_acc, current_epoch) self.trainer.logger.info("<<<<<<<<<<<<<<<<< End Evaluation <<<<<<<<<<<<<<<<<") self.trainer.comm_info["current_metric_value"] = all_acc # save for saver self.trainer.comm_info["current_metric_name"] = "allAcc" # save for saver def after_train(self): self.trainer.logger.info( "Best {}: {:.4f}".format("allAcc", self.trainer.best_metric_value) ) @HOOKS.register_module() class SemSegEvaluator(HookBase): def after_epoch(self): if self.trainer.cfg.evaluate: self.eval() def eval(self): self.trainer.logger.info(">>>>>>>>>>>>>>>> Start Evaluation >>>>>>>>>>>>>>>>") self.trainer.model.eval() for i, input_dict in enumerate(self.trainer.val_loader): for key in input_dict.keys(): if isinstance(input_dict[key], torch.Tensor): input_dict[key] = input_dict[key].cuda(non_blocking=True) with torch.no_grad(): output_dict = self.trainer.model(input_dict) output = output_dict["seg_logits"] loss = output_dict["loss"] pred = output.max(1)[1] segment = input_dict["segment"] if "origin_coord" in input_dict.keys(): idx, _ = pointops.knn_query( 1, input_dict["coord"].float(), input_dict["offset"].int(), input_dict["origin_coord"].float(), input_dict["origin_offset"].int(), ) pred = pred[idx.flatten().long()] segment = input_dict["origin_segment"] intersection, union, target = intersection_and_union_gpu( pred, segment, self.trainer.cfg.data.num_classes, self.trainer.cfg.data.ignore_index, ) if comm.get_world_size() > 1: dist.all_reduce(intersection), dist.all_reduce(union), dist.all_reduce( target ) intersection, union, target = ( intersection.cpu().numpy(), union.cpu().numpy(), target.cpu().numpy(), ) # Here there is no need to sync since sync happened in dist.all_reduce self.trainer.storage.put_scalar("val_intersection", intersection) self.trainer.storage.put_scalar("val_union", union) self.trainer.storage.put_scalar("val_target", target) self.trainer.storage.put_scalar("val_loss", loss.item()) info = "Test: [{iter}/{max_iter}] ".format( iter=i + 1, max_iter=len(self.trainer.val_loader) ) if "origin_coord" in input_dict.keys(): info = "Interp. " + info self.trainer.logger.info( info + "Loss {loss:.4f} ".format( iter=i + 1, max_iter=len(self.trainer.val_loader), loss=loss.item() ) ) loss_avg = self.trainer.storage.history("val_loss").avg intersection = self.trainer.storage.history("val_intersection").total union = self.trainer.storage.history("val_union").total target = self.trainer.storage.history("val_target").total iou_class = intersection / (union + 1e-10) acc_class = intersection / (target + 1e-10) m_iou = np.mean(iou_class) m_acc = np.mean(acc_class) all_acc = sum(intersection) / (sum(target) + 1e-10) self.trainer.logger.info( "Val result: mIoU/mAcc/allAcc {:.4f}/{:.4f}/{:.4f}.".format( m_iou, m_acc, all_acc ) ) for i in range(self.trainer.cfg.data.num_classes): self.trainer.logger.info( "Class_{idx}-{name} Result: iou/accuracy {iou:.4f}/{accuracy:.4f}".format( idx=i, name=self.trainer.cfg.data.names[i], iou=iou_class[i], accuracy=acc_class[i], ) ) current_epoch = self.trainer.epoch + 1 if self.trainer.writer is not None: self.trainer.writer.add_scalar("val/loss", loss_avg, current_epoch) self.trainer.writer.add_scalar("val/mIoU", m_iou, current_epoch) self.trainer.writer.add_scalar("val/mAcc", m_acc, current_epoch) self.trainer.writer.add_scalar("val/allAcc", all_acc, current_epoch) self.trainer.logger.info("<<<<<<<<<<<<<<<<< End Evaluation <<<<<<<<<<<<<<<<<") self.trainer.comm_info["current_metric_value"] = m_iou # save for saver self.trainer.comm_info["current_metric_name"] = "mIoU" # save for saver def after_train(self): self.trainer.logger.info( "Best {}: {:.4f}".format("mIoU", self.trainer.best_metric_value) ) @HOOKS.register_module() class InsSegEvaluator(HookBase): def __init__(self, segment_ignore_index=(-1,), instance_ignore_index=-1): self.segment_ignore_index = segment_ignore_index self.instance_ignore_index = instance_ignore_index self.valid_class_names = None # update in before train self.overlaps = np.append(np.arange(0.5, 0.95, 0.05), 0.25) self.min_region_sizes = 100 self.distance_threshes = float("inf") self.distance_confs = -float("inf") def before_train(self): self.valid_class_names = [ self.trainer.cfg.data.names[i] for i in range(self.trainer.cfg.data.num_classes) if i not in self.segment_ignore_index ] def after_epoch(self): if self.trainer.cfg.evaluate: self.eval() def associate_instances(self, pred, segment, instance): segment = segment.cpu().numpy() instance = instance.cpu().numpy() void_mask = np.in1d(segment, self.segment_ignore_index) assert ( pred["pred_classes"].shape[0] == pred["pred_scores"].shape[0] == pred["pred_masks"].shape[0] ) assert pred["pred_masks"].shape[1] == segment.shape[0] == instance.shape[0] # get gt instances gt_instances = dict() for i in range(self.trainer.cfg.data.num_classes): if i not in self.segment_ignore_index: gt_instances[self.trainer.cfg.data.names[i]] = [] instance_ids, idx, counts = np.unique( instance, return_index=True, return_counts=True ) segment_ids = segment[idx] for i in range(len(instance_ids)): if instance_ids[i] == self.instance_ignore_index: continue if segment_ids[i] in self.segment_ignore_index: continue gt_inst = dict() gt_inst["instance_id"] = instance_ids[i] gt_inst["segment_id"] = segment_ids[i] gt_inst["dist_conf"] = 0.0 gt_inst["med_dist"] = -1.0 gt_inst["vert_count"] = counts[i] gt_inst["matched_pred"] = [] gt_instances[self.trainer.cfg.data.names[segment_ids[i]]].append(gt_inst) # get pred instances and associate with gt pred_instances = dict() for i in range(self.trainer.cfg.data.num_classes): if i not in self.segment_ignore_index: pred_instances[self.trainer.cfg.data.names[i]] = [] instance_id = 0 for i in range(len(pred["pred_classes"])): if pred["pred_classes"][i] in self.segment_ignore_index: continue pred_inst = dict() pred_inst["uuid"] = uuid4() pred_inst["instance_id"] = instance_id pred_inst["segment_id"] = pred["pred_classes"][i] pred_inst["confidence"] = pred["pred_scores"][i] pred_inst["mask"] = np.not_equal(pred["pred_masks"][i], 0) pred_inst["vert_count"] = np.count_nonzero(pred_inst["mask"]) pred_inst["void_intersection"] = np.count_nonzero( np.logical_and(void_mask, pred_inst["mask"]) ) if pred_inst["vert_count"] < self.min_region_sizes: continue # skip if empty segment_name = self.trainer.cfg.data.names[pred_inst["segment_id"]] matched_gt = [] for gt_idx, gt_inst in enumerate(gt_instances[segment_name]): intersection = np.count_nonzero( np.logical_and( instance == gt_inst["instance_id"], pred_inst["mask"] ) ) if intersection > 0: gt_inst_ = gt_inst.copy() pred_inst_ = pred_inst.copy() gt_inst_["intersection"] = intersection pred_inst_["intersection"] = intersection matched_gt.append(gt_inst_) gt_inst["matched_pred"].append(pred_inst_) pred_inst["matched_gt"] = matched_gt pred_instances[segment_name].append(pred_inst) instance_id += 1 return gt_instances, pred_instances def evaluate_matches(self, scenes): overlaps = self.overlaps min_region_sizes = [self.min_region_sizes] dist_threshes = [self.distance_threshes] dist_confs = [self.distance_confs] # results: class x overlap ap_table = np.zeros( (len(dist_threshes), len(self.valid_class_names), len(overlaps)), float ) for di, (min_region_size, distance_thresh, distance_conf) in enumerate( zip(min_region_sizes, dist_threshes, dist_confs) ): for oi, overlap_th in enumerate(overlaps): pred_visited = {} for scene in scenes: for _ in scene["pred"]: for label_name in self.valid_class_names: for p in scene["pred"][label_name]: if "uuid" in p: pred_visited[p["uuid"]] = False for li, label_name in enumerate(self.valid_class_names): y_true = np.empty(0) y_score = np.empty(0) hard_false_negatives = 0 has_gt = False has_pred = False for scene in scenes: pred_instances = scene["pred"][label_name] gt_instances = scene["gt"][label_name] # filter groups in ground truth gt_instances = [ gt for gt in gt_instances if gt["vert_count"] >= min_region_size and gt["med_dist"] <= distance_thresh and gt["dist_conf"] >= distance_conf ] if gt_instances: has_gt = True if pred_instances: has_pred = True cur_true = np.ones(len(gt_instances)) cur_score = np.ones(len(gt_instances)) * (-float("inf")) cur_match = np.zeros(len(gt_instances), dtype=bool) # collect matches for gti, gt in enumerate(gt_instances): found_match = False for pred in gt["matched_pred"]: # greedy assignments if pred_visited[pred["uuid"]]: continue overlap = float(pred["intersection"]) / ( gt["vert_count"] + pred["vert_count"] - pred["intersection"] ) if overlap > overlap_th: confidence = pred["confidence"] # if already have a prediction for this gt, # the prediction with the lower score is automatically a false positive if cur_match[gti]: max_score = max(cur_score[gti], confidence) min_score = min(cur_score[gti], confidence) cur_score[gti] = max_score # append false positive cur_true = np.append(cur_true, 0) cur_score = np.append(cur_score, min_score) cur_match = np.append(cur_match, True) # otherwise set score else: found_match = True cur_match[gti] = True cur_score[gti] = confidence pred_visited[pred["uuid"]] = True if not found_match: hard_false_negatives += 1 # remove non-matched ground truth instances cur_true = cur_true[cur_match] cur_score = cur_score[cur_match] # collect non-matched predictions as false positive for pred in pred_instances: found_gt = False for gt in pred["matched_gt"]: overlap = float(gt["intersection"]) / ( gt["vert_count"] + pred["vert_count"] - gt["intersection"] ) if overlap > overlap_th: found_gt = True break if not found_gt: num_ignore = pred["void_intersection"] for gt in pred["matched_gt"]: if gt["segment_id"] in self.segment_ignore_index: num_ignore += gt["intersection"] # small ground truth instances if ( gt["vert_count"] < min_region_size or gt["med_dist"] > distance_thresh or gt["dist_conf"] < distance_conf ): num_ignore += gt["intersection"] proportion_ignore = ( float(num_ignore) / pred["vert_count"] ) # if not ignored append false positive if proportion_ignore <= overlap_th: cur_true = np.append(cur_true, 0) confidence = pred["confidence"] cur_score = np.append(cur_score, confidence) # append to overall results y_true = np.append(y_true, cur_true) y_score = np.append(y_score, cur_score) # compute average precision if has_gt and has_pred: # compute precision recall curve first # sorting and cumsum score_arg_sort = np.argsort(y_score) y_score_sorted = y_score[score_arg_sort] y_true_sorted = y_true[score_arg_sort] y_true_sorted_cumsum = np.cumsum(y_true_sorted) # unique thresholds (thresholds, unique_indices) = np.unique( y_score_sorted, return_index=True ) num_prec_recall = len(unique_indices) + 1 # prepare precision recall num_examples = len(y_score_sorted) # https://github.com/ScanNet/ScanNet/pull/26 # all predictions are non-matched but also all of them are ignored and not counted as FP # y_true_sorted_cumsum is empty # num_true_examples = y_true_sorted_cumsum[-1] num_true_examples = ( y_true_sorted_cumsum[-1] if len(y_true_sorted_cumsum) > 0 else 0 ) precision = np.zeros(num_prec_recall) recall = np.zeros(num_prec_recall) # deal with the first point y_true_sorted_cumsum = np.append(y_true_sorted_cumsum, 0) # deal with remaining for idx_res, idx_scores in enumerate(unique_indices): cumsum = y_true_sorted_cumsum[idx_scores - 1] tp = num_true_examples - cumsum fp = num_examples - idx_scores - tp fn = cumsum + hard_false_negatives p = float(tp) / (tp + fp) r = float(tp) / (tp + fn) precision[idx_res] = p recall[idx_res] = r # first point in curve is artificial precision[-1] = 1.0 recall[-1] = 0.0 # compute average of precision-recall curve recall_for_conv = np.copy(recall) recall_for_conv = np.append(recall_for_conv[0], recall_for_conv) recall_for_conv = np.append(recall_for_conv, 0.0) stepWidths = np.convolve( recall_for_conv, [-0.5, 0, 0.5], "valid" ) # integrate is now simply a dot product ap_current = np.dot(precision, stepWidths) elif has_gt: ap_current = 0.0 else: ap_current = float("nan") ap_table[di, li, oi] = ap_current d_inf = 0 o50 = np.where(np.isclose(self.overlaps, 0.5)) o25 = np.where(np.isclose(self.overlaps, 0.25)) oAllBut25 = np.where(np.logical_not(np.isclose(self.overlaps, 0.25))) ap_scores = dict() ap_scores["all_ap"] = np.nanmean(ap_table[d_inf, :, oAllBut25]) ap_scores["all_ap_50%"] = np.nanmean(ap_table[d_inf, :, o50]) ap_scores["all_ap_25%"] = np.nanmean(ap_table[d_inf, :, o25]) ap_scores["classes"] = {} for li, label_name in enumerate(self.valid_class_names): ap_scores["classes"][label_name] = {} ap_scores["classes"][label_name]["ap"] = np.average( ap_table[d_inf, li, oAllBut25] ) ap_scores["classes"][label_name]["ap50%"] = np.average( ap_table[d_inf, li, o50] ) ap_scores["classes"][label_name]["ap25%"] = np.average( ap_table[d_inf, li, o25] ) return ap_scores def eval(self): self.trainer.logger.info(">>>>>>>>>>>>>>>> Start Evaluation >>>>>>>>>>>>>>>>") self.trainer.model.eval() scenes = [] for i, input_dict in enumerate(self.trainer.val_loader): assert ( len(input_dict["offset"]) == 1 ) # currently only support bs 1 for each GPU for key in input_dict.keys(): if isinstance(input_dict[key], torch.Tensor): input_dict[key] = input_dict[key].cuda(non_blocking=True) with torch.no_grad(): output_dict = self.trainer.model(input_dict) loss = output_dict["loss"] segment = input_dict["segment"] instance = input_dict["instance"] # map to origin if "origin_coord" in input_dict.keys(): idx, _ = pointops.knn_query( 1, input_dict["coord"].float(), input_dict["offset"].int(), input_dict["origin_coord"].float(), input_dict["origin_offset"].int(), ) idx = idx.cpu().flatten().long() output_dict["pred_masks"] = output_dict["pred_masks"][:, idx] segment = input_dict["origin_segment"] instance = input_dict["origin_instance"] gt_instances, pred_instance = self.associate_instances( output_dict, segment, instance ) scenes.append(dict(gt=gt_instances, pred=pred_instance)) self.trainer.storage.put_scalar("val_loss", loss.item()) self.trainer.logger.info( "Test: [{iter}/{max_iter}] " "Loss {loss:.4f} ".format( iter=i + 1, max_iter=len(self.trainer.val_loader), loss=loss.item() ) ) loss_avg = self.trainer.storage.history("val_loss").avg comm.synchronize() scenes_sync = comm.gather(scenes, dst=0) scenes = [scene for scenes_ in scenes_sync for scene in scenes_] ap_scores = self.evaluate_matches(scenes) all_ap = ap_scores["all_ap"] all_ap_50 = ap_scores["all_ap_50%"] all_ap_25 = ap_scores["all_ap_25%"] self.trainer.logger.info( "Val result: mAP/AP50/AP25 {:.4f}/{:.4f}/{:.4f}.".format( all_ap, all_ap_50, all_ap_25 ) ) for i, label_name in enumerate(self.valid_class_names): ap = ap_scores["classes"][label_name]["ap"] ap_50 = ap_scores["classes"][label_name]["ap50%"] ap_25 = ap_scores["classes"][label_name]["ap25%"] self.trainer.logger.info( "Class_{idx}-{name} Result: AP/AP50/AP25 {AP:.4f}/{AP50:.4f}/{AP25:.4f}".format( idx=i, name=label_name, AP=ap, AP50=ap_50, AP25=ap_25 ) ) current_epoch = self.trainer.epoch + 1 if self.trainer.writer is not None: self.trainer.writer.add_scalar("val/loss", loss_avg, current_epoch) self.trainer.writer.add_scalar("val/mAP", all_ap, current_epoch) self.trainer.writer.add_scalar("val/AP50", all_ap_50, current_epoch) self.trainer.writer.add_scalar("val/AP25", all_ap_25, current_epoch) self.trainer.logger.info("<<<<<<<<<<<<<<<<< End Evaluation <<<<<<<<<<<<<<<<<") self.trainer.comm_info["current_metric_value"] = all_ap_50 # save for saver self.trainer.comm_info["current_metric_name"] = "AP50" # save for saver