{ "cells": [ { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "import json\n", "import pandas as pd\n", "\n", "from sklearn.metrics import classification_report, roc_auc_score, average_precision_score, f1_score\n", "from sklearn.preprocessing import LabelBinarizer\n", "from collections import defaultdict\n" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "# Load the prediction and ground truth JSON files\n", "with open('pred.json', 'r') as pred_file:\n", " predictions = json.load(pred_file)\n", "\n", "with open('GT.json', 'r') as gt_file:\n", " ground_truth = json.load(gt_file)\n", "\n", "# Convert to DataFrame\n", "pred_df = pd.DataFrame(predictions)\n", "gt_df = pd.DataFrame(ground_truth)\n", "\n", "# Create unique class name\n", "pred_df['unique_class'] = pred_df['damage_class'] + '_' + pred_df['damage_level']\n", "gt_df['unique_class'] = gt_df['damage_class'] + '_' + gt_df['damage_level']\n", "\n", "# Merge predictions with ground truth based on image_path\n", "merged_df = pd.merge(pred_df, gt_df, on='image_path', suffixes=('_pred', '_gt'))" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Total number of unique classes: 39\n" ] } ], "source": [ "# Extract true and predicted classes\n", "y_true = merged_df['unique_class_gt']\n", "y_pred = merged_df['unique_class_pred']\n", "\n", "# Count the number of unique classes\n", "unique_classes = set(y_true).union(set(y_pred))\n", "num_unique_classes = len(unique_classes)\n", "print(f\"Total number of unique classes: {num_unique_classes}\")" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Classification Report:\n", " precision recall f1-score support\n", "\n", " COS_Arc_mark_level_1 1.00 1.00 1.00 22\n", " COS_Arc_mark_level_2 0.94 0.97 0.96 34\n", " COS_Arc_mark_level_3 0.98 0.99 0.99 105\n", " COS_Crack_level_1 1.00 1.00 1.00 7\n", " COS_Crack_level_2 0.96 1.00 0.98 26\n", " COS_Crack_level_3 0.99 1.00 0.99 72\n", "COS_Fuse_link_corrosion_level_1 0.95 1.00 0.98 21\n", "COS_Fuse_link_corrosion_level_2 0.99 0.99 0.99 74\n", "COS_Fuse_link_corrosion_level_3 0.94 0.97 0.95 87\n", " COS_Normal_normal 0.99 0.97 0.98 115\n", " GS_Arc_mark_level_1 0.94 0.95 0.95 104\n", " GS_Arc_mark_level_2 0.93 0.93 0.93 14\n", " GS_Arc_mark_level_3 1.00 0.86 0.92 7\n", " GS_Normal_normal 0.95 1.00 0.98 40\n", " Insulator_Installation_level_1 0.96 0.96 0.96 54\n", " Insulator_Installation_level_2 0.93 1.00 0.96 38\n", " Insulator_Installation_level_3 1.00 0.96 0.98 27\n", " Insulator_Normal_normal 1.00 0.96 0.98 141\n", " LA_Arc_mark_level_1 1.00 1.00 1.00 4\n", " LA_Arc_mark_level_2 1.00 1.00 1.00 43\n", " LA_Arc_mark_level_3 0.99 0.97 0.98 76\n", " LA_Crack_level_1 0.92 1.00 0.96 11\n", " LA_Crack_level_2 0.95 1.00 0.98 40\n", " LA_Crack_level_3 0.97 0.95 0.96 65\n", " LA_Installation_level_1 1.00 1.00 1.00 14\n", " LA_Installation_level_2 0.91 0.97 0.94 33\n", " LA_Installation_level_3 0.97 0.97 0.97 60\n", " LA_Normal_normal 0.96 0.99 0.98 105\n", " LP_Crack_level_1 1.00 1.00 1.00 40\n", " LP_Crack_level_2 0.97 0.91 0.94 34\n", " LP_Crack_level_3 0.98 0.98 0.98 46\n", " LP_Installation_level_1 0.98 1.00 0.99 61\n", " LP_Installation_level_2 1.00 0.97 0.98 32\n", " LP_Installation_level_3 0.99 0.97 0.98 71\n", " LP_Normal_normal 0.98 0.97 0.98 131\n", " Pole_Crack_level_1 1.00 0.78 0.88 32\n", " Pole_Crack_level_2 0.95 0.95 0.95 39\n", " Pole_Crack_level_3 0.96 1.00 0.98 69\n", " Pole_Normal_normal 1.00 1.00 1.00 6\n", "\n", " accuracy 0.97 2000\n", " macro avg 0.97 0.97 0.97 2000\n", " weighted avg 0.97 0.97 0.97 2000\n", "\n", "ROC-AUC Score: 0.9854416425758674\n", "Mean Average Precision: 0.9457642273271785\n", "F1 Score: 0.9714765388965046\n" ] } ], "source": [ "# Initialize the LabelBinarizer\n", "lb = LabelBinarizer()\n", "y_true_bin = lb.fit_transform(y_true)\n", "y_pred_bin = lb.transform(y_pred)\n", "\n", "# Classification Report\n", "print(\"Classification Report:\")\n", "print(classification_report(y_true, y_pred))\n", "\n", "# ROC-AUC Score (one-vs-rest)\n", "roc_auc = roc_auc_score(y_true_bin, y_pred_bin, average='macro')\n", "print(f\"ROC-AUC Score: {roc_auc}\")\n", "\n", "# Mean Average Precision\n", "mean_ap = average_precision_score(y_true_bin, y_pred_bin, average='macro')\n", "print(f\"Mean Average Precision: {mean_ap}\")\n", "\n", "# F1 Score\n", "f1 = f1_score(y_true, y_pred, average='macro')\n", "print(f\"F1 Score: {f1}\")\n", "\n", "# Classification Report\n", "classification_rep = classification_report(y_true, y_pred)\n", "\n", "# Save to .txt file\n", "with open('performance_metrics.txt', 'w') as file:\n", " file.write(\"Classification Report:\\n\")\n", " file.write(classification_rep + \"\\n\")\n", " file.write(f\"ROC-AUC Score: {roc_auc}\\n\")\n", " file.write(f\"Mean Average Precision: {mean_ap}\\n\")\n", " file.write(f\"F1 Score: {f1}\\n\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Classification Report:\n", "Class Insulator_Normal_normal: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class LA_Installation_level_2: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class GS_Normal_normal: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class LP_Crack_level_2: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class Insulator_Installation_level_2: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class LA_Installation_level_1: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class LP_Crack_level_3: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class LP_Installation_level_1: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class LP_Normal_normal: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class LA_Installation_level_3: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class COS_Crack_level_3: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class COS_Crack_level_1: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class LA_Normal_normal: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class LA_Arc_mark_level_1: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class GS_Arc_mark_level_1: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class COS_Crack_level_2: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class LA_Crack_level_3: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class Pole_Normal_normal: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class COS_Normal_normal: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class COS_Fuse_link_corrosion_level_3: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class COS_Arc_mark_level_1: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class LP_Installation_level_2: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class COS_Arc_mark_level_3: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class LP_Crack_level_1: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class LA_Crack_level_2: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class LA_Crack_level_1: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class LA_Arc_mark_level_2: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class LP_Installation_level_3: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class COS_Fuse_link_corrosion_level_1: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class COS_Arc_mark_level_2: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class Pole_Crack_level_1: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class Pole_Crack_level_2: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class COS_Fuse_link_corrosion_level_2: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class LA_Arc_mark_level_3: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class Insulator_Installation_level_1: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "Class GS_Arc_mark_level_2: Precision: 1.00, Recall: 1.00, F1: 1.00\n", "ROC-AUC Score: 1.00\n", "Mean Average Precision: 1.00\n" ] } ], "source": [ "# # Calculate Precision, Recall, and F1 Score\n", "# def calculate_metrics(y_true, y_pred):\n", "# class_counts = defaultdict(lambda: {'tp': 0, 'fp': 0, 'fn': 0})\n", "# unique_classes = set(y_true).union(set(y_pred))\n", "\n", "# for true, pred in zip(y_true, y_pred):\n", "# if true == pred:\n", "# class_counts[true]['tp'] += 1\n", "# else:\n", "# class_counts[true]['fn'] += 1\n", "# class_counts[pred]['fp'] += 1\n", "\n", "# metrics = {}\n", "# for cls in unique_classes:\n", "# tp = class_counts[cls]['tp']\n", "# fp = class_counts[cls]['fp']\n", "# fn = class_counts[cls]['fn']\n", "# precision = tp / (tp + fp) if (tp + fp) > 0 else 0\n", "# recall = tp / (tp + fn) if (tp + fn) > 0 else 0\n", "# f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0\n", "# metrics[cls] = {'precision': precision, 'recall': recall, 'f1': f1}\n", " \n", "# return metrics\n", "\n", "# metrics = calculate_metrics(y_true, y_pred)\n", "# print(\"Classification Report:\")\n", "# for cls, met in metrics.items():\n", "# print(f\"Class {cls}: Precision: {met['precision']:.2f}, Recall: {met['recall']:.2f}, F1: {met['f1']:.2f}\")\n", "\n", "# # Calculate ROC-AUC Score (one-vs-rest)\n", "# def calculate_roc_auc(y_true, y_pred, unique_classes):\n", "# y_true_bin = {cls: [] for cls in unique_classes}\n", "# y_pred_bin = {cls: [] for cls in unique_classes}\n", "\n", "# for true, pred in zip(y_true, y_pred):\n", "# for cls in unique_classes:\n", "# y_true_bin[cls].append(1 if true == cls else 0)\n", "# y_pred_bin[cls].append(1 if pred == cls else 0)\n", "\n", "# def auc_score(y_true, y_pred):\n", "# pairs = sorted(zip(y_true, y_pred), key=lambda x: x[1], reverse=True)\n", "# pos = sum(y_true)\n", "# neg = len(y_true) - pos\n", "# tp = 0\n", "# fp = 0\n", "# auc = 0\n", "# for true, _ in pairs:\n", "# if true == 1:\n", "# tp += 1\n", "# else:\n", "# fp += 1\n", "# auc += tp\n", "# auc /= (pos * neg) if pos * neg > 0 else 1\n", "# return auc\n", "\n", "# roc_auc = {cls: auc_score(y_true_bin[cls], y_pred_bin[cls]) for cls in unique_classes}\n", "# mean_roc_auc = sum(roc_auc.values()) / len(unique_classes)\n", "# return mean_roc_auc\n", "\n", "# unique_classes = set(y_true).union(set(y_pred))\n", "# roc_auc = calculate_roc_auc(y_true, y_pred, unique_classes)\n", "# print(f\"ROC-AUC Score: {roc_auc:.2f}\")\n", "\n", "# # Calculate Mean Average Precision\n", "# def calculate_mean_ap(y_true, y_pred, unique_classes):\n", "# def average_precision(y_true, y_pred):\n", "# pairs = sorted(zip(y_true, y_pred), key=lambda x: x[1], reverse=True)\n", "# tp = 0\n", "# fp = 0\n", "# precisions = []\n", "# for true, _ in pairs:\n", "# if true == 1:\n", "# tp += 1\n", "# precisions.append(tp / (tp + fp))\n", "# else:\n", "# fp += 1\n", "# return sum(precisions) / len(precisions) if precisions else 0\n", "\n", "# y_true_bin = {cls: [] for cls in unique_classes}\n", "# y_pred_bin = {cls: [] for cls in unique_classes}\n", "\n", "# for true, pred in zip(y_true, y_pred):\n", "# for cls in unique_classes:\n", "# y_true_bin[cls].append(1 if true == cls else 0)\n", "# y_pred_bin[cls].append(1 if pred == cls else 0)\n", "\n", "# ap = {cls: average_precision(y_true_bin[cls], y_pred_bin[cls]) for cls in unique_classes}\n", "# mean_ap = sum(ap.values()) / len(unique_classes)\n", "# return mean_ap\n", "\n", "# mean_ap = calculate_mean_ap(y_true, y_pred, unique_classes)\n", "# print(f\"Mean Average Precision: {mean_ap:.2f}\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "mmpretrain", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.0" }, "orig_nbformat": 4 }, "nbformat": 4, "nbformat_minor": 2 }