{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "a6065b0a", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/root/anaconda3/envs/torch190-test/lib/python3.8/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", " from .autonotebook import tqdm as notebook_tqdm\n", "/tmp/ipykernel_2889656/1491415481.py:14: DeprecationWarning: `np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n", "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n", " from cython_bbox import bbox_overlaps as bbox_ious\n" ] } ], "source": [ "import torch\n", "import numpy as np\n", "import cv2\n", "import matplotlib.pyplot as plt\n", "from IPython.display import clear_output\n", "import os\n", "from PIL import Image\n", "import pandas as pd\n", "import time\n", "import natsort\n", "from tqdm import tqdm\n", "\n", "from multicam_utils import give_color\n", "from cython_bbox import bbox_overlaps as bbox_ious\n", "\n", "import cv2\n", "import time\n", "import torch\n", "import torch.nn.functional as F\n", "import argparse\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "from torchvision import transforms\n", "\n", "import copy\n", "import scipy\n", "import lap\n", "from scipy.spatial.distance import cdist\n", "import warnings\n", "\n", "%matplotlib inline\n", "warnings.filterwarnings(\"ignore\")" ] }, { "cell_type": "markdown", "id": "d6819de4", "metadata": {}, "source": [ "## Load obj in recent frames" ] }, { "cell_type": "code", "execution_count": 2, "id": "5e16dd14", "metadata": {}, "outputs": [], "source": [ "# load obj_img and calculate average feat\n", "\n", "# average feat based matching\n", "\n", "DISTANCE = 100\n", "\n", "# new id is in then get average\n", "\n", "tracking_result_path = '/data/cvprw/AIC23/tracking/SUBMISSION/S018'\n", "\n", "channel_list = []\n", "temp = [file for file in os.listdir(tracking_result_path)]\n", "for channel in temp:\n", " d = os.path.join(tracking_result_path, channel)\n", " if os.path.isdir(d):\n", "# print(channel)\n", " channel_list.append(channel)\n", " " ] }, { "cell_type": "code", "execution_count": 3, "id": "1958df76", "metadata": {}, "outputs": [], "source": [ "channel_list=natsort.natsorted(channel_list)\n" ] }, { "cell_type": "code", "execution_count": 4, "id": "12b15a3b", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['c100', 'c101', 'c102', 'c103', 'c104', 'c105']" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "del channel_list[0]\n", "channel_list" ] }, { "cell_type": "code", "execution_count": 5, "id": "6d0d43d1", "metadata": {}, "outputs": [], "source": [ "\n", "# Open two video capture objects\n", "cap1 = cv2.VideoCapture(os.path.join(tracking_result_path, channel_list[0], 'video.mp4'))\n", "cap2 = cv2.VideoCapture(os.path.join(tracking_result_path, channel_list[1], 'video.mp4'))\n", "cap3 = cv2.VideoCapture(os.path.join(tracking_result_path, channel_list[2], 'video.mp4'))\n", "cap4 = cv2.VideoCapture(os.path.join(tracking_result_path, channel_list[3], 'video.mp4'))\n", "cap5 = cv2.VideoCapture(os.path.join(tracking_result_path, channel_list[4], 'video.mp4'))\n", "cap6 = cv2.VideoCapture(os.path.join(tracking_result_path, channel_list[5], 'video.mp4'))\n", "\n", "_, frame1 = cap1.read()\n", "_, frame2 = cap2.read()\n", "_, frame3 = cap3.read()\n", "_, frame4 = cap4.read()\n", "_, frame5 = cap5.read()\n", "_, frame6 = cap6.read()\n", " \n", "with open(os.path.join(tracking_result_path, channel_list[0], 'label.txt')) as label1:\n", " gt1 = label1.readlines()\n", "with open(os.path.join(tracking_result_path, channel_list[1], 'label.txt')) as label2:\n", " gt2 = label2.readlines()\n", "with open(os.path.join(tracking_result_path, channel_list[2], 'label.txt')) as label3:\n", " gt3 = label3.readlines()\n", "with open(os.path.join(tracking_result_path, channel_list[3], 'label.txt')) as label4:\n", " gt4 = label4.readlines()\n", "with open(os.path.join(tracking_result_path, channel_list[4], 'label.txt')) as label5:\n", " gt5 = label5.readlines()\n", "with open(os.path.join(tracking_result_path, channel_list[5], 'label.txt')) as label6:\n", " gt6 = label6.readlines()\n", "\n" ] }, { "cell_type": "code", "execution_count": 6, "id": "b57613a6", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "18010\n" ] } ], "source": [ "\n", "width = cap1.get(cv2.CAP_PROP_FRAME_WIDTH) # float\n", "height = cap1.get(cv2.CAP_PROP_FRAME_HEIGHT) # float\n", "fps = cap1.get(cv2.CAP_PROP_FPS)\n", "vid_length = int(cap1.get(cv2.CAP_PROP_FRAME_COUNT))\n", "\n", "print(vid_length)" ] }, { "cell_type": "markdown", "id": "2f40821b-7ff8-4e7c-80de-811dda26c6e9", "metadata": {}, "source": [ "# Tools" ] }, { "cell_type": "code", "execution_count": 7, "id": "53bf4e9a", "metadata": {}, "outputs": [], "source": [ "def draw_bbox_tracking(img, results, thres):\n", " for obj in results:\n", " if obj[4]>thres:\n", " x = int(obj[0])\n", " y = int(obj[1])\n", " x2 = int(obj[2])\n", " y2 = int(obj[3])\n", " \n", " tracking_id = obj[5]\n", " \n", " color = give_color(tracking_id)\n", " \n", " tk = 3\n", " \n", " content = str(tracking_id)\n", " \n", " font = cv2.FONT_HERSHEY_PLAIN\n", " img = cv2.rectangle(img, (x,y), (x2,y2), color, tk) # bbox\n", " img = cv2.putText(img, content, (x, y-2), font, tk, color, tk, cv2.LINE_AA) # label\n", " return img\n", "\n", "\n", "def extract_tracking_result(gt, frame_count, W, H):\n", " track_bbox=[]\n", " for row in gt:\n", " if int(row.split(',')[0]) == frame_count:\n", " track_id=int(row.split(',')[1])\n", " \n", " x = int(max(0, int(float(row.split(',')[2]))))\n", " y = int(max(0, int(float(row.split(',')[3]))))\n", " if int(float(row.split(',')[4]))+x >W:\n", " obj_w=W-x-1\n", " else:\n", " obj_w=int(float(row.split(',')[4]))\n", " \n", " if int(float(row.split(',')[5]))+y >H:\n", " obj_h=H-y-1\n", " else:\n", " obj_h=int(float(row.split(',')[5]))\n", " \n", " track_bbox.append([x,y,x+obj_w,y+obj_h,float(row.split(',')[6]),track_id])\n", " \n", " elif int(row.split(',')[0]) > frame_count:\n", " break\n", " \n", " return track_bbox\n", "\n", "def plot_site(result_img1,result_img2,result_img3,result_img4,result_img5,map_img):\n", " fig = plt.figure(figsize=(20,8))\n", " rows = 2\n", " cols = 3\n", "\n", " ax1 = fig.add_subplot(rows, cols, 1)\n", " ax1.imshow(cv2.cvtColor(result_img1, cv2.COLOR_BGR2RGB))\n", " ax1.set_title('C025')\n", " ax1.axis(\"off\")\n", "\n", " ax2 = fig.add_subplot(rows, cols, 2)\n", " ax2.imshow(cv2.cvtColor(result_img2, cv2.COLOR_BGR2RGB))\n", " ax2.set_title('C026')\n", " ax2.axis(\"off\")\n", "\n", " ax3 = fig.add_subplot(rows, cols, 3)\n", " ax3.imshow(cv2.cvtColor(result_img3, cv2.COLOR_BGR2RGB))\n", " ax3.set_title('C027')\n", " ax3.axis(\"off\")\n", "\n", " ax4 = fig.add_subplot(rows, cols, 4)\n", " ax4.imshow(cv2.cvtColor(result_img4, cv2.COLOR_BGR2RGB))\n", " ax4.set_title('C028')\n", " ax4.axis(\"off\")\n", "\n", " ax5 = fig.add_subplot(rows, cols, 5)\n", " ax5.imshow(cv2.cvtColor(result_img5, cv2.COLOR_BGR2RGB))\n", " ax5.set_title('C029')\n", " ax5.axis(\"off\")\n", "\n", " ax6 = fig.add_subplot(rows, cols, 6)\n", " ax6.imshow(cv2.cvtColor(map_img, cv2.COLOR_BGR2RGB))\n", " ax6.set_title('MAP')\n", " ax6.axis(\"off\")\n", "\n", " plt.show()" ] }, { "cell_type": "code", "execution_count": 8, "id": "1ab18d4c", "metadata": {}, "outputs": [], "source": [ "def make_square(img):\n", " \"\"\"\n", " Given an image as a cv2 object, add padding to make it square.\n", " \"\"\"\n", " # Get the image dimensions\n", " height, width = img.shape[:2]\n", "\n", " # Determine the size of the square to make\n", " square_size = max(width, height)\n", "\n", " # Create a new image of the appropriate size, with a black background\n", " new_img = np.zeros((square_size, square_size, 3), dtype=np.uint8)\n", "\n", " # Determine where to paste the original image in the new image\n", " x_offset = (square_size - width) // 2\n", " y_offset = (square_size - height) // 2\n", "\n", " # Paste the original image in the center of the new image\n", " new_img[y_offset:y_offset+height, x_offset:x_offset+width] = img\n", "\n", " return new_img\n", "\n", "def extract_global_position2(frame_bbox, H):\n", " # format (x,y,raw_track_id)\n", " c_x = (frame_bbox[0]+frame_bbox[2])/2\n", " bottom_y = frame_bbox[1]+(frame_bbox[3]-frame_bbox[1])*0.95\n", "\n", " point1 = np.array([[c_x, bottom_y, 1]])\n", " point2 = np.dot(H, point1.T)\n", " point2 /= point2[2]\n", " \n", " return int(point2[0]),int(point2[1])\n", "\n", "def add_projected_point2(projected_point_group, obj_list, channel_idx, global_obj_mapping):\n", " # projected_point_group [[[],[]].[[]],...]\n", " # Fromat : ( ), )\n", " \n", " distance = DISTANCE # 두개 그룹 이상 매칭 되면 feat으로 한 그룹에 묶어줌\n", " \n", " if len(projected_point_group)==0:\n", " for row in obj_list:\n", " row.append(channel_idx)\n", " projected_point_group.append([row])\n", " else:\n", " for row in obj_list:\n", " match_idx = []\n", " for idx,group in enumerate(projected_point_group): # 클러스터링 된 수만큼 그룹수 존재 (idx)\n", " _, avg_x, avg_y = np.mean(np.array(group)[:,:3], axis=0)\n", " if (row[1] - avg_x)**2 + (row[2] - avg_y)**2 < distance**2: # distanec under 100pixel\n", " match_idx.append(idx)\n", "\n", " if len(match_idx)==1:\n", " row.append(channel_idx)\n", " projected_point_group[match_idx[0]].append(row)\n", " elif len(match_idx)>1:\n", " min_distance = 0.05\n", " match_id = None\n", " for group_id in match_idx:\n", " for obj in projected_point_group[group_id]:\n", " if embedding_distance(obj[3],row[3])0:\n", " ious = bbox_ious(\n", " np.ascontiguousarray(dets, dtype=np.float64),\n", " np.ascontiguousarray(dets, dtype=np.float64)\n", " )\n", "\n", " for iou_row in ious:\n", " if sum(iou_row)>1.05:\n", " occluded_val_list.append(True)\n", " else:\n", " occluded_val_list.append(False)\n", " \n", " return occluded_val_list\n", "\n", "def run_keypoint(frame_bbox, frame_img, keypoint_model, strong=True):\n", " thres = 0.5\n", " padding_size = 10\n", " \n", " # padding 40px and make square based on height\n", " padding_height = frame_bbox[3]-frame_bbox[1]+padding_size\n", " center_x = (frame_bbox[0]+frame_bbox[2])/2\n", "\n", " patch_x1 = int(max(0, frame_bbox[0]-padding_size))\n", " patch_y1 = int(max(0, frame_bbox[1]-padding_size))\n", " patch_x2 = int(min(W - 1, frame_bbox[2]+padding_size))\n", " patch_y2 = int(min(H - 1, frame_bbox[3]+padding_size))\n", "\n", " patch = frame_img[patch_y1:patch_y2, patch_x1:patch_x2, :]\n", " patch_squared = make_square(patch)\n", "\n", "# resized_patch = cv2.resize(patch, (256,256))\n", " square_size = 256\n", " resized_patch = cv2.resize(cv2.cvtColor(patch_squared, cv2.COLOR_BGR2RGB), (square_size,square_size))\n", " \n", " # %matplotlib inline\n", " # plt.imshow(resized_patch)\n", " # plt.show()\n", " \n", " resized_patch = torch.tensor(np.array([transforms.ToTensor()(resized_patch).numpy()]))\n", "\n", " resized_patch = resized_patch.to(\"cuda:0\") #convert image data to device\n", " resized_patch = resized_patch.float() #convert image to float precision (cpu)\n", "\n", " with torch.no_grad(): #get predictions\n", " output_data, _ = keypoint_model(resized_patch)\n", "\n", " output_data = non_max_suppression_kpt(output_data, #Apply non max suppression\n", " 0.25, # Conf. Threshold.\n", " 0.65, # IoU Threshold.\n", " nc=keypoint_model.yaml['nc'], # Number of classes.\n", " nkpt=keypoint_model.yaml['nkpt'], # Number of keypoints.\n", " kpt_label=True)\n", " output = output_to_keypoint(output_data)\n", " \n", " total_crop = frame_img[frame_bbox[1]:frame_bbox[3], frame_bbox[0]:frame_bbox[2], :]\n", "\n", " obj_width = frame_bbox[2] - frame_bbox[0]\n", " obj_height = frame_bbox[3] - frame_bbox[1]\n", " # 전체 감지 : 2, 감지안됨 : 0\n", " state_bit = 0\n", " \n", " max_area = 10\n", " for o in output:\n", " obj_area = o[4]*o[5]\n", " if (obj_area>max_area) and (o[54]>thres or o[57]>thres) and (o[18]>thres or o[21]>thres or o[24]>thres or o[27]>thres):\n", " max_area = obj_area\n", " state_bit = 2 \n", " \n", " \n", " # 전체 인데 반만 감지되거나, 아예 감지되지 않는 경우도 있음.\n", " #감지는 안되지만 가로세로 비로 추정\n", " # if not strong and obj_height>obj_width*2.5:\n", " # state_bit = 2\n", "\n", " return state_bit, total_crop\n", "\n", "def embedding_distance(tracks, detections, metric='cosine'):\n", " \"\"\"\n", " :param tracks: list[STrack]\n", " :param detections: list[BaseTrack]\n", " :param metric:\n", " :return: cost_matrix np.ndarray\n", " \"\"\"\n", "\n", " cost_matrix = np.zeros((len(tracks), len(detections)), dtype=np.float64)\n", " if cost_matrix.size == 0:\n", " return cost_matrix\n", "\n", " det_features = np.asarray([tracks], dtype=np.float64)\n", " track_features = np.asarray([detections], dtype=np.float64)\n", "\n", " cost_matrix = np.maximum(0.0, cdist(track_features, det_features, metric)) # / 2.0 # Nomalized features\n", " return cost_matrix\n", "\n", "def postprocess(features):\n", " # Normalize feature to compute cosine distance\n", " features = F.normalize(features)\n", " features = features.cpu().data.numpy()\n", " return features\n", "\n", "def extract_reid_feature(img, model):\n", " img = cv2.resize(img, tuple((100,300)), interpolation=cv2.INTER_LINEAR)\n", " \n", " patches = []\n", " \n", " patch_img = torch.as_tensor(img.astype(\"float32\").transpose(2, 0, 1))\n", " patch_img = patch_img.to(device=\"cuda:0\").half()\n", " patches.append(patch_img)\n", " \n", " if len(patches):\n", " patches = torch.stack(patches, dim=0)\n", "\n", " features = np.zeros((0, 2048))\n", " \n", " # Run model\n", " patches_ = torch.clone(patches)\n", " pred = model(patches)\n", " pred[torch.isinf(pred)] = 1.0\n", "\n", " feat = postprocess(pred)\n", "\n", " return feat[0]\n", " \n", "def check_occlusion(local_obj_mapping):\n", " \n", " occlusion_xy = [] \n", " \n", " distance = DISTANCE\n", " \n", " # ***한채널에 대해서 분석해서 글로벌 좌표 뽑음\n", " for channel_obj in local_obj_mapping:\n", " for obj1 in channel_obj:\n", " for obj2 in channel_obj:\n", " if (obj1 != obj2) and (obj1[-1] == 2 and obj2[-1] == 2) and ((obj1[1]-obj2[1])**2+(obj1[2]-obj2[2])**2) < distance**2:\n", " occlusion_xy.append((int((obj1[1]+obj2[1])/2), int((obj1[2]+obj2[2])/2)))\n", " \n", " # local -> [[],[],[],[],[]]\n", " for idx, channel_obj in enumerate(local_obj_mapping):\n", " for jdx, obj in enumerate(channel_obj):\n", " if obj[-1] == 2:\n", " for item in occlusion_xy:\n", " if (item[0]-obj[1])**2 + (item[1]-obj[2])**2 < (distance/2+10)**2:\n", " local_obj_mapping[idx][jdx][-1]=3\n", " \n", " return local_obj_mapping\n", "\n", "def update_feature(origin_feat_list, new_feat_list):\n", " origin_feat_list = list(origin_feat_list)\n", " new_feat_avg = np.mean(new_feat_list, axis=0)\n", " \n", " update_bit = False\n", " for idx, feat in enumerate(origin_feat_list):\n", " if embedding_distance(feat,new_feat_avg)<0.05:\n", " update_bit = True\n", " origin_feat_list[idx]= origin_feat_list[idx]*0.99 + new_feat_avg*0.01\n", " \n", " if not update_bit:\n", " origin_feat_list.append(new_feat_avg)\n", " \n", " new_feat_list = np.array(origin_feat_list)\n", "\n", " return new_feat_list" ] }, { "cell_type": "code", "execution_count": null, "id": "5dfadbde-f472-44a5-ab18-11754f7ff33e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "b66fcc80", "metadata": {}, "source": [ "# Camera Calibration" ] }, { "cell_type": "code", "execution_count": 9, "id": "d870dd94", "metadata": {}, "outputs": [], "source": [ "pts1 = np.array([(7, 563),(110, 563),(390, 563),(494, 563),(1159, 563),(1164, 563),(1545, 563),(1648, 563),(698,262),(1074,262),(1292,262)])\n", "pts2 = np.array([(304, 46),(304, 85),(304, 187),(304, 226),(304, 472),(304, 511),(304, 617),(304, 657),(810, 226),(810, 472),(810, 617)])\n", "H_c100, _ = cv2.findHomography(pts1, pts2, cv2.RANSAC, 5.0)\n", "\n", "pts1 = np.array([(27, 842),(203, 826),(635, 781),(786, 765),(1157, 728),(1288, 715),(1612, 682),(1727, 671),(585,380),(846,368),(464,446),(849,484)])\n", "pts2 = np.array([(304, 472),(304, 511),(304, 617),(304, 657),(304, 761),(304, 801),(304, 909),(304, 949),(810,617),(810,761),(677,555),(566,723)])\n", "H_c101, _ = cv2.findHomography(pts1, pts2, cv2.RANSAC, 5.0)\n", "\n", "pts1 = np.array([(655, 163),(649,175),(770, 175),(773, 163), (1134, 231),(1147, 255), (1271, 480),(1302, 535), (1412, 736),(1470, 840)])\n", "pts2 = np.array([(72, 36),(72,66),(135, 66),(135, 36), (304, 187),(304, 226), (304, 472),(304, 511), (304, 617),(304, 657)])\n", "H_c102, _ = cv2.findHomography(pts1, pts2, cv2.RANSAC, 5.0)\n", "\n", "pts1 = np.array([(738,1065),(1009,1054),(1703,1027),(558,271),(1162,262)])\n", "pts2 = np.array([(783,621),(783,681),(783,516),(304,761),(304,511)])\n", "H_c103, _ = cv2.findHomography(pts1, pts2, cv2.RANSAC, 5.0)\n", "\n", "pts1 = np.array([(216, 844),(366, 613),(541, 366),(1484, 347),(1646, 450),(1837, 578),(584,225)])\n", "pts2 = np.array([(681, 617),(553, 617),(304, 617),(304, 226),(439, 226),(553, 226),(0,636)])\n", "H_c104, _ = cv2.findHomography(pts1, pts2, cv2.RANSAC, 5.0)\n", "\n", "pts1 = np.array([(95,528),(570,540),(601,489),(668,383),(687,352),(729,284),(743,262),(1517,378)])\n", "pts2 = np.array([(439,472),(304,472),(304,511),(304,617),(304,657),(304,761),(304,801),(0,636)])\n", "H_c105, _ = cv2.findHomography(pts1, pts2, cv2.RANSAC, 5.0)\n", "\n", "\n", "H_list = [H_c100,H_c101,H_c102,H_c103,H_c104,H_c105]" ] }, { "cell_type": "markdown", "id": "ee620350", "metadata": {}, "source": [ "# Read Frame" ] }, { "cell_type": "code", "execution_count": 10, "id": "1550164e", "metadata": {}, "outputs": [], "source": [ "from models.experimental import attempt_load\n", "from utils.general import non_max_suppression_kpt,strip_optimizer,xyxy2xywh\n", "from utils.plots import output_to_keypoint, plot_skeleton_kpts,colors,plot_one_box_kpt" ] }, { "cell_type": "code", "execution_count": 11, "id": "7cb6d3c3", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Fusing layers... \n" ] } ], "source": [ "# Define MAX\n", "MAX_OBJ_NUM = 8\n", "\n", "poseweights = \"yolov7-w6-pose.pt\"\n", "device = \"cuda:0\"\n", "\n", "keypoint_model = attempt_load(poseweights, map_location=device) #Load model\n", "_ = keypoint_model.eval()" ] }, { "cell_type": "code", "execution_count": 12, "id": "f06a5aff-5704-4a83-b712-67f3f5fccbf5", "metadata": {}, "outputs": [], "source": [ "from fast_reid.fastreid.config import get_cfg\n", "from fast_reid.fastreid.modeling.meta_arch import build_model\n", "from fast_reid.fastreid.utils.checkpoint import Checkpointer\n", "from fast_reid.fastreid.engine import DefaultTrainer, default_argument_parser, default_setup, launch\n", "\n", "def setup_cfg(config_file, opts):\n", " # load config from file and command-line arguments\n", " cfg = get_cfg()\n", " cfg.merge_from_file(config_file)\n", " cfg.merge_from_list(opts)\n", " cfg.MODEL.BACKBONE.PRETRAIN = False\n", "\n", " cfg.freeze()\n", "\n", " return cfg\n", "\n", "config_file = '/data/cvprw/AIC23/BoT-SORT/logs/AIC23/sbs_S101/config.yaml'\n", "weights_path = '/data/cvprw/AIC23/BoT-SORT/logs/AIC23/sbs_S101/model_0099.pth'\n", "\n", "cfg = setup_cfg(config_file, ['MODEL.WEIGHTS', weights_path])\n", "\n", "reid_model = build_model(cfg)\n", "reid_model.eval()\n", "\n", "Checkpointer(reid_model).load(weights_path)\n", "\n", "reid_model = reid_model.eval().to(device='cuda:0').half()" ] }, { "cell_type": "code", "execution_count": 13, "id": "7a642bb6", "metadata": {}, "outputs": [], "source": [ "start = 0\n", "\n", "# channel_dic c025~c029\n", "# Format : ({channel},{},...) | channel -> ( : )\n", "channel_id_mapping = [{},{},{},{},{},{}]\n", "\n", "# global_dic\n", "# Format : ({id},{},...) | \n", "# id -> ( : , , [,...])\n", "global_obj_mapping = {}\n", "\n", "# loss for head only or occlusion\n", "# channel loss c025~c029\n", "# Format : channel_loss_mapping -> ([],[],...) | \n", "# channel -> ( )\n", "channel_loss_final_output = [[],[],[],[],[],[]]\n", "\n", "# Save Final Result\n", "# Format : channel_final_output -> ([channel],[],...)\n", "# channel -> (〈channel_idx〉 〈global_track_id〉 〈frame_id〉 〈xmin〉 〈ymin〉 〈width〉 〈height〉 〈xworld〉 〈yworld〉)\n", "# ==> \n", "channel_final_output = [[],[],[],[],[],[]]\n" ] }, { "cell_type": "code", "execution_count": 14, "id": "8c46b316-7307-47fe-9c1b-cd605644b517", "metadata": {}, "outputs": [], "source": [ "def add_local2gloabl(global_obj_mapping, channel_id_mapping, local_obj_mapping, new_id):\n", " \n", " local_channel_loss_mapping = [[],[],[],[],[],[]]\n", " # projected_point_group [[[],[]].[[]],...]\n", " # Fromat : (, , , , , )\n", " projected_point_group = []\n", " \n", " # local_obj_mapping\n", " # Format : ([channel],[],...) |\n", " # id -> ( )\n", " \n", " #1. Projection 기반 ID 생성\n", " for idx,channel_obj in enumerate(copy.deepcopy(local_obj_mapping)):\n", " temp = []\n", " for obj in channel_obj:\n", " if obj[-1]==2:\n", " temp.append(obj)\n", " projected_point_group = add_projected_point2(projected_point_group, temp, idx, global_obj_mapping)\n", " \n", " for group in projected_point_group:\n", " _, avg_x, avg_y = np.mean(np.array(group)[:,:3], axis=0)\n", " \n", " # 글로벌 좌표 추가, 여기에다가 feat도 추가해야함\n", " # ( : , , [,...])\n", " global_obj_mapping[new_id] = [int(avg_x), int(avg_y), np.array(group)[:,3]]\n", " \n", " # 각 채널 매핑\n", " for row in group:\n", " channel_id_mapping[row[-1]][row[0]] = new_id\n", " \n", " new_id+=1\n", " \n", " for idx,channel_obj in enumerate(copy.deepcopy(local_obj_mapping)):\n", " for obj in channel_obj:\n", " if obj[-1]!=2:\n", " local_channel_loss_mapping[idx].append(obj)\n", " \n", " return global_obj_mapping, channel_id_mapping, local_channel_loss_mapping, new_id\n", "\n", "\n", "\n", "def match_local2gloabl(global_obj_mapping, channel_id_mapping, local_obj_mapping):\n", " # 틀리거나 feature 안맞으면 loss에 추가, 이때 type은 2로 그대로 가져가서 재 배치가 가능하게끔 함. -> reid 사용 하고 실패하면 add\n", " local_channel_loss_mapping = [[],[],[],[],[],[]]\n", " \n", " origin_channel_id_mapping = copy.deepcopy(channel_id_mapping)\n", "\n", " projected_point_group = []\n", " # #1. Projection 기반 먼저추가\n", " for idx,channel_result in enumerate(copy.deepcopy(local_obj_mapping)):\n", " temp = []\n", " for obj in channel_result:\n", " if obj[-1]==2:\n", " temp.append(obj)\n", " projected_point_group = add_projected_point2(projected_point_group, temp, idx, global_obj_mapping)\n", " \n", " \n", " # 그룹간 FEAT 출력\n", " # print(\"그룹간 FEAT 출력\")\n", " # for group1 in projected_point_group:\n", " # for group2 in projected_point_group:\n", " # if group1 != group2:\n", " # print(embedding_distance(np.mean(np.array(group1)[:,3], axis=0),np.mean(np.array(group2)[:,3], axis=0)))\n", " \n", " \n", " # None 없음, 2에 대해서만\n", " matched_id_list = []\n", " for group in projected_point_group:\n", " _, avg_x, avg_y = np.mean(np.array(group)[:,:3], axis=0)\n", " \n", " # 이전 글로벌 매핑과 매칭\n", " # 글로벌 좌표 추가, 여기에다가 feat도 추가해야함 상,하체 모두\n", " already_matched_global_id = np.array(matched_id_list)\n", " \n", " defined_global_id = []\n", " for key in global_obj_mapping.keys():\n", " if global_obj_mapping[key][0] is not None: # None 이 아닌 x,y좌표애 대해서만\n", " defined_global_id.append(key)\n", " defined_global_id = np.array(defined_global_id)\n", " \n", " target_global_id = np.setdiff1d(defined_global_id, already_matched_global_id)\n", " \n", " match_id = None\n", " min_distance = DISTANCE**2\n", " \n", " for tg_id in target_global_id:\n", " row = global_obj_mapping[tg_id]\n", " frame_distance = (avg_x - row[0])**2 + (avg_y - row[1])**2\n", " if frame_distance 기존*0.9 + 새로운*0.1, 아니면 append\n", " origin_feat_list = global_obj_mapping[match_id][2]\n", " new_feat_list = update_feature(origin_feat_list, np.array(group)[:,3])\n", " # group 내 하나의 리스트 : \n", " \n", " global_obj_mapping[match_id] = [int(avg_x), int(avg_y), new_feat_list]\n", " for row in group: # 각 채널상에 나타난 새로운 id 추가\n", " channel_id_mapping[row[-1]][row[0]] = match_id\n", " \n", " else:\n", " print(\"채널간 매핑 정보가 다름\")\n", " # # print(channel_id_mapping)\n", " # for row in group: # 그룹안에 있는거는 모두 type2\n", " # if row[0] in list(channel_id_mapping[row[-1]].keys()):\n", " # if channel_id_mapping[row[-1]][row[0]] != match_id:\n", " # # print(\"channel\",row[-1],\"channel_track_id\",row[0])\n", " # # print(match_id)\n", " \n", " for row in group:\n", " local_channel_loss_mapping[row[-1]].append(list(np.array(row)[:-1]))\n", " channel_id_mapping[row[-1]].pop(row[0], None) # 채널 매핑 정보 지워줌\n", " # print(\"지워진 채널 매핑\")\n", " # print(channel_id_mapping)\n", " \n", " else: \n", " # print(\"매치 아이디가 none 임\")\n", " for row in group:\n", " local_channel_loss_mapping[row[-1]].append(list(np.array(row)[:-1])) \n", " \n", " # print(\"그룹핑 결과\")\n", " # for idx, group in enumerate(projected_point_group):\n", " # print(idx)\n", " # print(np.array(group))\n", " \n", " # 업데이트 안된 글로벌 좌표?? -> None\n", " for global_key in list(global_obj_mapping.keys()):\n", " if global_key not in matched_id_list:\n", " global_obj_mapping[global_key][0] = None\n", " global_obj_mapping[global_key][1] = None\n", "\n", " return global_obj_mapping, channel_id_mapping, local_channel_loss_mapping\n" ] }, { "cell_type": "code", "execution_count": 15, "id": "84f87f35-869e-46be-9320-43f5bfd06a6b", "metadata": {}, "outputs": [], "source": [ "def process_online(global_obj_mapping, channel_id_mapping, local_channel_loss_mapping, new_id):\n", " \n", " for idx,channel_obj in enumerate(local_channel_loss_mapping):\n", " for jdx,obj in enumerate(channel_obj):\n", " if obj[-1] == 2:\n", " defined_global_id = np.array(list(global_obj_mapping.keys())) # 모든 key\n", " \n", " target_global_id = defined_global_id\n", " \n", " # 자기 그룹내 re-id에 대해서, 글로벌 feature 들과 비교한 값중에 최솟값을 가지는 그룹이랑 글로벌 id 매칭\n", " # 단순 비교가 아닌 글로벌 feature와 현재 매핑되어 있는 feature간 거리 vs 글로벌 feature와 매핑할 feature간 거리\n", " min_distance = 0.05\n", " match_id = None\n", " \n", " temp = []\n", " \n", " for key in target_global_id:\n", " # 해당 글로벌 id 에 해당하는 obj가 존재한다면 -> 그것과 글로벌 좌표간 매칭 보다 작아야 하고\n", " row = global_obj_mapping[key]\n", " if row[0] is not None: # 현재 프레임에서 업데이트되었으니, obj 존재\n", " temp.append((obj[1] - row[0])**2 + (obj[2] - row[1])**2)\n", " if (obj[1] - row[0])**2 + (obj[2] - row[1])**2 < 330**2:\n", " # 새로운 OBJ 거리\n", " for feat1 in row[2]:\n", " now_dist = embedding_distance(np.array(feat1),np.array(obj[3]))\n", " if now_dist : , , [,...])\n", " global_obj_mapping[new_id] = [int(obj[1]), int(obj[2]), np.array([obj[3]])]\n", " channel_id_mapping[idx][row[0]] = new_id\n", "\n", " new_id+=1\n", " \n", " # 채널 로스 local 로 돌리고 type2인건 다 처리 했으니 return 할필요 없지 않나?\n", " return global_obj_mapping, channel_id_mapping, new_id\n", "\n", "\n", "def process_loss(channel_loss_final_output, channel_final_output, channel_id_mapping):\n", " \n", " new_channel_loss_final_output = [[],[],[],[],[],[]]\n", " \n", " for idx, channel_obj in enumerate(channel_loss_final_output):\n", " for obj in channel_obj: # obj -> (final_format , local_obj_format)\n", " channel_track_id = obj[1][0]\n", " if channel_track_id in channel_id_mapping[idx].keys():\n", " \n", " submit_format = obj[0]\n", " submit_format[1] = channel_id_mapping[idx][channel_track_id]\n", " \n", " channel_final_output[idx].append(submit_format)\n", " \n", " else:\n", " new_channel_loss_final_output[idx].append(obj)\n", " \n", " return new_channel_loss_final_output, channel_final_output\n", " " ] }, { "cell_type": "code", "execution_count": 16, "id": "0bfdb81d", "metadata": { "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ " 0%| | 1/18010 [00:02<10:28:09, 2.09s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[[], [], [], [], [], []]\n", "local_obj\n", "[list([[1, 31, 333, array([ 0, 0, 0, ..., 0.09314, 0, 0], dtype=float16), 2], [2, 268, 328, array([ 0, 0, 0, ..., 0.072754, 0, 0], dtype=float16), 2], [3, 350, 563, array([ 0, 0, 0, ..., 0.069824, 0, 0], dtype=float16), 2], [4, 745, 334, array([ 0, 0, 0, ..., 0.070251, 0, 0], dtype=float16), 2], [5, 577, 282, array([ 0, 0, 0, ..., 0.074341, 0, 0], dtype=float16), 2]])\n", " list([[1, 349, 556, array([ 0, 0, 0, ..., 0.078369, 0, 0], dtype=float16), 2], [2, 676, 556, array([ 0, 0, 0, ..., 0.069946, 0, 0], dtype=float16), 2]])\n", " list([[1, 27, 345, array([ 0, 0, 0, ..., 0.071899, 0, 0], dtype=float16), 2], [2, 268, 325, array([ 0, 0, 0, ..., 0.070129, 0, 0], dtype=float16), 2]])\n", " list([[1, 695, 550, array([ 0, 0, 0, ..., 0.071472, 0, 0], dtype=float16), 2], [2, 601, 688, array([ 0, 0, 0, ..., 0.075745, 0, 0], dtype=float16), 2], [3, 358, 557, array([ 0, 0, 0, ..., 0.070435, 0, 0], dtype=float16), 2]])\n", " list([[1, 677, 556, array([ 0, 0, 0, ..., 0.072021, 0, 0], dtype=float16), 2], [2, 732, 329, array([ 0, 0, 0, ..., 0.072205, 0, 0], dtype=float16), 2], [3, 568, 280, array([ 0, 0, 0, ..., 0.077393, 0, 0], dtype=float16), 2], [4, 265, 324, array([ 0, 0, 0, ..., 0.072815, 0, 0], dtype=float16), 2], [5, 347, 558, array([ 0, 0, 0, ..., 0.072144, 0, 0], dtype=float16), 2], [6, -30, 648, array([ 0, 0, 0, ..., 0.074097, 0, 0], dtype=float16), 2], [7, 26, 329, array([ 0, 0, 0, ..., 0.069519, 0, 0], dtype=float16), 2]])\n", " list([[1, 31, 333, array([ 0, 0, 0, ..., 0.070374, 0, 0], dtype=float16), 2], [2, 0, 649, array([ 0, 0, 0, ..., 0.073181, 0, 0], dtype=float16), 2]])]\n", "[{1: 100, 2: 101, 3: 102, 4: 103, 5: 104}, {1: 102, 2: 105}, {1: 100, 2: 101}, {3: 102, 1: 105, 2: 106}, {7: 100, 4: 101, 5: 102, 2: 103, 3: 104, 1: 105, 6: 107}, {1: 100, 2: 107}]\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 0%| | 2/18010 [00:04<10:47:41, 2.16s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[[], [], [], [], [], []]\n", "local_obj\n", "[list([[1, 31, 332, array([ 0, 0, 0, ..., 0.090759, 0, 0], dtype=float16), 2], [2, 268, 328, array([ 0, 0, 0, ..., 0.072632, 0, 0], dtype=float16), 2], [3, 350, 563, array([ 0, 0, 0, ..., 0.070251, 0, 0], dtype=float16), 2], [4, 745, 334, array([ 0, 0, 0, ..., 0.070862, 0, 0], dtype=float16), 2], [5, 577, 282, array([ 0, 0, 0, ..., 0.073914, 0, 0], dtype=float16), 2]])\n", " list([[1, 349, 556, array([ 0, 0, 0, ..., 0.069519, 0, 0], dtype=float16), 2], [2, 676, 555, array([ 0, 0, 0, ..., 0.069519, 0, 0], dtype=float16), 2]])\n", " list([[1, 27, 345, array([ 0, 0, 0, ..., 0.071899, 0, 0], dtype=float16), 2], [2, 268, 325, array([ 0, 0, 0, ..., 0.069885, 0, 0], dtype=float16), 2]])\n", " list([[1, 695, 550, array([ 0, 0, 0, ..., 0.071594, 0, 0], dtype=float16), 2], [2, 601, 688, array([ 0, 0, 0, ..., 0.076416, 0, 0], dtype=float16), 2], [3, 358, 557, array([ 0, 0, 0, ..., 0.070251, 0, 0], dtype=float16), 2]])\n", " list([[1, 677, 556, array([ 0, 0, 0, ..., 0.072266, 0, 0], dtype=float16), 2], [2, 732, 329, array([ 0, 0, 0, ..., 0.072754, 0, 0], dtype=float16), 2], [3, 568, 280, array([ 0, 0, 0, ..., 0.078674, 0, 0], dtype=float16), 2], [4, 265, 324, array([ 0, 0, 0, ..., 0.072693, 0, 0], dtype=float16), 2], [5, 346, 558, array([ 0, 0, 0, ..., 0.071106, 0, 0], dtype=float16), 2], [6, -30, 648, array([ 0, 0, 0, ..., 0.074219, 0, 0], dtype=float16), 2], [7, 26, 329, array([ 0, 0, 0, ..., 0.069458, 0, 0], dtype=float16), 2]])\n", " list([[1, 31, 333, array([ 0, 0, 0, ..., 0.070251, 0, 0], dtype=float16), 2], [2, 0, 649, array([ 0, 0, 0, ..., 0.073181, 0, 0], dtype=float16), 2]])]\n", "[{1: 100, 2: 101, 3: 102, 4: 103, 5: 104}, {1: 102, 2: 105}, {1: 100, 2: 101}, {3: 102, 1: 105, 2: 106}, {7: 100, 4: 101, 5: 102, 2: 103, 3: 104, 1: 105, 6: 107}, {1: 100, 2: 107}]\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 0%| | 3/18010 [00:06<11:12:18, 2.24s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[[], [], [], [], [], []]\n", "local_obj\n", "[list([[1, 32, 332, array([ 0, 0, 0, ..., 0.088684, 0, 0], dtype=float16), 2], [2, 268, 328, array([ 0, 0, 0, ..., 0.073059, 0, 0], dtype=float16), 2], [3, 349, 562, array([ 0, 0, 0, ..., 0.07019, 0, 0], dtype=float16), 2], [4, 745, 334, array([ 0, 0, 0, ..., 0.071228, 0, 0], dtype=float16), 2], [5, 577, 282, array([ 0, 0, 0, ..., 0.074524, 0, 0], dtype=float16), 2]])\n", " list([[1, 350, 556, array([ 0, 0, 0, ..., 0.082947, 0, 0], dtype=float16), 2], [2, 678, 555, array([ 0, 0, 0, ..., 0.07019, 0, 0], dtype=float16), 2]])\n", " list([[1, 27, 345, array([ 0, 0, 0, ..., 0.071228, 0, 0], dtype=float16), 2], [2, 268, 325, array([ 0, 0, 0, ..., 0.070251, 0, 0], dtype=float16), 2]])\n", " list([[1, 695, 550, array([ 0, 0, 0, ..., 0.071716, 0, 0], dtype=float16), 2], [2, 601, 688, array([ 0, 0, 0, ..., 0.077881, 0, 0], dtype=float16), 2], [3, 358, 557, array([ 0, 0, 0, ..., 0.070679, 0, 0], dtype=float16), 2]])\n", " list([[1, 677, 556, array([ 0, 0, 0, ..., 0.071899, 0, 0], dtype=float16), 2], [2, 732, 329, array([ 0, 0, 0, ..., 0.072327, 0, 0], dtype=float16), 2], [3, 568, 280, array([ 0, 0, 0, ..., 0.080017, 0, 0], dtype=float16), 2], [4, 265, 324, array([ 0, 0, 0, ..., 0.072876, 0, 0], dtype=float16), 2], [5, 346, 558, array([ 0, 0, 0, ..., 0.073242, 0, 0], dtype=float16), 2], [6, -30, 648, array([ 0, 0, 0, ..., 0.075317, 0, 0], dtype=float16), 2], [7, 26, 329, array([ 0, 0, 0, ..., 0.069397, 0, 0], dtype=float16), 2]])\n", " list([[1, 30, 334, array([ 0, 0, 0, ..., 0.071045, 0, 0], dtype=float16), 2], [2, 0, 649, array([ 0, 0, 0, ..., 0.072815, 0, 0], dtype=float16), 2]])]\n", "[{1: 100, 2: 101, 3: 102, 4: 103, 5: 104}, {1: 102, 2: 105}, {1: 100, 2: 101}, {3: 102, 1: 105, 2: 106}, {7: 100, 4: 101, 5: 102, 2: 103, 3: 104, 1: 105, 6: 107}, {1: 100, 2: 107}]\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 0%| | 4/18010 [00:08<11:08:27, 2.23s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[[], [], [], [], [], []]\n", "local_obj\n", "[list([[1, 32, 332, array([ 0, 0, 0, ..., 0.09314, 0, 0], dtype=float16), 2], [2, 267, 328, array([ 0, 0, 0, ..., 0.073364, 0, 0], dtype=float16), 2], [3, 349, 562, array([ 0, 0, 0, ..., 0.069946, 0, 0], dtype=float16), 2], [4, 745, 334, array([ 0, 0, 0, ..., 0.071594, 0, 0], dtype=float16), 2], [5, 577, 282, array([ 0, 0, 0, ..., 0.07135, 0, 0], dtype=float16), 2]])\n", " list([[1, 350, 556, array([ 0, 0, 0, ..., 0.083557, 0, 0], dtype=float16), 2], [2, 678, 555, array([ 0, 0, 0, ..., 0.070007, 0, 0], dtype=float16), 2]])\n", " list([[1, 27, 345, array([ 0, 0, 0, ..., 0.071594, 0, 0], dtype=float16), 2], [2, 268, 325, array([ 0, 0, 0, ..., 0.069336, 0, 0], dtype=float16), 2]])\n", " list([[1, 695, 550, array([ 0, 0, 0, ..., 0.071411, 0, 0], dtype=float16), 2], [2, 602, 689, array([ 0, 0, 0, ..., 0.073242, 0, 0], dtype=float16), 2], [3, 358, 557, array([ 0, 0, 0, ..., 0.071106, 0, 0], dtype=float16), 2]])\n", " list([[1, 677, 556, array([ 0, 0, 0, ..., 0.071655, 0, 0], dtype=float16), 2], [2, 732, 329, array([ 0, 0, 0, ..., 0.072266, 0, 0], dtype=float16), 2], [3, 568, 280, array([ 0, 0, 0, ..., 0.09491, 0, 0], dtype=float16), 2], [4, 265, 324, array([ 0, 0, 0, ..., 0.072937, 0, 0], dtype=float16), 2], [5, 346, 558, array([ 0, 0, 0, ..., 0.071167, 0, 0], dtype=float16), 2], [6, -33, 648, array([ 0, 0, 0, ..., 0.076111, 0, 0], dtype=float16), 2], [7, 26, 329, array([ 0, 0, 0, ..., 0.069214, 0, 0], dtype=float16), 2]])\n", " list([[1, 30, 334, array([ 0, 0, 0, ..., 0.071594, 0, 0], dtype=float16), 2], [2, 0, 650, array([ 0, 0, 0, ..., 0.07312, 0, 0], dtype=float16), 2]])]\n", "[{1: 100, 2: 101, 3: 102, 4: 103, 5: 104}, {1: 102, 2: 105}, {1: 100, 2: 101}, {3: 102, 1: 105, 2: 106}, {7: 100, 4: 101, 5: 102, 2: 103, 3: 104, 1: 105, 6: 107}, {1: 100, 2: 107}]\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 0%| | 5/18010 [00:11<11:50:43, 2.37s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[[], [], [], [], [], []]\n", "local_obj\n", "[list([[1, 33, 333, array([ 0, 0, 0, ..., 0.087952, 0, 0], dtype=float16), 2], [2, 268, 328, array([ 0, 0, 0, ..., 0.073059, 0, 0], dtype=float16), 2], [3, 350, 563, array([ 0, 0, 0, ..., 0.069885, 0, 0], dtype=float16), 2], [4, 745, 334, array([ 0, 0, 0, ..., 0.075989, 0, 0], dtype=float16), 2], [5, 577, 282, array([ 0, 0, 0, ..., 0.07489, 0, 0], dtype=float16), 2]])\n", " list([[1, 349, 557, array([ 0, 0, 0, ..., 0.083496, 0, 0], dtype=float16), 2], [2, 678, 555, array([ 0, 0, 0, ..., 0.069885, 0, 0], dtype=float16), 2]])\n", " list([[1, 27, 345, array([ 0, 0, 0, ..., 0.070557, 0, 0], dtype=float16), 2], [2, 268, 326, array([ 0, 0, 0, ..., 0.069458, 0, 0], dtype=float16), 2], [3, 349, 548, array([ 0, 0, 0, ..., 0.073486, 0, 0], dtype=float16), 2]])\n", " list([[1, 695, 550, array([ 0, 0, 0, ..., 0.071411, 0, 0], dtype=float16), 2], [2, 602, 689, array([ 0, 0, 0, ..., 0.074341, 0, 0], dtype=float16), 2], [3, 358, 557, array([ 0, 0, 0, ..., 0.072449, 0, 0], dtype=float16), 2]])\n", " list([[1, 677, 557, array([ 0, 0, 0, ..., 0.071289, 0, 0], dtype=float16), 2], [2, 732, 328, array([ 0, 0, 0, ..., 0.071777, 0, 0], dtype=float16), 2], [3, 569, 280, array([ 0, 0, 0, ..., 0.10718, 0, 0], dtype=float16), 2], [4, 265, 324, array([ 0, 0, 0, ..., 0.072998, 0, 0], dtype=float16), 2], [5, 346, 558, array([ 0, 0, 0, ..., 0.070862, 0, 0], dtype=float16), 2], [6, -33, 648, array([ 0, 0, 0, ..., 0.07428, 0, 0], dtype=float16), 2], [7, 29, 329, array([ 0, 0, 0, ..., 0.070496, 0, 0], dtype=float16), 2]])\n", " list([[1, 31, 335, array([ 0, 0, 0, ..., 0.075928, 0, 0], dtype=float16), 2], [2, 0, 649, array([ 0, 0, 0, ..., 0.07373, 0, 0], dtype=float16), 2], [3, 267, 328, array([ 0, 0, 0, ..., 0.07428, 0, 0], dtype=float16), 2]])]\n", "[{1: 100, 2: 101, 3: 102, 4: 103, 5: 104}, {1: 102, 2: 105}, {1: 100, 2: 101, 3: 102}, {3: 102, 1: 105, 2: 106}, {7: 100, 4: 101, 5: 102, 2: 103, 3: 104, 1: 105, 6: 107}, {1: 100, 2: 107, 3: 101}]\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 0%| | 6/18010 [00:14<12:40:54, 2.54s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[[], [], [], [], [], []]\n", "local_obj\n", "[list([[1, 34, 334, array([ 0, 0, 0, ..., 0.085571, 0, 0], dtype=float16), 2], [2, 269, 330, array([ 0, 0, 0, ..., 0.074646, 0, 0], dtype=float16), 2], [3, 351, 563, array([ 0, 0, 0, ..., 0.070496, 0, 0], dtype=float16), 2], [4, 747, 334, array([ 0, 0, 0, ..., 0.076965, 0, 0], dtype=float16), 2], [5, 576, 282, array([ 0, 0, 0, ..., 0.080933, 0, 0], dtype=float16), 2]])\n", " list([[1, 349, 558, array([ 0, 0, 0, ..., 0.076355, 0, 0], dtype=float16), 2], [2, 678, 555, array([ 0, 0, 0, ..., 0.07019, 0, 0], dtype=float16), 2]])\n", " list([[1, 27, 344, array([ 0, 0, 0, ..., 0.072937, 0, 0], dtype=float16), 2], [2, 268, 328, array([ 0, 0, 0, ..., 0.070007, 0, 0], dtype=float16), 2], [3, 349, 548, array([ 0, 0, 0, ..., 0.070923, 0, 0], dtype=float16), 2]])\n", " list([[1, 695, 550, array([ 0, 0, 0, ..., 0.071045, 0, 0], dtype=float16), 2], [2, 602, 689, array([ 0, 0, 0, ..., 0.07428, 0, 0], dtype=float16), 2], [3, 356, 558, array([ 0, 0, 0, ..., 0.072327, 0, 0], dtype=float16), 2]])\n", " list([[1, 677, 557, array([ 0, 0, 0, ..., 0.071106, 0, 0], dtype=float16), 2], [2, 731, 328, array([ 0, 0, 0, ..., 0.072693, 0, 0], dtype=float16), 2], [3, 568, 280, array([ 0, 0, 0, ..., 0.1106, 0, 0], dtype=float16), 2], [4, 266, 325, array([ 0, 0, 0, ..., 0.07312, 0, 0], dtype=float16), 2], [5, 347, 558, array([ 0, 0, 0, ..., 0.074341, 0, 0], dtype=float16), 2], [6, -33, 648, array([ 0, 0, 0, ..., 0.072083, 0, 0], dtype=float16), 2], [7, 26, 330, array([ 0, 0, 0, ..., 0.071716, 0, 0], dtype=float16), 2]])\n", " list([[1, 31, 335, array([ 0, 0, 0, ..., 0.070435, 0, 0], dtype=float16), 2], [2, 0, 650, array([ 0, 0, 0, ..., 0.072815, 0, 0], dtype=float16), 2], [3, 267, 326, array([ 0, 0, 0, ..., 0.072693, 0, 0], dtype=float16), 2]])]\n", "[{1: 100, 2: 101, 3: 102, 4: 103, 5: 104}, {1: 102, 2: 105}, {1: 100, 2: 101, 3: 102}, {3: 102, 1: 105, 2: 106}, {7: 100, 4: 101, 5: 102, 2: 103, 3: 104, 1: 105, 6: 107}, {1: 100, 2: 107, 3: 101}]\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n", "KeyboardInterrupt\n", "\n" ] } ], "source": [ "frame_count = 0\n", "end = 50\n", "new_id = 100\n", "\n", "gts=[gt1,gt2,gt3,gt4,gt5,gt6]\n", "\n", "t = tqdm(total=vid_length)\n", "\n", "while frame1 is not None:\n", " # read GT\n", " H, W, _ = np.shape(frame1)\n", " \n", " # format (x,y,x2,y2,det_score,track_id)\n", " track_bbox1 = extract_tracking_result(gt1, frame_count, W, H)\n", " track_bbox2 = extract_tracking_result(gt2, frame_count, W, H)\n", " track_bbox3 = extract_tracking_result(gt3, frame_count, W, H)\n", " track_bbox4 = extract_tracking_result(gt4, frame_count, W, H)\n", " track_bbox5 = extract_tracking_result(gt5, frame_count, W, H)\n", " track_bbox6 = extract_tracking_result(gt6, frame_count, W, H)\n", " \n", " track_result = [track_bbox1,track_bbox2,track_bbox3,track_bbox4,track_bbox5,track_bbox6]\n", " channel_frame = [frame1,frame2,frame3,frame4,frame5,frame6]\n", " \n", " # (1) 혼자인지 겹치는지 | 채널 하나에 대하여 각각 분석\n", " # 채널 ID는 0,1,2,3,4로 통일\n", " \n", " # Format : ([channel_id],[],...) |\n", " # channel_id -> \n", " local_obj_mapping = [[],[],[],[],[],[]]\n", " # local_channel_loss_mapping = [[],[],[],[],[],[]]\n", " \n", " \n", " for idx,track_bbox in enumerate(track_result):\n", " if len(track_bbox)>0:\n", " occlusion_check=make_occlusion_matrix(np.array(track_bbox)[:,:4])\n", " for jdx,v in enumerate(occlusion_check):\n", " frame_img = channel_frame[idx]\n", " H, W, _ = np.shape(frame_img)\n", " # format : x1,y1,x2,y2,score,track_id\n", " frame_bbox = track_bbox[jdx] \n", " frame_img = np.array(channel_frame[idx])\n", "\n", " if not v: # 겹치지 않음\n", " # 2: 전체, 1: 상체, 0:unkwon\n", " if frame_count == 0:\n", " state_bit, total_crop = run_keypoint(frame_bbox, frame_img, keypoint_model, strong=False)\n", " else:\n", " state_bit, total_crop = run_keypoint(frame_bbox, frame_img, keypoint_model)\n", " \n", " reid_feature_total = extract_reid_feature(total_crop, reid_model)\n", " \n", " if state_bit==2: # 다보이는 경우 \n", " projected_x, projected_y = extract_global_position2(frame_bbox, H_list[idx]) \n", " # \n", " local_obj_mapping[idx].append([frame_bbox[5], projected_x, projected_y, reid_feature_total, 2]) \n", "\n", " else: # Unkown \n", " local_obj_mapping[idx].append([frame_bbox[5], None, None, reid_feature_total,0])\n", "\n", " else : # 겹침 \n", " local_obj_mapping[idx].append([frame_bbox[5], None, None, None, 3])\n", " \n", " \n", " \n", " # print(np.array(local_obj_mapping))\n", " # local_obj_mapping 객체 중에서 근처로 접근 하는것 같으면 -> 해당 객체는 겹침으로 간주 class 3\n", " local_obj_mapping = check_occlusion(local_obj_mapping)\n", " \n", " # Control type 2\n", " if frame_count == 0:\n", " global_obj_mapping, channel_id_mapping, local_channel_loss_mapping, new_id = add_local2gloabl(global_obj_mapping, channel_id_mapping, local_obj_mapping, new_id)\n", " else:\n", " global_obj_mapping, channel_id_mapping, local_channel_loss_mapping = match_local2gloabl(global_obj_mapping, channel_id_mapping, local_obj_mapping)\n", " \n", " # print(\"전체 OBJ\")\n", " # print(np.array(local_obj_mapping))\n", " \n", " # print(\"매핑된 channel\")\n", " # print(np.array(channel_id_mapping))\n", " \n", "# print(\"loss된 channel\")\n", "# print(np.array(local_channel_loss_mapping))\n", " \n", " # print(np.array(global_obj_mapping))\n", " # loss 를 처리해야함.\n", " \n", " # loss type 0: 전체 다 안보임, 3: 겹침 -> 있으면 그냥 쓰고 없으면 channel_loss_mapping에 추가. -> final 쓸때 처리 ***\n", " # type 2: 모델이 100% 정확한건 아니기 때문에 생기는 문제 ( 트래킹 문제일 수도 있고 감지 문제일 수도 있고) 모름 -> reid 사용하고 실패하면 new id 부여\n", " \n", " # online -> type2 번 loss 처리 (실시간) | local_channel_loss_mapping\n", " global_obj_mapping, channel_id_mapping, new_id = process_online(global_obj_mapping, channel_id_mapping, local_channel_loss_mapping, new_id)\n", " \n", " \n", " # local_channel_loss_mapping \n", " # channel_loss_mapping\n", " # local_obj_mapping\n", " # Format : ([channel_id],[],...) |\n", " # channel_id -> \n", " \n", "\n", " # local_obj 우선 쓰고\n", " for i in range (len(channel_final_output)):\n", " for row in track_result[i]:\n", " channel_track_id = row[-1]\n", " # print(\"채널 아이디\",i,\"채널 트랙 아이디\",channel_track_id)\n", " \n", " if channel_track_id in channel_id_mapping[i].keys(): # 매핑 성공 시 add \n", " global_track_id = channel_id_mapping[i][channel_track_id]\n", " channel_final_output[i].append([i, global_track_id , frame_count , row[0] , row[1] , row[2]-row[0] , row[3]-row[1] , \n", " global_obj_mapping[global_track_id][0], global_obj_mapping[global_track_id][1]])\n", " else: # 매칭 실패한 현재 프레임 obj\n", " temp = None\n", " for obj in local_obj_mapping[i]:\n", " if obj[0] == channel_track_id:\n", " temp = obj \n", " channel_loss_final_output[i].append(([i, None , frame_count , row[0] , row[1] , row[2]-row[0] , row[3]-row[1] , \n", " -1, -1], temp)) # <- 추후에 매칭 기다리거나 or 매칭 안되면 offline 매칭 수행\n", " \n", " # print(local_channel_loss_mapping)\n", " print(channel_loss_final_output)\n", " print(\"local_obj\")\n", " print(np.array(local_obj_mapping))\n", " print(channel_id_mapping)\n", " \n", " # 저장된 loss_channel_mappiing에 대하여 새롭게 찾은게 있나 \n", " channel_loss_final_output, channel_final_output = process_loss(channel_loss_final_output, channel_final_output, channel_id_mapping)\n", " \n", " frame_count+=1\n", " t.update(1)\n", " \n", " # %matplotlib inline\n", " # plot_site(frame1,frame2,frame3,frame4,frame5,frame6)\n", " \n", " if frame_count>end:\n", " print(\"stop\")\n", " break\n", " \n", " _, frame1 = cap1.read()\n", " _, frame2 = cap2.read()\n", " _, frame3 = cap3.read()\n", " _, frame4 = cap4.read()\n", " _, frame5 = cap5.read()\n", " _, frame6 = cap6.read()\n", "\n", "t.close()" ] }, { "cell_type": "code", "execution_count": null, "id": "6d4dfbab-653c-4e7c-b9b0-f2a92b3010ab", "metadata": {}, "outputs": [], "source": [ "# track re-id 기반 매칭 수행\n", "# channel_loss_final_output" ] }, { "cell_type": "code", "execution_count": null, "id": "4d8f20ce-b8b5-4c28-be50-33a46248a05e", "metadata": {}, "outputs": [], "source": [ "# 마지막 loss_obj 강제 매칭\n", "# channel_final_output,channel_loss_mapping, channel_id_mapping=process_offline(channel_final_output, channel_loss_mapping, channel_id_mapping, global_obj_mapping, gts, H_list, W, H)" ] }, { "cell_type": "markdown", "id": "bc5fc3f8", "metadata": {}, "source": [ "# Sort and Make TXT" ] }, { "cell_type": "code", "execution_count": null, "id": "c98e8e9c-6c26-45a7-9b81-da951374a297", "metadata": {}, "outputs": [], "source": [ "np.array(channel_final_output[0]).shape" ] }, { "cell_type": "code", "execution_count": null, "id": "aacd039b-3de7-48e8-a32d-08394a84f09b", "metadata": {}, "outputs": [], "source": [ "# SAVE_PATH = '/data/cvprw/AIC23/tracking/SUBMISSION/S022'\n", "\n", "# # just save\n", "# for idx, channel_result in enumerate(channel_final_output):\n", "# # Open a file for writing\n", "# with open(os.path.join(SAVE_PATH,str(idx)+'_associated.txt'), 'w') as f:\n", "# # Loop through the rows of the array and write them to the file\n", "# for row in channel_result:\n", "# # Convert the row to a string and add a newline character\n", "# row_str = ','.join(str(elem) for elem in row) + '\\n'\n", "# f.write(row_str)\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "id": "2ac53167-2fc1-4d05-b4d3-790044ac7c5f", "metadata": {}, "outputs": [], "source": [ "# # save after sort\n", "# # just save\n", "# for idx, channel_result in enumerate(channel_final_output):\n", "# sort_indices = np.argsort(np.array(channel_result)[:, 2])\n", "# sorted_channel_result = list(np.array(channel_result)[sort_indices])\n", " \n", " \n", " \n", "# # Open a file for writing\n", "# with open(os.path.join(tracking_result_path, channel_list[idx],'sorted_associated.txt'), 'w') as f:\n", "# # Loop through the rows of the array and write them to the file\n", "# for row in sorted_channel_result:\n", "# # Convert the row to a string and add a newline character\n", "# row_str = ','.join(str(elem) for elem in row) + '\\n'\n", "# f.write(row_str)\n" ] }, { "cell_type": "markdown", "id": "bd5dbb22-5503-48ad-ba22-873be2b83375", "metadata": {}, "source": [ "# Save Vid" ] }, { "cell_type": "code", "execution_count": null, "id": "2a2d2487-35d2-459d-8d86-f05bd1f2139a", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "torch190-test", "language": "python", "name": "torch190-test" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.0" } }, "nbformat": 4, "nbformat_minor": 5 }