"""Offline trials only. GPU sequential, original files/model mounts read-only.""" import concurrent.futures import datetime import hashlib import json import pathlib import subprocess import sys import time from collections import Counter LOCAL=pathlib.Path(__file__).resolve().parent.parent REMOTE='/storage/contextra/experiments/action-model-eval/2026/'+LOCAL.name BASE=LOCAL.parent PREP=BASE/'2026-10-26_contextra-camera24-prep_r01' RP='/storage/contextra/experiments/action-model-eval/2026/'+PREP.name SHORT='/storage/contextra/experiments/action-model-eval/2026/2026-10-27_contextra-short-input-7clips_r01' FOUR=BASE/'2026-10-27_contextra-four-scenarios-prep_r01' FALL=BASE/'2026-10-27_contextra-camera27-prep_r01' PREFIX='contextra-nonfight-r01-' CPU_IMAGE='sha256:671c7416f503481d2be2092a5aa0c035938a07fc514cfc0b45f94d78e86370bd' GPU_IMAGE='sha256:7c0fdf5e607a2a5eae998738327f8e4902b37caf6fe83acb832462fab21b9f4e' ENGINE='/storage/contextra/models/triton/RFDETRKeypointPreview/8' FILES={ '01_camera22_crowd':(FOUR/'camera22/inputs/camera22_original.mp4',[242,723],[242,483]), '02_camera27_fall':(FALL/'inputs/camera27_original.mp4',[362,1083],[362,723]), '03_camera29_fire':(FOUR/'camera29/inputs/camera29_original.mp4',[242,482],[242]), '04_KISA_C00_051_0002':(pathlib.Path('/data/playlist/uploads/260914_아이서티_컨텍스트라_테스트용_kisa_영상들_클립들/(검증용)kisa_data/쓰러짐/배포/clips/C00_051_0002_s230_d52_h264.mp4'),None,[])} REPORT=LOCAL/'reports';REPORT.mkdir(exist_ok=True) sys.path.insert(0,str(PREP/'helpers')) from remote import run,SSH def checked(args,**kwargs): p=run(args,**kwargs) if p.returncode:raise RuntimeError(p.stderr.decode()[-1500:]+p.stdout.decode()[-1000:]) return p.stdout def scp(src,dst): subprocess.run(['scp','-q','-r','-i',SSH[2],'-o',SSH[4],'-o','StrictHostKeyChecking=yes','-o','BatchMode=yes', '-o','ConnectTimeout=10','-o','ServerAliveInterval=15','-o','ServerAliveCountMax=3','-P','1022',src,dst], check=True,timeout=900) def resources(gpu=False,starting=False): code="import json,os,pathlib,shutil; m={l.split()[0].rstrip(':'):int(l.split()[1])*1024 for l in pathlib.Path('/proc/meminfo').read_text().splitlines() if len(l.split())>2 and l.split()[2]=='kB'};print(json.dumps(dict(root_free=shutil.disk_usage('/').free,storage_free=shutil.disk_usage('/storage').free,ram_available=m['MemAvailable'],cpus=os.cpu_count(),load=os.getloadavg())))" s=json.loads(checked(['python3','-c',code],timeout=40)) assert s['root_free']>=15*1024**3 and s['storage_free']>=20*1024**3 and s['ram_available']>=12*1024**3,s assert s['load'][0]=(8192 if starting else 4096),s return s def snapshot(label): raw=checked(['docker','exec','-i','contextra-ai2-analytics-1','python3','-'],data=(PREP/'helpers/snapshot.py').read_bytes(),timeout=90) state=json.loads(raw);assert state['health']['ready'] (REPORT/f'production_{label}.json').write_bytes(raw) (REPORT/f'containers_{label}.txt').write_bytes(checked(['docker','ps','--format','{{.Names}} {{.Image}} {{.ID}}'])) return state def remote_sources(): return {clip:('/storage/contextra/experiments/action-model-eval/2026/'+str(p.relative_to(BASE)) if clip!='04_KISA_C00_051_0002' else REMOTE+'/sources/'+p.name) for clip,(p,_,_) in FILES.items()} def sync(): # One bounded connection for this run's helper bundle. scp(str(LOCAL/'helpers'),'si@58.233.207.31:'+REMOTE+'/') def setup(): for script in ('test_contract.py','test_sampling.py'): p=subprocess.run([sys.executable,'-B',str(LOCAL/'helpers'/script),'-v'],capture_output=True,timeout=30) (REPORT/(script+'.txt')).write_bytes(p.stdout+p.stderr);assert p.returncode==0,p.stderr.decode() resources(gpu=True,starting=True) for image in (CPU_IMAGE,GPU_IMAGE):checked(['docker','image','inspect',image,'--format','{{.Id}}']) specs={};source_manifest=[] for clip,(path,bounds,loops) in FILES.items(): digest=hashlib.sha256(path.read_bytes()).hexdigest() if clip!='04_KISA_C00_051_0002': root=FALL if clip=='02_camera27_fall' else FOUR saved={f['path']:f['sha256'] for f in json.loads((root/'manifest.json').read_text())['files']} assert saved[str(path.relative_to(root))]==digest specs[clip]=dict(sha256=digest,display_bounds=bounds,known_loop_boundaries=loops) source_manifest.append(dict(clip=clip,source_file=path.name,sha256=digest,display_bounds=bounds, user_context='User-confirmed non-fight comparison; no agent semantic verification')) (LOCAL/'specs.json').write_text(json.dumps(specs,indent=2)) checked(['test','!','-e',REMOTE]);checked(['mkdir',REMOTE],sudo=True);checked(['chown','si:si',REMOTE],sudo=True);checked(['chmod','700',REMOTE]) dirs=['helpers','basehelpers','sources','download/validation'] for clip in FILES:dirs += [f'clips/{clip}/{d}' for d in ('inputs','reports','outputs','pose_models/RFDETRKeypointPreview/8')] checked(['mkdir','-p']+[REMOTE+'/'+d for d in dirs]);sync() for n in ('specs.json','checkpoint.ko.md'):scp(str(LOCAL/n),'si@58.233.207.31:'+REMOTE+'/') scp(str(BASE/'2026-10-26_contextra-camera24-eval_r01/helpers/evaluate.py'),'si@58.233.207.31:'+REMOTE+'/basehelpers/evaluate.py') scp(str(FILES['04_KISA_C00_051_0002'][0]),'si@58.233.207.31:'+REMOTE+'/sources/') for clip,path in remote_sources().items(): assert checked(['sha256sum',path]).decode().split()[0]==specs[clip]['sha256'] verify_setup(source_manifest) def verify_setup(source_manifest=None): if source_manifest is None: specs=json.loads((LOCAL/'specs.json').read_text()) source_manifest=[dict(clip=clip,source_file=path.name,sha256=specs[clip]['sha256'],display_bounds=bounds, user_context='User-confirmed non-fight comparison; no agent semantic verification') for clip,(path,bounds,_) in FILES.items()] reference=json.loads(checked(['cat',SHORT+'/clips/02_KISA_C00_005_0018/reports/pose_summary.json'])) assert reference['engine_sha256']=='952464a98eca457dd15006f30444f44455261f5bcd210ad7a9d1b3c9677e84de' assert checked(['sha256sum',ENGINE+'/model.plan'],sudo=True).decode().split()[0]==reference['engine_sha256'] for name,digest in reference['source_files'].items(): actual=checked(['docker','exec','contextra-ai2-analytics-1','sha256sum','/opt/contextra/app/'+name]).decode().split()[0] assert actual==digest,(name,actual,digest) assert checked(['sha256sum',RP+'/weights/posec3d.pth']).decode().split()[0]=='38db104be0139bc2d57d662031de2e59eaa5302b6d33548628de1bf4af2a0f9d' model_manifest=json.loads(checked(['cat',SHORT+'/download/00_model_manifest.json'])) model_manifest['current_trial']=dict(models_executed=['posec3d'],temporal_samples=16,pose_reference=reference, cpu_image=CPU_IMAGE,action_image=GPU_IMAGE,other_models_not_executed=True) for name,value in [('00_model_manifest.json',model_manifest),('00_source_manifest.json',source_manifest)]: p=LOCAL/name;p.write_text(json.dumps(value,indent=2));scp(str(p),'si@58.233.207.31:'+REMOTE+'/download/') scp(str(LOCAL/'00_README_ko.txt'),'si@58.233.207.31:'+REMOTE+'/download/') snapshot('before');print('SETUP_PASS: input, engine, code, checkpoint fingerprints match',flush=True) def args(stage,clip): gpu=stage in ('pose','evaluate');name=PREFIX+stage+'-'+clip.lower().replace('_','-') cmd=['docker','run','--rm','--pull','never','--name',name,'--user','0:0','--read-only','--network','none', '--cap-drop','ALL','--cap-add','DAC_OVERRIDE','--security-opt','no-new-privileges', '--cpus','2','--memory','8g','--pids-limit','256','--shm-size','256m','--tmpfs','/tmp:rw,size=512m', '-e','PYTHONDONTWRITEBYTECODE=1','-e','OMP_NUM_THREADS=2','-e','OPENBLAS_NUM_THREADS=2','-e','MPLCONFIGDIR=/tmp/mpl','-e','CLIP='+clip, '-v',REMOTE+'/clips/'+clip+':/eval','-v',REMOTE+'/helpers:/helpers:ro','-v',REMOTE+'/basehelpers:/basehelpers:ro', '-v',RP+':/work:ro','-v',REMOTE+'/specs.json:/specs.json:ro','-v',remote_sources()[clip]+':/source.mp4:ro'] if gpu:cmd+=['--gpus','device=0'] else:cmd+=['--runtime','runc','-e','NVIDIA_VISIBLE_DEVICES=void'] if stage=='pose':cmd+=['-v',ENGINE+':/eval/pose_models/RFDETRKeypointPreview/8:ro'] if stage=='render':cmd+=['-v',REMOTE+':/source:ro','-v',REMOTE+'/download:/download', '-v',RP+'/reports/classes_posec3d.csv:/classes.csv:ro','-v',SHORT+'/fonts/NotoSansCJK-Regular.ttc:/font.ttc:ro', '-v',REMOTE+'/expected_source_hashes.json:/expected.json:ro'] cmd+=['--entrypoint','python3',GPU_IMAGE if stage=='evaluate' else CPU_IMAGE,'/helpers/'+stage+'.py'] if stage=='evaluate':cmd+=['posec3d'] return name,cmd def trial(stage,clip): gpu=stage in ('pose','evaluate');resources(gpu=gpu,starting=gpu);name,cmd=args(stage,clip) print('START',stage,clip,flush=True);samples=[] with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: future=pool.submit(run,cmd,timeout=1800) try: while True: try:result=future.result(timeout=30);break except concurrent.futures.TimeoutError: if future.done():raise s=resources(gpu=gpu);samples.append(s) print('RESOURCE',stage,clip,s.get('gpu_free_mib'),flush=True) except BaseException: run(['docker','stop','--time','15',name],timeout=40);raise (REPORT/f'{stage}_{clip}.log').write_bytes(result.stdout+result.stderr) (REPORT/f'{stage}_{clip}_resources.json').write_text(json.dumps(samples,indent=2)) if result.returncode:raise RuntimeError(stage+'/'+clip+': '+result.stderr.decode()[-2000:]+result.stdout.decode()[-1200:]) print('DONE',stage,clip,result.stdout.decode()[-1000:],flush=True) def infer(): for clip in FILES: for stage in ('plan','pose','evaluate'):trial(stage,clip) snapshot('after_inference') expected={} for clip in FILES: for name,path in [('posec3d_raw.json',f'clips/{clip}/outputs/posec3d.json'), ('posec3d_summary.json',f'clips/{clip}/reports/posec3d_summary.json')]: expected[f'{clip}/data/{name}']=checked(['sha256sum',REMOTE+'/'+path]).decode().split()[0] expected[f'{clip}/data/classes_posec3d.csv']=checked(['sha256sum',RP+'/reports/classes_posec3d.csv']).decode().split()[0] for name in ('00_model_manifest.json','00_source_manifest.json'): expected[name]=hashlib.sha256((LOCAL/name).read_bytes()).hexdigest() p=LOCAL/'expected_source_hashes.json';p.write_text(json.dumps(expected,indent=2));scp(str(p),'si@58.233.207.31:'+REMOTE+'/') print('INFERENCE_COMPLETE',flush=True) def render_all(): with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool: jobs=[pool.submit(trial,'render',clip) for clip in FILES] for f in concurrent.futures.as_completed(jobs):f.result() if __name__=='__main__': stage=sys.argv[1] if stage=='setup':setup() elif stage=='verify-setup':verify_setup() elif stage=='sync':sync() elif stage=='infer':infer() elif stage=='render':render_all() elif stage=='one':trial(sys.argv[2],sys.argv[3])